commit e2561cb287796bd161648d3879ec562d17fbb623 Author: Mike Perry mikeperry+git@torproject.org Date: Wed May 27 16:05:21 2015 -0700

Create option to add in exits if we don't have enough in a slice. --- PathSupport.py | 44 +++++++++++++++++++++++++++++++++++++++++++- SQLSupport.py | 10 ++++++++-- ScanSupport.py | 7 +++++-- 3 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/PathSupport.py b/PathSupport.py index c9c5ee1..8d86d3f 100644 --- a/PathSupport.py +++ b/PathSupport.py @@ -317,6 +317,20 @@ class ConserveExitsRestriction(NodeRestriction): def __str__(self): return self.__class__.__name__+"()"

+class ExitPortRestriction(NodeRestriction): + "Restriction to select exits that can exit to a port list" + def __init__(self, exit_ports=None): + self.exit_ports = exit_ports + + def r_is_ok(self, r): + for port in self.exit_ports: + if not r.will_exit_to("255.255.255.255", port): + return False + return True + + def __str__(self): + return self.__class__.__name__+"()" + class FlagsRestriction(NodeRestriction): "Restriction for mandatory and forbidden router flags" def __init__(self, mandatory, forbidden=[]): @@ -1030,7 +1044,7 @@ class SelectionManager(BaseSelectionManager): percent_fast, percent_skip, min_bw, use_all_exits, uniform, use_exit, use_guards,geoip_config=None, restrict_guards=False, extra_node_rstr=None, exit_ports=None, - order_by_ratio=False): + order_by_ratio=False, min_exits=0): BaseSelectionManager.__init__(self) self.__ordered_exit_gen = None self.pathlen = pathlen @@ -1049,6 +1063,8 @@ class SelectionManager(BaseSelectionManager): self.exit_ports = exit_ports self.extra_node_rstr=extra_node_rstr self.order_by_ratio = order_by_ratio + self.min_exits = min_exits + self.added_exits = []

def reconfigure(self, consensus=None): try: @@ -1126,6 +1142,21 @@ class SelectionManager(BaseSelectionManager): mid_rstr.add_restriction(self.extra_node_rstr) self.exit_rstr.add_restriction(self.extra_node_rstr)

+ # This is a hack just for the bw auths to avoid slices with no exits + if self.min_exits and self.exit_ports: + test_rstr = NodeRestrictionList( + [PctRstr(nonentry_skip, nonentry_fast, sorted_r), + ExitPortRestriction(self.exit_ports), + FlagsRestriction(["Running"], ["BadExit"])]) + exit_count = len(filter(lambda r: test_rstr.r_is_ok(r), sorted_r)) + if exit_count < self.min_exits: + self.added_exits = self.find_emergency_exits(sorted_r, + self.min_exits-exit_count) + plog("NOTICE", "Only "+str(exit_count)+" exits remain in slice "+str(nonentry_skip)+"-"+str(nonentry_fast)+" after restrictions. Adding in "+str(self.added_exits)) + idhex_list = map(IdHexRestriction, self.added_exits) + idhex_list.append(self.exit_rstr) + self.exit_rstr = NodeRestrictionList([OrNodeRestriction(idhex_list)]) + # GeoIP configuration if self.geoip_config: # Every node needs country_code @@ -1198,6 +1229,17 @@ class SelectionManager(BaseSelectionManager): exitgen, self.path_rstr) return

+ # Picks num of the top_n fastest exits that can handle our exit ports. + def find_emergency_exits(self, sorted_r, num, top_n=100): + new_exits = [] + test_rstr = NodeRestrictionList( + [ExitPortRestriction(self.exit_ports), + FlagsRestriction(["Running"], ["BadExit"])]) + for r in sorted_r: + if test_rstr.r_is_ok(r): new_exits.append(r.idhex) + if len(new_exits) >= top_n: break + return random.sample(new_exits, min(len(new_exits), num)) + def _set_exit(self, exit_name): # sets an exit, if bad, sets bad_exit exit_id = None diff --git a/SQLSupport.py b/SQLSupport.py index 28963fc..ee15d79 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -565,7 +565,7 @@ class RouterStats(Entity): tc_session.remove() compute = Callable(compute)

- def write_stats(f, pct_low=0, pct_high=100, order_by=None, recompute=False, stat_clause=None, filter_clause=None, disp_clause=None): + def write_stats(f, pct_low=0, pct_high=100, order_by=None, recompute=False, stat_clause=None, filter_clause=None, disp_clause=None, ignore_by_idhex=[]): l_session = tc_session()

if not order_by: @@ -626,6 +626,9 @@ class RouterStats(Entity):

for s in RouterStats.query.filter(pct_clause).filter(stat_clause).\ filter(disp_clause).order_by(order_by).all(): + if s.router.idhex in ignore_by_idhex: + plog("NOTICE", "Skipping stats on added exit "+s.router.idhex+" in "+f.name) + continue f.write(s.router.idhex+" ("+s.router.nickname+")\n") f.write(" CF="+str(cvt(s.circ_from_rate,2))) f.write(" CT="+str(cvt(s.circ_to_rate,2))) @@ -649,7 +652,7 @@ class RouterStats(Entity): write_stats = Callable(write_stats)

- def write_bws(f, pct_low=0, pct_high=100, order_by=None, recompute=False, stat_clause=None, filter_clause=None, disp_clause=None): + def write_bws(f, pct_low=0, pct_high=100, order_by=None, recompute=False, stat_clause=None, filter_clause=None, disp_clause=None, ignore_by_idhex=[]): l_session = tc_session() if not order_by: order_by=RouterStats.avg_first_ext @@ -678,6 +681,9 @@ class RouterStats(Entity):

for s in RouterStats.query.filter(pct_clause).filter(stat_clause).\ filter(disp_clause).order_by(order_by).all(): + if s.router.idhex in ignore_by_idhex: + plog("NOTICE", "Skipping stats on added exit "+s.router.idhex+" in "+f.name) + continue f.write("node_id=$"+s.router.idhex+" nick="+s.router.nickname) f.write(" strm_bw="+str(cvt(s.sbw,0))) f.write(" filt_bw="+str(cvt(s.filt_sbw,0))) diff --git a/ScanSupport.py b/ScanSupport.py index e6752c5..797fe39 100644 --- a/ScanSupport.py +++ b/ScanSupport.py @@ -79,6 +79,7 @@ class ScanHandler(PathSupport.PathBuilder): cond.acquire() this.new_nym = True if this.selmgr.bad_restrictions: + # XXX: Maybe add in exits here? plog("NOTICE", "Clearing bad restrictions with reconfigure..") this.selmgr.reconfigure(this.current_consensus()) lines = this.c.sendAndRecv("SIGNAL CLEARDNSCACHE\r\n") @@ -198,7 +199,8 @@ class SQLScanHandler(ScanHandler): cond.acquire() SQLSupport.RouterStats.write_stats(file(rfilename, "w"), 0, 100, order_by=SQLSupport.RouterStats.sbw, - recompute=True, disp_clause=stats_filter) + recompute=True, disp_clause=stats_filter, + ignore_by_idhex=h.selmgr.added_exits) cond.notify() cond.release() cond.acquire() @@ -216,7 +218,8 @@ class SQLScanHandler(ScanHandler): f.write("slicenum="+str(slice_num)+"\n") SQLSupport.RouterStats.write_bws(f, 0, 100, order_by=SQLSupport.RouterStats.sbw, - recompute=False, disp_clause=stats_filter) + recompute=False, disp_clause=stats_filter, + ignore_by_idhex=this.selmgr.added_exits) f.close() cond.notify() cond.release()