commit 4f272b1bf606df6e9788ad5bc60dbe3f9027eb23 Author: Mike Perry <mikeperry-git@torproject.org> Date: Thu May 21 18:18:55 2015 -0700 Add a 5th scanner for Unmeasured=1 nodes. --- NetworkScanners/BwAuthority/bwauthority_child.py | 36 ++++++++++++++++---- .../BwAuthority/data/scanner.1/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.2/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.3/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.4/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.5/bwauthority.cfg | 30 ++++++++++++++++ NetworkScanners/BwAuthority/run_scan.sh | 2 +- 7 files changed, 68 insertions(+), 8 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 2584991..4bac13a 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -95,9 +95,13 @@ def read_config(filename): pid_file = config.get('BwAuthority', 'pid_file') db_url = config.get('BwAuthority', 'db_url') + only_unmeasured = config.getint('BwAuthority', 'only_unmeasured') + min_unmeasured = config.getint('BwAuthority', 'min_unmeasured') + return (start_pct,stop_pct,nodes_per_slice,save_every, circs_per_node,out_dir,max_fetch_time,tor_dir, - sleep_start,sleep_stop,min_streams,pid_file,db_url) + sleep_start,sleep_stop,min_streams,pid_file,db_url,only_unmeasured, + min_unmeasured) def choose_url(percentile): # TODO: Maybe we don't want to read the file *every* time? @@ -205,7 +209,7 @@ class BwScanHandler(ScanSupport.SQLScanHandler): def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir, max_fetch_time, sleep_start_tp, sleep_stop_tp, slice_num, - min_streams, sql_file): + min_streams, sql_file, only_unmeasured): hdlr.set_pct_rstr(start_pct, stop_pct) attempt = 0 @@ -214,8 +218,6 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir, if hdlr.is_count_met(circs_per_node, successful): break hdlr.wait_for_consensus() - - # Check local time. Do not scan between 01:30 and 05:30 local time lt = time.localtime() sleep_start = time.mktime(lt[0:3]+sleep_start_tp+(0,0,0)+(lt[-1],)) @@ -239,7 +241,14 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir, # stream.. however, we count it as 'successful' below timer = threading.Timer(max_fetch_time, lambda: hdlr.close_streams(7)) timer.start() - url = choose_url(start_pct) + + # Always use median URL size for unmeasured nodes + # They may be too slow.. + if only_unmeasured: + url = choose_url(50) + else: + url = choose_url(start_pct) + plog("DEBUG", "Launching stream request for url "+url+" in "+str(start_pct)+'-'+str(stop_pct) + '%') ret = http_request(url) timer.cancel() @@ -297,7 +306,8 @@ def main(argv): TorUtil.read_config(argv[1]) (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, max_fetch_time,tor_dir,sleep_start,sleep_stop, - min_streams,pid_file_name,db_url) = read_config(argv[1]) + min_streams,pid_file_name,db_url,only_unmeasured, + min_unmeasured) = read_config(argv[1]) # make sure necessary out_dir directory exists path = os.getcwd()+'/'+out_dir @@ -332,9 +342,21 @@ def main(argv): socket.socket = socks.socksocket plog("INFO", "Set socks proxy to "+TorUtil.tor_host+":"+str(TorUtil.tor_port)) + hdlr.schedule_selmgr(lambda s: setattr(s, "only_unmeasured", only_unmeasured)) + hdlr.wait_for_consensus() + # We should go to sleep if there are less than 5 unmeasured nodes after + # consensus update + while min_unmeasured and hdlr.get_unmeasured() < min_unmeasured: + plog("NOTICE", "Less than "+str(min_unmeasured)+" unmeasured nodes ("+str(hdlr.get_unmeasured())+"). Sleeping for a bit") + time.sleep(3600) # Until next consensus arrives + plog("NOTICE", "Woke up from waiting for more unmeasured nodes. Getting consensus and checking again") + hdlr.wait_for_consensus() + pct_step = hdlr.rank_to_percent(nodes_per_slice) + plog("INFO", "Percent per slice is: "+str(pct_step)) + if pct_step > 100: pct_step = 100 # check to see if we are done if (slice_num * pct_step + start_pct > stop_pct): @@ -344,7 +366,7 @@ def main(argv): plog("DEBUG", "Starting slice number %s" % slice_num) successful = speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, save_every, out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, - min_streams, sql_file) + min_streams, sql_file, only_unmeasured) # For debugging memory leak.. #TorUtil.dump_class_ref_counts(referrer_depth=1) diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg index 4cced6b..7f81a0b 100644 --- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg @@ -26,3 +26,5 @@ min_streams = 1 max_fetch_time = 600 sleep_start = 01:30 sleep_stop = 01:30 +only_unmeasured = 0 +min_unmeasured = 0 diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg index 53dc469..28cfaa6 100644 --- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg @@ -26,3 +26,5 @@ min_streams = 1 max_fetch_time = 300 sleep_start = 01:30 sleep_stop = 01:30 +only_unmeasured = 0 +min_unmeasured = 0 diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg index 6df68e1..7b71aa8 100644 --- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg @@ -26,3 +26,5 @@ min_streams = 1 max_fetch_time = 300 sleep_start = 01:30 sleep_stop = 01:30 +only_unmeasured = 0 +min_unmeasured = 0 diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg index ff19b87..0dfea36 100644 --- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg @@ -26,3 +26,5 @@ min_streams = 1 max_fetch_time = 300 sleep_start = 01:30 sleep_stop = 01:30 +only_unmeasured = 0 +min_unmeasured = 0 diff --git a/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg new file mode 100644 index 0000000..468a62e --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg @@ -0,0 +1,30 @@ +[TorCtl] +loglevel=WARN +tor_host = 127.0.0.1 +tor_port = 9110 +control_host = 127.0.0.1 +control_port = 9111 +control_pass = +# XXX: Unused +meta_host = 127.0.0.1 +meta_port = 9112 + +[BwAuthority] +out_dir = ./data/scanner.5/scan-data +pid_file = ./data/scanner.5/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4 +tor_dir = ./data/tor +start_pct = 0 +stop_pct = 100 +save_every = 0 +nodes_per_slice = 50 +circs_per_node = 5 +min_streams = 1 +max_fetch_time = 360 +sleep_start = 01:30 +sleep_stop = 01:30 +only_unmeasured = 1 +min_unmeasured = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.5/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.5/scan-data/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/NetworkScanners/BwAuthority/run_scan.sh b/NetworkScanners/BwAuthority/run_scan.sh index c21e871..7853647 100755 --- a/NetworkScanners/BwAuthority/run_scan.sh +++ b/NetworkScanners/BwAuthority/run_scan.sh @@ -1,7 +1,7 @@ #!/bin/sh # Number of scanners to run. -SCANNER_COUNT=4 +SCANNER_COUNT=5 # This tor must have the w status line fix as well as the stream bw fix # Ie git master or 0.2.2.x