commit 9e9127b82b65b5ed5f397b8dd267cd101702b012 Author: aagbsn aagbsn@extc.org Date: Thu Jun 16 18:38:00 2011 -0700

renamed files for consistency --- NetworkScanners/BwAuthority/bwauthority-parent.py | 48 --- NetworkScanners/BwAuthority/bwauthority.py | 360 ++------------------- NetworkScanners/BwAuthority/bwauthority_child.py | 366 +++++++++++++++++++++ 3 files changed, 386 insertions(+), 388 deletions(-)

diff --git a/NetworkScanners/BwAuthority/bwauthority-parent.py b/NetworkScanners/BwAuthority/bwauthority-parent.py deleted file mode 100644 index aadcd0e..0000000 --- a/NetworkScanners/BwAuthority/bwauthority-parent.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -import sys -import subprocess -import ConfigParser -import os -import traceback -sys.path.append("../../") -from TorCtl import TorUtil -from TorCtl.TorUtil import plog -import bwauthority - -p = None - -def main(argv): - TorUtil.read_config(argv[1]) - (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, - max_fetch_time,tor_dir,sleep_start,sleep_stop, - min_streams,pid_file_name) = bwauthority.read_config(argv[1]) - - if pid_file_name: - pidfd = file(pid_file_name, 'w') - pidfd.write('%d\n' % os.getpid()) - pidfd.close() - - slice_num = 0 - while True: - plog('INFO', 'Beginning time loop') - - p = subprocess.Popen("python bwauthority.py %s %s" % - (argv[1], str(slice_num)),shell=True) - p.wait() - if (p.returncode == 0): - slice_num += 1 - elif (p.returncode == bwauthority.STOP_PCT_REACHED): - slice_num = 0 - else: - plog('WARN', 'Child process returned %s' % p.returncode) - -if __name__ == '__main__': - try: - main(sys.argv) - except KeyboardInterrupt: - p.terminate() - plog('INFO', "Ctrl + C was pressed. Exiting ... ") - traceback.print_exc() - except Exception, e: - plog('ERROR', "An unexpected error occured.") - traceback.print_exc() diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py old mode 100755 new mode 100644 index 0eace2e..0d57dfe --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -1,364 +1,44 @@ -#!/usr/bin/python -# -# 2009 Mike Perry, Karsten Loesing - -""" -Speedracer - -Speedracer continuously requests the Tor design paper over the Tor network -and measures how long circuit building and downloading takes. -""" - -import atexit -import socket -import time +#!/usr/bin/env python import sys -import urllib2 +import subprocess +import ConfigParser import os import traceback -import copy -import shutil -import threading -import ConfigParser -import sqlalchemy -import sets - sys.path.append("../../") - +from TorCtl import TorUtil from TorCtl.TorUtil import plog +import bwauthority_child

-# WAAAYYYYYY too noisy. -#import gc -#gc.set_debug(gc.DEBUG_COLLECTABLE|gc.DEBUG_UNCOLLECTABLE|gc.DEBUG_INSTANCES|gc.DEBUG_OBJECTS) - -from TorCtl import ScanSupport,PathSupport,SQLSupport,TorCtl,TorUtil - -sys.path.append("../libs") -# Make our SocksiPy use our socket -__origsocket = socket.socket -socket.socket = PathSupport.SmartSocket -from SocksiPy import socks -socket.socket = __origsocket - -user_agent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)" - -# Note these urls should be https due to caching considerations. -# If you really must make them http, be sure to change exit_ports to [80] -# below, or else the scan will not finish. -# TODO: As the network balances, these can become more uniform in size -# TODO: We'll also want to try to prefer pairing unmeasured nodes -# together then, and use a different url set for them. -# cutoff percent URL -urls = [(5, "https://38.229.70.2/16M"), # fbw 1499k..500k - (10, "https://38.229.70.2/8M"), # fbw 500k..350k - (20, "https://38.229.70.2/4M"), # fbw 350k..200k - (30, "https://38.229.70.2/2M"), # fbw 200k..128k - (50, "https://38.229.70.2/512k"), # fbw 128k..50k - (80, "https://38.229.70.2/256k"), # fbw 50k..26k - (100, "https://38.229.70.2/128k")] # fbw 26k..10k - - -# Do NOT modify this object directly after it is handed to PathBuilder -# Use PathBuilder.schedule_selmgr instead. -# (Modifying the arguments here is OK) -__selmgr = PathSupport.SelectionManager( - pathlen=2, - order_exits=False, - percent_fast=100, - percent_skip=0, - min_bw=1024, - use_all_exits=False, - uniform=True, - use_exit=None, - use_guards=False, - exit_ports=[443]) - -# exit code to indicate scan completion -STOP_PCT_REACHED = -9 - -def read_config(filename): - config = ConfigParser.SafeConfigParser() - config.read(filename) - - start_pct = config.getint('BwAuthority', 'start_pct') - stop_pct = config.getint('BwAuthority', 'stop_pct') - - nodes_per_slice = config.getint('BwAuthority', 'nodes_per_slice') - save_every = config.getint('BwAuthority', 'save_every') - circs_per_node = config.getint('BwAuthority', 'circs_per_node') - min_streams = config.getint('BwAuthority', 'min_streams') - out_dir = config.get('BwAuthority', 'out_dir') - tor_dir = config.get('BwAuthority', 'tor_dir') - max_fetch_time = config.getint('BwAuthority', 'max_fetch_time') - - sleep_start = config.get('BwAuthority', 'sleep_start') - sleep_stop = config.get('BwAuthority', 'sleep_stop') - - sleep_start = tuple(map(int, sleep_start.split(":"))) - sleep_stop = tuple(map(int, sleep_stop.split(":"))) - - pid_file = config.get('BwAuthority', 'pid_file') - - return (start_pct,stop_pct,nodes_per_slice,save_every, - circs_per_node,out_dir,max_fetch_time,tor_dir, - sleep_start,sleep_stop,min_streams,pid_file) - -def choose_url(percentile): - for (pct, url) in urls: - if percentile < pct: - return url - #return "https://86.59.21.36/torbrowser/dist/tor-im-browser-1.2.0_ru_split/tor-im-bro..." - raise PathSupport.NoNodesRemain("No nodes left for url choice!") - -def http_request(address): - ''' perform an http GET-request and return 1 for success or 0 for failure ''' - - request = urllib2.Request(address) - request.add_header('User-Agent', user_agent) - - try: - reply = urllib2.urlopen(request) - decl_length = reply.info().get("Content-Length") - read_len = len(reply.read()) - plog("DEBUG", "Read: "+str(read_len)+" of declared "+str(decl_length)) - return 1 - except (ValueError, urllib2.URLError): - plog('ERROR', 'The http-request address ' + address + ' is malformed') - return 0 - except (IndexError, TypeError): - plog('ERROR', 'An error occured while negotiating socks5 with Tor') - return 0 - except KeyboardInterrupt: - raise KeyboardInterrupt - except socks.Socks5Error, e: - if e.value[0] == 6: - plog("NOTICE", "Tor timed out our SOCKS stream request.") - else: - plog('ERROR', 'An unknown HTTP error occured') - traceback.print_exc() - return 0 - except: - plog('ERROR', 'An unknown HTTP error occured') - traceback.print_exc() - return 0 - -class BwScanHandler(ScanSupport.SQLScanHandler): - def is_count_met(self, count, num_streams, position=0): - cond = threading.Condition() - cond._finished = True # lol python haxx. Could make subclass, but why?? :) - def notlambda(this): - cond.acquire() - # TODO: Using the entry_gen router list is somewhat ghetto.. - if this.selmgr.bad_restrictions: - plog("NOTICE", - "Bad restrictions on last attempt. Declaring this slice finished") - elif (this.selmgr.path_selector.entry_gen.rstr_routers and \ - this.selmgr.path_selector.exit_gen.rstr_routers): - for r in this.selmgr.path_selector.entry_gen.rstr_routers: - if r._generated[position] < count: - cond._finished = False - plog("DEBUG", "Entry router "+r.idhex+"="+r.nickname+" not done: "+str(r._generated[position])+", down: "+str(r.down)+", OK: "+str(this.selmgr.path_selector.entry_gen.rstr_list.r_is_ok(r))+", sorted_r: "+str(r in this.sorted_r)) - # XXX: - #break - for r in this.selmgr.path_selector.exit_gen.rstr_routers: - if r._generated[position] < count: - cond._finished = False - plog("DEBUG", "Exit router "+r.idhex+"="+r.nickname+" not done: "+str(r._generated[position])+", down: "+str(r.down)+", OK: "+str(this.selmgr.path_selector.exit_gen.rstr_list.r_is_ok(r))+", sorted_r: "+str(r in this.sorted_r)) - # XXX: - #break - # Also run for at least 2*circs_per_node*nodes/3 successful fetches to - # ensure we don't skip slices in the case of temporary network failure - if cond._finished: - num_routers = len( - sets.Set(this.selmgr.path_selector.entry_gen.rstr_routers - + this.selmgr.path_selector.exit_gen.rstr_routers)) - # If more than 35% of the 2-hop paths failed, keep going to get - # more measurements - if num_streams < 0.65*((num_routers*count)/2.0): - plog("WARN", "Not enough streams yet. "+str(num_streams)+" < "+ - str(0.65*(num_routers*count/2.0))) - cond._finished = False - cond.notify() - cond.release() - plog("DEBUG", "Checking if scan count is met...") - cond.acquire() - self.schedule_low_prio(notlambda) - cond.wait() - cond.release() - plog("DEBUG", "Scan count met: "+str(cond._finished)) - return cond._finished - -def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir, - max_fetch_time, sleep_start_tp, sleep_stop_tp, slice_num, - min_streams, sql_file): - hdlr.set_pct_rstr(start_pct, stop_pct) - - attempt = 0 - successful = 0 - while True: - if hdlr.is_count_met(circs_per_node, successful): break - hdlr.wait_for_consensus() - - # Check local time. Do not scan between 01:30 and 05:30 local time - lt = time.localtime() - sleep_start = time.mktime(lt[0:3]+sleep_start_tp+(0,0,0)+(lt[-1],)) - sleep_stop = time.mktime(lt[0:3]+sleep_stop_tp+(0,0,0)+(lt[-1],)) - t0 = time.time() - if sleep_start <= t0 and t0 <= sleep_stop: - plog("NOTICE", "It's bedtime. Sleeping for "+str(round((sleep_stop-t0)/3600.0,1))+"h") - time.sleep(sleep_stop - t0) - t0 = time.time() - - hdlr.new_exit() - attempt += 1 - - # FIXME: This noise is due to a difficult to find Tor bug that - # causes some exits to hang forever on streams :( - timer = threading.Timer(max_fetch_time, lambda: hdlr.close_streams(7)) - timer.start() - url = choose_url(start_pct) - plog("DEBUG", "Launching stream request for url "+url+" in "+str(start_pct)+'-'+str(stop_pct) + '%') - ret = http_request(url) - timer.cancel() - PathSupport.SmartSocket.clear_port_table() - - delta_build = time.time() - t0 - if delta_build >= max_fetch_time: - plog('WARN', 'Timer exceeded limit: ' + str(delta_build) + '\n') - - build_exit = hdlr.get_exit_node() - if ret == 1 and build_exit: - successful += 1 - plog('DEBUG', str(start_pct) + '-' + str(stop_pct) + '% circuit build+fetch took ' + str(delta_build) + ' for ' + str(build_exit)) - else: - plog('DEBUG', str(start_pct)+'-'+str(stop_pct)+'% circuit build+fetch failed for ' + str(build_exit)) - - if save_every and ret and successful and (successful % save_every) == 0: - race_time = time.strftime("20%y-%m-%d-%H:%M:%S") - hdlr.close_circuits() - hdlr.commit() - lo = str(round(start_pct,1)) - hi = str(round(stop_pct,1)) - # Warning, don't remove the sql stats without changing the recompute - # param in write_strm_bws to True - hdlr.write_sql_stats(os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-"+str(successful)+"-"+race_time, sqlalchemy.or_(SQLSupport.RouterStats.circ_try_from > 0, SQLSupport.RouterStats.circ_try_to > 0)) - hdlr.write_strm_bws(os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-"+str(successful)+"-"+race_time, stats_filter=SQLSupport.RouterStats.strm_closed >= 1) - - plog('INFO', str(start_pct) + '-' + str(stop_pct) + '% ' + str(successful) + ' fetches took ' + str(attempt) + ' tries.') - - hdlr.close_circuits() - hdlr.commit() - - lo = str(round(start_pct,1)) - hi = str(round(stop_pct,1)) - - # There may be a consensus change between the point of speed - # racing and the writing of stats causing a discrepency - # between the immediate, current consensus result used to determine - # termination and this average-based result. - # So instead of using percentiles to filter here, we filter based on - # circuit chosen. - hdlr.write_sql_stats(os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"), stats_filter=sqlalchemy.or_(SQLSupport.RouterStats.circ_try_from > 0, SQLSupport.RouterStats.circ_try_to > 0)) - # Warning, don't remove the sql stats call without changing the recompute - # param in write_strm_bws to True - hdlr.write_strm_bws(os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"), slice_num, stats_filter=sqlalchemy.and_(SQLSupport.RouterStats.strm_closed >= min_streams, SQLSupport.RouterStats.filt_sbw >= 0, SQLSupport.RouterStats.sbw >=0 )) - plog('DEBUG', 'Wrote stats') - #hdlr.save_sql_file(sql_file, os.getcwd()+"/"+out_dir+"/bw-db-"+str(lo)+":"+str(hi)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S")+".sqlite")

def main(argv): TorUtil.read_config(argv[1]) (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, max_fetch_time,tor_dir,sleep_start,sleep_stop, - min_streams,pid_file_name) = read_config(argv[1]) + min_streams,pid_file_name) = bwauthority_child.read_config(argv[1])

if pid_file_name: pidfd = file(pid_file_name, 'w') pidfd.write('%d\n' % os.getpid()) pidfd.close()

- slice_num = int(argv[2]) - - try: - (c,hdlr) = setup_handler(out_dir, tor_dir+"/control_auth_cookie") - except Exception, e: - traceback.print_exc() - plog("WARN", "Can't connect to Tor: "+str(e)) - - sql_file = os.getcwd()+'/'+out_dir+'/bwauthority.sqlite' - hdlr.attach_sql_listener('sqlite:///'+sql_file) - - # set SOCKS proxy - socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, TorUtil.tor_host, TorUtil.tor_port) - socket.socket = socks.socksocket - plog("INFO", "Set socks proxy to "+TorUtil.tor_host+":"+str(TorUtil.tor_port)) - - hdlr.wait_for_consensus() - pct_step = hdlr.rank_to_percent(nodes_per_slice) - - # check to see if we are done - if (slice_num * pct_step > stop_pct): - sys.exit(STOP_PCT_REACHED) - - speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, save_every, out_dir, - max_fetch_time, sleep_start, sleep_stop, slice_num, - min_streams, sql_file) - - # For debugging memory leak.. - #TorUtil.dump_class_ref_counts(referrer_depth=1) - - # TODO: Change pathlen to 3 and kill exit+ConserveExit restrictions - # And record circ failure rates.. - - #circ_measure(hdlr, pct, pct+pct_step, circs_per_node, save_every, - # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) - sys.exit(0) - -def cleanup(c, f): - plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors="+f) - try: - # XXX: Remember __LeaveStreamsUnattached and use saved value! - c.set_option("__LeaveStreamsUnattached", "0") - c.set_option("FetchUselessDescriptors", f) - except TorCtl.TorCtlClosed: - pass - -def setup_handler(out_dir, cookie_file): - plog('INFO', 'Connecting to Tor at '+TorUtil.control_host+":"+str(TorUtil.control_port)) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((TorUtil.control_host,TorUtil.control_port)) - c = PathSupport.Connection(s) - #c.debug(file(out_dir+"/control.log", "w", buffering=0)) - c.authenticate_cookie(file(cookie_file, "r")) - #f = c.get_option("__LeaveStreamsUnattached")[0] - h = BwScanHandler(c, __selmgr, - strm_selector=PathSupport.SmartSocket.StreamSelector) - - c.set_event_handler(h) - #c.set_periodic_timer(2.0, "PULSE") - - c.set_events([TorCtl.EVENT_TYPE.STREAM, - TorCtl.EVENT_TYPE.BW, - TorCtl.EVENT_TYPE.NEWCONSENSUS, - TorCtl.EVENT_TYPE.NEWDESC, - TorCtl.EVENT_TYPE.CIRC, - TorCtl.EVENT_TYPE.STREAM_BW], True) - - c.set_option("__LeaveStreamsUnattached", "1") - f = c.get_option("FetchUselessDescriptors")[0][1] - c.set_option("FetchUselessDescriptors", "1") - atexit.register(cleanup, *(c, f)) - return (c,h) - -def usage(argv): - print "Usage: "+argv[0]+" <configfile>" - return + slice_num = 0 + while True: + plog('INFO', 'Beginning time loop') + global p + p = subprocess.Popen(["python", "bwauthority_child.py", argv[1], str(slice_num)]) + p.wait() + if (p.returncode == 0): + slice_num += 1 + elif (p.returncode == bwauthority_child.STOP_PCT_REACHED): + slice_num = 0 + else: + plog('WARN', 'Child process returned %s' % p.returncode)

-# initiate the program if __name__ == '__main__': try: - if len(sys.argv) < 2: usage(sys.argv) - else: main(sys.argv) + main(sys.argv) except KeyboardInterrupt: + p.kill() plog('INFO', "Ctrl + C was pressed. Exiting ... ") traceback.print_exc() except Exception, e: diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py new file mode 100755 index 0000000..0eace2e --- /dev/null +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -0,0 +1,366 @@ +#!/usr/bin/python +# +# 2009 Mike Perry, Karsten Loesing + +""" +Speedracer + +Speedracer continuously requests the Tor design paper over the Tor network +and measures how long circuit building and downloading takes. +""" + +import atexit +import socket +import time +import sys +import urllib2 +import os +import traceback +import copy +import shutil +import threading +import ConfigParser +import sqlalchemy +import sets + +sys.path.append("../../") + +from TorCtl.TorUtil import plog + +# WAAAYYYYYY too noisy. +#import gc +#gc.set_debug(gc.DEBUG_COLLECTABLE|gc.DEBUG_UNCOLLECTABLE|gc.DEBUG_INSTANCES|gc.DEBUG_OBJECTS) + +from TorCtl import ScanSupport,PathSupport,SQLSupport,TorCtl,TorUtil + +sys.path.append("../libs") +# Make our SocksiPy use our socket +__origsocket = socket.socket +socket.socket = PathSupport.SmartSocket +from SocksiPy import socks +socket.socket = __origsocket + +user_agent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)" + +# Note these urls should be https due to caching considerations. +# If you really must make them http, be sure to change exit_ports to [80] +# below, or else the scan will not finish. +# TODO: As the network balances, these can become more uniform in size +# TODO: We'll also want to try to prefer pairing unmeasured nodes +# together then, and use a different url set for them. +# cutoff percent URL +urls = [(5, "https://38.229.70.2/16M"), # fbw 1499k..500k + (10, "https://38.229.70.2/8M"), # fbw 500k..350k + (20, "https://38.229.70.2/4M"), # fbw 350k..200k + (30, "https://38.229.70.2/2M"), # fbw 200k..128k + (50, "https://38.229.70.2/512k"), # fbw 128k..50k + (80, "https://38.229.70.2/256k"), # fbw 50k..26k + (100, "https://38.229.70.2/128k")] # fbw 26k..10k + + +# Do NOT modify this object directly after it is handed to PathBuilder +# Use PathBuilder.schedule_selmgr instead. +# (Modifying the arguments here is OK) +__selmgr = PathSupport.SelectionManager( + pathlen=2, + order_exits=False, + percent_fast=100, + percent_skip=0, + min_bw=1024, + use_all_exits=False, + uniform=True, + use_exit=None, + use_guards=False, + exit_ports=[443]) + +# exit code to indicate scan completion +STOP_PCT_REACHED = -9 + +def read_config(filename): + config = ConfigParser.SafeConfigParser() + config.read(filename) + + start_pct = config.getint('BwAuthority', 'start_pct') + stop_pct = config.getint('BwAuthority', 'stop_pct') + + nodes_per_slice = config.getint('BwAuthority', 'nodes_per_slice') + save_every = config.getint('BwAuthority', 'save_every') + circs_per_node = config.getint('BwAuthority', 'circs_per_node') + min_streams = config.getint('BwAuthority', 'min_streams') + out_dir = config.get('BwAuthority', 'out_dir') + tor_dir = config.get('BwAuthority', 'tor_dir') + max_fetch_time = config.getint('BwAuthority', 'max_fetch_time') + + sleep_start = config.get('BwAuthority', 'sleep_start') + sleep_stop = config.get('BwAuthority', 'sleep_stop') + + sleep_start = tuple(map(int, sleep_start.split(":"))) + sleep_stop = tuple(map(int, sleep_stop.split(":"))) + + pid_file = config.get('BwAuthority', 'pid_file') + + return (start_pct,stop_pct,nodes_per_slice,save_every, + circs_per_node,out_dir,max_fetch_time,tor_dir, + sleep_start,sleep_stop,min_streams,pid_file) + +def choose_url(percentile): + for (pct, url) in urls: + if percentile < pct: + return url + #return "https://86.59.21.36/torbrowser/dist/tor-im-browser-1.2.0_ru_split/tor-im-bro..." + raise PathSupport.NoNodesRemain("No nodes left for url choice!") + +def http_request(address): + ''' perform an http GET-request and return 1 for success or 0 for failure ''' + + request = urllib2.Request(address) + request.add_header('User-Agent', user_agent) + + try: + reply = urllib2.urlopen(request) + decl_length = reply.info().get("Content-Length") + read_len = len(reply.read()) + plog("DEBUG", "Read: "+str(read_len)+" of declared "+str(decl_length)) + return 1 + except (ValueError, urllib2.URLError): + plog('ERROR', 'The http-request address ' + address + ' is malformed') + return 0 + except (IndexError, TypeError): + plog('ERROR', 'An error occured while negotiating socks5 with Tor') + return 0 + except KeyboardInterrupt: + raise KeyboardInterrupt + except socks.Socks5Error, e: + if e.value[0] == 6: + plog("NOTICE", "Tor timed out our SOCKS stream request.") + else: + plog('ERROR', 'An unknown HTTP error occured') + traceback.print_exc() + return 0 + except: + plog('ERROR', 'An unknown HTTP error occured') + traceback.print_exc() + return 0 + +class BwScanHandler(ScanSupport.SQLScanHandler): + def is_count_met(self, count, num_streams, position=0): + cond = threading.Condition() + cond._finished = True # lol python haxx. Could make subclass, but why?? :) + def notlambda(this): + cond.acquire() + # TODO: Using the entry_gen router list is somewhat ghetto.. + if this.selmgr.bad_restrictions: + plog("NOTICE", + "Bad restrictions on last attempt. Declaring this slice finished") + elif (this.selmgr.path_selector.entry_gen.rstr_routers and \ + this.selmgr.path_selector.exit_gen.rstr_routers): + for r in this.selmgr.path_selector.entry_gen.rstr_routers: + if r._generated[position] < count: + cond._finished = False + plog("DEBUG", "Entry router "+r.idhex+"="+r.nickname+" not done: "+str(r._generated[position])+", down: "+str(r.down)+", OK: "+str(this.selmgr.path_selector.entry_gen.rstr_list.r_is_ok(r))+", sorted_r: "+str(r in this.sorted_r)) + # XXX: + #break + for r in this.selmgr.path_selector.exit_gen.rstr_routers: + if r._generated[position] < count: + cond._finished = False + plog("DEBUG", "Exit router "+r.idhex+"="+r.nickname+" not done: "+str(r._generated[position])+", down: "+str(r.down)+", OK: "+str(this.selmgr.path_selector.exit_gen.rstr_list.r_is_ok(r))+", sorted_r: "+str(r in this.sorted_r)) + # XXX: + #break + # Also run for at least 2*circs_per_node*nodes/3 successful fetches to + # ensure we don't skip slices in the case of temporary network failure + if cond._finished: + num_routers = len( + sets.Set(this.selmgr.path_selector.entry_gen.rstr_routers + + this.selmgr.path_selector.exit_gen.rstr_routers)) + # If more than 35% of the 2-hop paths failed, keep going to get + # more measurements + if num_streams < 0.65*((num_routers*count)/2.0): + plog("WARN", "Not enough streams yet. "+str(num_streams)+" < "+ + str(0.65*(num_routers*count/2.0))) + cond._finished = False + cond.notify() + cond.release() + plog("DEBUG", "Checking if scan count is met...") + cond.acquire() + self.schedule_low_prio(notlambda) + cond.wait() + cond.release() + plog("DEBUG", "Scan count met: "+str(cond._finished)) + return cond._finished + +def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir, + max_fetch_time, sleep_start_tp, sleep_stop_tp, slice_num, + min_streams, sql_file): + hdlr.set_pct_rstr(start_pct, stop_pct) + + attempt = 0 + successful = 0 + while True: + if hdlr.is_count_met(circs_per_node, successful): break + hdlr.wait_for_consensus() + + # Check local time. Do not scan between 01:30 and 05:30 local time + lt = time.localtime() + sleep_start = time.mktime(lt[0:3]+sleep_start_tp+(0,0,0)+(lt[-1],)) + sleep_stop = time.mktime(lt[0:3]+sleep_stop_tp+(0,0,0)+(lt[-1],)) + t0 = time.time() + if sleep_start <= t0 and t0 <= sleep_stop: + plog("NOTICE", "It's bedtime. Sleeping for "+str(round((sleep_stop-t0)/3600.0,1))+"h") + time.sleep(sleep_stop - t0) + t0 = time.time() + + hdlr.new_exit() + attempt += 1 + + # FIXME: This noise is due to a difficult to find Tor bug that + # causes some exits to hang forever on streams :( + timer = threading.Timer(max_fetch_time, lambda: hdlr.close_streams(7)) + timer.start() + url = choose_url(start_pct) + plog("DEBUG", "Launching stream request for url "+url+" in "+str(start_pct)+'-'+str(stop_pct) + '%') + ret = http_request(url) + timer.cancel() + PathSupport.SmartSocket.clear_port_table() + + delta_build = time.time() - t0 + if delta_build >= max_fetch_time: + plog('WARN', 'Timer exceeded limit: ' + str(delta_build) + '\n') + + build_exit = hdlr.get_exit_node() + if ret == 1 and build_exit: + successful += 1 + plog('DEBUG', str(start_pct) + '-' + str(stop_pct) + '% circuit build+fetch took ' + str(delta_build) + ' for ' + str(build_exit)) + else: + plog('DEBUG', str(start_pct)+'-'+str(stop_pct)+'% circuit build+fetch failed for ' + str(build_exit)) + + if save_every and ret and successful and (successful % save_every) == 0: + race_time = time.strftime("20%y-%m-%d-%H:%M:%S") + hdlr.close_circuits() + hdlr.commit() + lo = str(round(start_pct,1)) + hi = str(round(stop_pct,1)) + # Warning, don't remove the sql stats without changing the recompute + # param in write_strm_bws to True + hdlr.write_sql_stats(os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-"+str(successful)+"-"+race_time, sqlalchemy.or_(SQLSupport.RouterStats.circ_try_from > 0, SQLSupport.RouterStats.circ_try_to > 0)) + hdlr.write_strm_bws(os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-"+str(successful)+"-"+race_time, stats_filter=SQLSupport.RouterStats.strm_closed >= 1) + + plog('INFO', str(start_pct) + '-' + str(stop_pct) + '% ' + str(successful) + ' fetches took ' + str(attempt) + ' tries.') + + hdlr.close_circuits() + hdlr.commit() + + lo = str(round(start_pct,1)) + hi = str(round(stop_pct,1)) + + # There may be a consensus change between the point of speed + # racing and the writing of stats causing a discrepency + # between the immediate, current consensus result used to determine + # termination and this average-based result. + # So instead of using percentiles to filter here, we filter based on + # circuit chosen. + hdlr.write_sql_stats(os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"), stats_filter=sqlalchemy.or_(SQLSupport.RouterStats.circ_try_from > 0, SQLSupport.RouterStats.circ_try_to > 0)) + # Warning, don't remove the sql stats call without changing the recompute + # param in write_strm_bws to True + hdlr.write_strm_bws(os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"), slice_num, stats_filter=sqlalchemy.and_(SQLSupport.RouterStats.strm_closed >= min_streams, SQLSupport.RouterStats.filt_sbw >= 0, SQLSupport.RouterStats.sbw >=0 )) + plog('DEBUG', 'Wrote stats') + #hdlr.save_sql_file(sql_file, os.getcwd()+"/"+out_dir+"/bw-db-"+str(lo)+":"+str(hi)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S")+".sqlite") + +def main(argv): + TorUtil.read_config(argv[1]) + (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, + max_fetch_time,tor_dir,sleep_start,sleep_stop, + min_streams,pid_file_name) = read_config(argv[1]) + + if pid_file_name: + pidfd = file(pid_file_name, 'w') + pidfd.write('%d\n' % os.getpid()) + pidfd.close() + + slice_num = int(argv[2]) + + try: + (c,hdlr) = setup_handler(out_dir, tor_dir+"/control_auth_cookie") + except Exception, e: + traceback.print_exc() + plog("WARN", "Can't connect to Tor: "+str(e)) + + sql_file = os.getcwd()+'/'+out_dir+'/bwauthority.sqlite' + hdlr.attach_sql_listener('sqlite:///'+sql_file) + + # set SOCKS proxy + socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, TorUtil.tor_host, TorUtil.tor_port) + socket.socket = socks.socksocket + plog("INFO", "Set socks proxy to "+TorUtil.tor_host+":"+str(TorUtil.tor_port)) + + hdlr.wait_for_consensus() + pct_step = hdlr.rank_to_percent(nodes_per_slice) + + # check to see if we are done + if (slice_num * pct_step > stop_pct): + sys.exit(STOP_PCT_REACHED) + + speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, save_every, out_dir, + max_fetch_time, sleep_start, sleep_stop, slice_num, + min_streams, sql_file) + + # For debugging memory leak.. + #TorUtil.dump_class_ref_counts(referrer_depth=1) + + # TODO: Change pathlen to 3 and kill exit+ConserveExit restrictions + # And record circ failure rates.. + + #circ_measure(hdlr, pct, pct+pct_step, circs_per_node, save_every, + # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) + sys.exit(0) + +def cleanup(c, f): + plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors="+f) + try: + # XXX: Remember __LeaveStreamsUnattached and use saved value! + c.set_option("__LeaveStreamsUnattached", "0") + c.set_option("FetchUselessDescriptors", f) + except TorCtl.TorCtlClosed: + pass + +def setup_handler(out_dir, cookie_file): + plog('INFO', 'Connecting to Tor at '+TorUtil.control_host+":"+str(TorUtil.control_port)) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((TorUtil.control_host,TorUtil.control_port)) + c = PathSupport.Connection(s) + #c.debug(file(out_dir+"/control.log", "w", buffering=0)) + c.authenticate_cookie(file(cookie_file, "r")) + #f = c.get_option("__LeaveStreamsUnattached")[0] + h = BwScanHandler(c, __selmgr, + strm_selector=PathSupport.SmartSocket.StreamSelector) + + c.set_event_handler(h) + #c.set_periodic_timer(2.0, "PULSE") + + c.set_events([TorCtl.EVENT_TYPE.STREAM, + TorCtl.EVENT_TYPE.BW, + TorCtl.EVENT_TYPE.NEWCONSENSUS, + TorCtl.EVENT_TYPE.NEWDESC, + TorCtl.EVENT_TYPE.CIRC, + TorCtl.EVENT_TYPE.STREAM_BW], True) + + c.set_option("__LeaveStreamsUnattached", "1") + f = c.get_option("FetchUselessDescriptors")[0][1] + c.set_option("FetchUselessDescriptors", "1") + atexit.register(cleanup, *(c, f)) + return (c,h) + +def usage(argv): + print "Usage: "+argv[0]+" <configfile>" + return + +# initiate the program +if __name__ == '__main__': + try: + if len(sys.argv) < 2: usage(sys.argv) + else: main(sys.argv) + except KeyboardInterrupt: + plog('INFO', "Ctrl + C was pressed. Exiting ... ") + traceback.print_exc() + except Exception, e: + plog('ERROR', "An unexpected error occured.") + traceback.print_exc()