commit 7dc70f2efadf641a2f036a1d366715b622dc3a5d Author: aagbsn <aagbsn@extc.org> Date: Thu Jun 16 14:07:48 2011 -0700 Split bwauthority.py into parent and child This approach mitigates the memory leaks that plagued long-running bwauths by launching each slice as a separate process --- NetworkScanners/BwAuthority/bwauthority-parent.py | 48 +++++++++++++++ NetworkScanners/BwAuthority/bwauthority.py | 65 ++++++++++----------- 2 files changed, 80 insertions(+), 33 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority-parent.py b/NetworkScanners/BwAuthority/bwauthority-parent.py new file mode 100644 index 0000000..aadcd0e --- /dev/null +++ b/NetworkScanners/BwAuthority/bwauthority-parent.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +import sys +import subprocess +import ConfigParser +import os +import traceback +sys.path.append("../../") +from TorCtl import TorUtil +from TorCtl.TorUtil import plog +import bwauthority + +p = None + +def main(argv): + TorUtil.read_config(argv[1]) + (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, + max_fetch_time,tor_dir,sleep_start,sleep_stop, + min_streams,pid_file_name) = bwauthority.read_config(argv[1]) + + if pid_file_name: + pidfd = file(pid_file_name, 'w') + pidfd.write('%d\n' % os.getpid()) + pidfd.close() + + slice_num = 0 + while True: + plog('INFO', 'Beginning time loop') + + p = subprocess.Popen("python bwauthority.py %s %s" % + (argv[1], str(slice_num)),shell=True) + p.wait() + if (p.returncode == 0): + slice_num += 1 + elif (p.returncode == bwauthority.STOP_PCT_REACHED): + slice_num = 0 + else: + plog('WARN', 'Child process returned %s' % p.returncode) + +if __name__ == '__main__': + try: + main(sys.argv) + except KeyboardInterrupt: + p.terminate() + plog('INFO', "Ctrl + C was pressed. Exiting ... ") + traceback.print_exc() + except Exception, e: + plog('ERROR', "An unexpected error occured.") + traceback.print_exc() diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index 70a8344..0eace2e 100755 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -73,6 +73,9 @@ __selmgr = PathSupport.SelectionManager( use_guards=False, exit_ports=[443]) +# exit code to indicate scan completion +STOP_PCT_REACHED = -9 + def read_config(filename): config = ConfigParser.SafeConfigParser() config.read(filename) @@ -273,46 +276,42 @@ def main(argv): pidfd.write('%d\n' % os.getpid()) pidfd.close() - try: - (c,hdlr) = setup_handler(out_dir, tor_dir+"/control_auth_cookie") - except Exception, e: - traceback.print_exc() - plog("WARN", "Can't connect to Tor: "+str(e)) - - sql_file = os.getcwd()+'/'+out_dir+'/bwauthority.sqlite' - hdlr.attach_sql_listener('sqlite:///'+sql_file) + slice_num = int(argv[2]) - # set SOCKS proxy - socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, TorUtil.tor_host, TorUtil.tor_port) - socket.socket = socks.socksocket - plog("INFO", "Set socks proxy to "+TorUtil.tor_host+":"+str(TorUtil.tor_port)) + try: + (c,hdlr) = setup_handler(out_dir, tor_dir+"/control_auth_cookie") + except Exception, e: + traceback.print_exc() + plog("WARN", "Can't connect to Tor: "+str(e)) + + sql_file = os.getcwd()+'/'+out_dir+'/bwauthority.sqlite' + hdlr.attach_sql_listener('sqlite:///'+sql_file) + + # set SOCKS proxy + socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, TorUtil.tor_host, TorUtil.tor_port) + socket.socket = socks.socksocket + plog("INFO", "Set socks proxy to "+TorUtil.tor_host+":"+str(TorUtil.tor_port)) - while True: - pct = start_pct - plog('INFO', 'Beginning time loop') - slice_num = 0 - while pct < stop_pct: - hdlr.wait_for_consensus() - pct_step = hdlr.rank_to_percent(nodes_per_slice) - hdlr.reset_stats() - hdlr.commit() - plog('DEBUG', 'Reset stats') + hdlr.wait_for_consensus() + pct_step = hdlr.rank_to_percent(nodes_per_slice) - speedrace(hdlr, pct, pct+pct_step, circs_per_node, save_every, out_dir, - max_fetch_time, sleep_start, sleep_stop, slice_num, - min_streams, sql_file) + # check to see if we are done + if (slice_num * pct_step > stop_pct): + sys.exit(STOP_PCT_REACHED) - # For debugging memory leak.. - #TorUtil.dump_class_ref_counts(referrer_depth=1) + speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, save_every, out_dir, + max_fetch_time, sleep_start, sleep_stop, slice_num, + min_streams, sql_file) - # TODO: Change pathlen to 3 and kill exit+ConserveExit restrictions - # And record circ failure rates.. + # For debugging memory leak.. + #TorUtil.dump_class_ref_counts(referrer_depth=1) - #circ_measure(hdlr, pct, pct+pct_step, circs_per_node, save_every, - # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) + # TODO: Change pathlen to 3 and kill exit+ConserveExit restrictions + # And record circ failure rates.. - pct += pct_step - slice_num += 1 + #circ_measure(hdlr, pct, pct+pct_step, circs_per_node, save_every, + # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) + sys.exit(0) def cleanup(c, f): plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors="+f)