[tor-commits] [torflow/master] Split bwauthority.py into parent and child

[mikeperry at torproject.org]

commit 7dc70f2efadf641a2f036a1d366715b622dc3a5d
Author: aagbsn <aagbsn at extc.org>
Date:   Thu Jun 16 14:07:48 2011 -0700

    Split bwauthority.py into parent and child
    
    This approach mitigates the memory leaks that plagued
    long-running bwauths by launching each slice as a separate process
---
 NetworkScanners/BwAuthority/bwauthority-parent.py |   48 +++++++++++++++
 NetworkScanners/BwAuthority/bwauthority.py        |   65 ++++++++++-----------
 2 files changed, 80 insertions(+), 33 deletions(-)

diff --git a/NetworkScanners/BwAuthority/bwauthority-parent.py b/NetworkScanners/BwAuthority/bwauthority-parent.py
new file mode 100644
index 0000000..aadcd0e
--- /dev/null
+++ b/NetworkScanners/BwAuthority/bwauthority-parent.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+import sys
+import subprocess
+import ConfigParser
+import os
+import traceback
+sys.path.append("../../")
+from TorCtl import TorUtil
+from TorCtl.TorUtil import plog
+import bwauthority
+
+p = None
+
+def main(argv):
+  TorUtil.read_config(argv[1])
+  (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir,
+      max_fetch_time,tor_dir,sleep_start,sleep_stop,
+             min_streams,pid_file_name) = bwauthority.read_config(argv[1])
+ 
+  if pid_file_name:
+    pidfd = file(pid_file_name, 'w')
+    pidfd.write('%d\n' % os.getpid())
+    pidfd.close()
+
+  slice_num = 0 
+  while True:
+    plog('INFO', 'Beginning time loop')
+
+    p = subprocess.Popen("python bwauthority.py %s %s" %
+            (argv[1], str(slice_num)),shell=True)
+    p.wait()
+    if (p.returncode == 0):
+      slice_num += 1
+    elif (p.returncode == bwauthority.STOP_PCT_REACHED):
+      slice_num = 0
+    else:
+      plog('WARN', 'Child process returned %s' % p.returncode)
+
+if __name__ == '__main__':
+  try:
+    main(sys.argv)
+  except KeyboardInterrupt:
+    p.terminate()
+    plog('INFO', "Ctrl + C was pressed. Exiting ... ")
+    traceback.print_exc()
+  except Exception, e:
+    plog('ERROR', "An unexpected error occured.")
+    traceback.print_exc()
diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py
index 70a8344..0eace2e 100755
--- a/NetworkScanners/BwAuthority/bwauthority.py
+++ b/NetworkScanners/BwAuthority/bwauthority.py
@@ -73,6 +73,9 @@ __selmgr = PathSupport.SelectionManager(
       use_guards=False,
       exit_ports=[443])
 
+# exit code to indicate scan completion
+STOP_PCT_REACHED = -9
+
 def read_config(filename):
   config = ConfigParser.SafeConfigParser()
   config.read(filename)
@@ -273,46 +276,42 @@ def main(argv):
     pidfd.write('%d\n' % os.getpid())
     pidfd.close()
 
-  try:
-    (c,hdlr) = setup_handler(out_dir, tor_dir+"/control_auth_cookie")
-  except Exception, e:
-    traceback.print_exc()
-    plog("WARN", "Can't connect to Tor: "+str(e))
-
-  sql_file = os.getcwd()+'/'+out_dir+'/bwauthority.sqlite'
-  hdlr.attach_sql_listener('sqlite:///'+sql_file)
+    slice_num = int(argv[2])
 
-  # set SOCKS proxy
-  socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, TorUtil.tor_host, TorUtil.tor_port)
-  socket.socket = socks.socksocket
-  plog("INFO", "Set socks proxy to "+TorUtil.tor_host+":"+str(TorUtil.tor_port))
+    try:
+      (c,hdlr) = setup_handler(out_dir, tor_dir+"/control_auth_cookie")
+    except Exception, e:
+      traceback.print_exc()
+      plog("WARN", "Can't connect to Tor: "+str(e))
+    
+    sql_file = os.getcwd()+'/'+out_dir+'/bwauthority.sqlite'
+    hdlr.attach_sql_listener('sqlite:///'+sql_file)
+    
+    # set SOCKS proxy
+    socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, TorUtil.tor_host, TorUtil.tor_port)
+    socket.socket = socks.socksocket
+    plog("INFO", "Set socks proxy to "+TorUtil.tor_host+":"+str(TorUtil.tor_port))
 
-  while True:
-    pct = start_pct
-    plog('INFO', 'Beginning time loop')
-    slice_num = 0 
-    while pct < stop_pct:
-      hdlr.wait_for_consensus()
-      pct_step = hdlr.rank_to_percent(nodes_per_slice)
-      hdlr.reset_stats()
-      hdlr.commit()
-      plog('DEBUG', 'Reset stats')
+    hdlr.wait_for_consensus()
+    pct_step = hdlr.rank_to_percent(nodes_per_slice)
 
-      speedrace(hdlr, pct, pct+pct_step, circs_per_node, save_every, out_dir,
-                max_fetch_time, sleep_start, sleep_stop, slice_num,
-                min_streams, sql_file)
+    # check to see if we are done
+    if (slice_num * pct_step > stop_pct):
+        sys.exit(STOP_PCT_REACHED)
 
-      # For debugging memory leak..
-      #TorUtil.dump_class_ref_counts(referrer_depth=1)
+    speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, save_every, out_dir,
+              max_fetch_time, sleep_start, sleep_stop, slice_num,
+              min_streams, sql_file)
 
-      # TODO: Change pathlen to 3 and kill exit+ConserveExit restrictions
-      # And record circ failure rates..
+    # For debugging memory leak..
+    #TorUtil.dump_class_ref_counts(referrer_depth=1)
 
-      #circ_measure(hdlr, pct, pct+pct_step, circs_per_node, save_every, 
-      #  out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file)
+    # TODO: Change pathlen to 3 and kill exit+ConserveExit restrictions
+    # And record circ failure rates..
 
-      pct += pct_step
-      slice_num += 1
+    #circ_measure(hdlr, pct, pct+pct_step, circs_per_node, save_every, 
+    #  out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file)
+    sys.exit(0)
 
 def cleanup(c, f):
   plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors="+f)