[or-cvs] r19461: {torflow} Allow buildtimes to optionally use SQL support. (torflow/trunk/CircuitAnalysis/BuildTimes)

[mikeperry at seul.org]

Author: mikeperry
Date: 2009-05-06 07:36:12 -0400 (Wed, 06 May 2009)
New Revision: 19461

Modified:
   torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py
   torflow/trunk/CircuitAnalysis/BuildTimes/full_run.sh
Log:

Allow buildtimes to optionally use SQL support.



Modified: torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py
===================================================================
--- torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py	2009-05-06 11:35:14 UTC (rev 19460)
+++ torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py	2009-05-06 11:36:12 UTC (rev 19461)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # uses metatroller to collect circuit build times for 5% slices of guard nodes
 # [OUTPUT] one directory, with three files: StatsHandler aggregate stats file, file with all circuit events (for detailed reference), file with just buildtimes
 
@@ -130,7 +130,7 @@
   plog("INFO", "Resetting FetchUselessDescriptors="+FUDValue)
   c.set_option("FetchUselessDescriptors", FUDValue) 
 
-def open_controller(filename,ncircuits):
+def open_controller(filename,ncircuits,use_sql):
   """ starts stat gathering thread """
 
   s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
@@ -140,6 +140,13 @@
   c.debug(file(filename+".log", "w", buffering=0))
   h = CircStatsGatherer(c,__selmgr,filename,ncircuits)
   c.set_event_handler(h)
+
+  if use_sql:
+    from TorCtl import SQLSupport
+    SQLSupport.setup_db("sqlite:///"+filename+".sqlite", drop=True)
+    c.add_event_listener(SQLSupport.ConsensusTrackerListener())
+    c.add_event_listener(SQLSupport.CircuitListener())
+  
   global FUDValue
   if not FUDValue:
     FUDValue = c.get_option("FetchUselessDescriptors")[0][1]
@@ -160,7 +167,7 @@
     usage()
     sys.exit(2)
   try:
-    opts,args = getopt.getopt(sys.argv[1:],"b:e:s:n:d:c:g")
+    opts,args = getopt.getopt(sys.argv[1:],"b:e:s:n:d:c:gq")
   except getopt.GetoptError,err:
     print str(err)
     usage()
@@ -169,6 +176,7 @@
   end=80
   pslice=5
   dirname=""
+  use_sql=False
   guard_slices = False
   max_circuits=60
   for o,a in opts:
@@ -187,18 +195,21 @@
       else: usage()
     elif o == '-g':
       guard_slices = True
+    elif o == '-q':
+      use_sql = True
+      from TorCtl import SQLSupport
     elif o == '-c':
       if a.isdigit(): max_circuits = int(a)
       else: usage()
     else:
       assert False, "Bad option"
-  return guard_slices,ncircuits,max_circuits,begin,end,pslice,dirname
+  return guard_slices,ncircuits,max_circuits,begin,end,pslice,dirname,use_sql
 
 def usage():
-    print 'usage: buildtimes.py [-b <#begin percentile>] [-e <end percentile] [-s <percentile slice size>] [-g] -n <# circuits> -d <output dir name> [-c <max concurrent circuits>]'
+    print 'usage: buildtimes.py [-b <#begin percentile>] [-e <end percentile] [-s <percentile slice size>] [-g] [-q] -n <# circuits> -d <output dir name> [-c <max concurrent circuits>]'
     sys.exit(1)
 
-def guardslice(guard_slices,p,s,end,ncircuits,max_circuits,dirname):
+def guardslice(guard_slices,p,s,end,ncircuits,max_circuits,dirname,use_sql):
 
   print 'Making new directory:',dirname
   if not os.path.isdir(dirname):
@@ -236,7 +247,7 @@
   __selmgr.__ordered_exit_gen = None
 
   try:
-    c = open_controller(basefile_name,ncircuits)
+    c = open_controller(basefile_name,ncircuits,use_sql)
   except PathSupport.NoNodesRemain:
     print 'No nodes remain at this percentile range ('+str(p)+"-"+str(s)+")"
     return
@@ -270,11 +281,19 @@
       break
 
   cond = threading.Condition() 
-  def notlambda(h):
+  def notlambda(h, use_sql=use_sql):
     cond.acquire()
     h.close_all_circuits()
     h.write_stats(aggfile_name)
 
+    if use_sql:
+      from TorCtl import SQLSupport
+      SQLSupport.RouterStats.write_stats(file(aggfile_name+"-sql", "w"), 
+                     recompute=True,
+                     order_by=SQLSupport.RouterStats.circ_bi_rate,
+                     stats_clause=SQLSupport.RouterStats.circ_try_to>0)
+
+
     f = open(uptime_name, "w")
     # Write out idhex+uptime info
     for r in h.sorted_r:
@@ -315,14 +334,14 @@
   print "Done in main."
 
 def main():
-  guard_slices,ncircuits,max_circuits,begin,end,pct,dirname = getargs()
+  guard_slices,ncircuits,max_circuits,begin,end,pct,dirname,use_sql = getargs()
  
   atexit.register(cleanup) 
 
   print "Using max_circuits: "+str(max_circuits)
 
   for p in xrange(begin,end,pct):
-    guardslice(guard_slices,p,p+pct,end,ncircuits,max_circuits,dirname)
+    guardslice(guard_slices,p,p+pct,end,ncircuits,max_circuits,dirname,use_sql)
 
 if __name__ == '__main__':
   main()

Modified: torflow/trunk/CircuitAnalysis/BuildTimes/full_run.sh
===================================================================
--- torflow/trunk/CircuitAnalysis/BuildTimes/full_run.sh	2009-05-06 11:35:14 UTC (rev 19460)
+++ torflow/trunk/CircuitAnalysis/BuildTimes/full_run.sh	2009-05-06 11:36:12 UTC (rev 19461)
@@ -2,8 +2,8 @@
 
 mkdir slices
 
-./buildtimes.py -n 10000 -s 3 -e 93 -c 15 -d ./slices >& ./slices/bt-slices.log
-./buildtimes.py -n 10000 -s 3 -g -e 50 -c 30 -d ./slices >& ./slices/bt-guards.log
-./buildtimes.py -n 100000 -d slices/ -s 93 -c 100 >& ./slices/bt-all.log
+./buildtimes.py -n 1000 -s 3 -e 93 -c 15 -q -d ./slices >& ./slices/bt-slices.log
+./buildtimes.py -n 1000 -s 3 -g -e 50 -c 30 -q -d ./slices >& ./slices/bt-guards.log
+./buildtimes.py -n 10000 -d slices/ -s 93 -c 100 -q >& ./slices/bt-all.log
 
 mv slices slices-`date +%Y-%m-%d-%H:%M`