[or-cvs] r17913: {} Begin reorganizing repository. The clutter has been botherin (in torflow/trunk: . CircuitAnalysis tools)
mikeperry at seul.org
mikeperry at seul.org
Mon Jan 5 16:04:10 UTC 2009
Author: mikeperry
Date: 2009-01-05 11:04:10 -0500 (Mon, 05 Jan 2009)
New Revision: 17913
Added:
torflow/trunk/CircuitAnalysis/
torflow/trunk/CircuitAnalysis/buildtimes.py
torflow/trunk/CircuitAnalysis/numpy_pareto.py
torflow/trunk/CircuitAnalysis/shufflebt.py
torflow/trunk/moniTor.py
Removed:
torflow/trunk/guardslices.py
torflow/trunk/tools/BTAnalysis/
torflow/trunk/tools/moniTor.py
Log:
Begin reorganizing repository. The clutter has been bothering
me for some time. Start with the circuit build time analysis
utilities.
Copied: torflow/trunk/CircuitAnalysis/buildtimes.py (from rev 17873, torflow/trunk/guardslices.py)
===================================================================
--- torflow/trunk/CircuitAnalysis/buildtimes.py (rev 0)
+++ torflow/trunk/CircuitAnalysis/buildtimes.py 2009-01-05 16:04:10 UTC (rev 17913)
@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+# uses metatroller to collect circuit build times for 5% slices of guard nodes
+# [OUTPUT] one directory, with three files: StatsHandler aggregate stats file, file with all circuit events (for detailed reference), file with just buildtimes
+
+import socket,sys,time,getopt,os
+from TorCtl.TorUtil import meta_port,meta_host,control_port,control_host
+from TorCtl.StatsSupport import StatsHandler
+from TorCtl import PathSupport, TorCtl
+__selmgr = PathSupport.SelectionManager(
+ pathlen=3,
+ order_exits=True,
+ percent_fast=80,
+ percent_skip=0,
+ min_bw=1024,
+ use_all_exits=True,
+ uniform=True,
+ use_exit=None,
+ use_guards=True,
+ restrict_guards=True)
+
+class Connection(PathSupport.Connection):
+ """ thread quits when required number of circuits found, otherwise identical"""
+ def __init__(self,s):
+ PathSupport.Connection.__init__(self,s)
+ def _loop(self):
+ while 1:
+ try:
+ isEvent, reply = self._read_reply()
+ except:
+ self._err(sys.exc_info())
+ return
+
+ if isEvent:
+ if self._handler is not None:
+ self._eventQueue.put((time.time(), reply))
+ else:
+ cb = self._queue.get() # atomic..
+ cb(reply)
+
+ if self._handler is not None:
+ if self._handler.circ_failed + self._handler.circ_built >= self._handler.nstats:
+ print 'Finished gathering',self._handler.circ_failed + self._handler.circ_built,'circuits'
+ print self._handler.circ_failed,'failed',self._handler.circ_built,'built'
+ return
+
+class StatsGatherer(StatsHandler):
+ def __init__(self,c, selmgr,basefile_name,nstats):
+ StatsHandler.__init__(self,c, selmgr)
+
+ self.detailfile = open(basefile_name + '.detail','w')
+ self.buildtimesfile = open(basefile_name + '.buildtimes','w')
+ self.circ_built = 0
+ self.nstats = nstats
+
+ # sometimes relevant CircEvents occur before the circ_id is
+ # added to self.circuits, which means they get discarded
+ # we track them in self.othercircs: a dictionary of list of events
+ self.othercircs = {}
+
+ def circ_event_str(self,now,circ_event):
+ """ returns an string summarizing the circuit event"""
+ output = [circ_event.event_name, str(circ_event.circ_id),
+ circ_event.status]
+ if circ_event.path:
+ output.append(",".join(circ_event.path))
+ if circ_event.reason:
+ output.append("REASON=" + circ_event.reason)
+ if circ_event.remote_reason:
+ output.append("REMOTE_REASON=" + circ_event.remote_reason)
+ output = [now]+ output
+ outstr = ' '.join(output) + '\n'
+ return outstr
+
+ def add_missed_events(self,circ_id):
+ """ if there are events for a circuit that were missed, add them"""
+ if circ_id in self.othercircs:
+ for e_str in self.othercircs[circ_id]:
+ self.detailfile.write(e_str)
+ self.detailfile.flush()
+ # now in self.circuits, so can delete it from self.othercircs
+ del self.othercircs[circ_id]
+
+
+ def circ_status_event(self, circ_event):
+ """ handles circuit status event """
+ now = time.time()
+ now = '%3.10f' % now
+
+ if circ_event.circ_id in self.circuits.keys():
+ self.add_missed_events(circ_event.circ_id)
+ if circ_event.status == 'EXTENDED':
+ extend_time = circ_event.arrived_at-self.circuits[circ_event.circ_id].last_extended_at
+ self.circuits[circ_event.circ_id].extend_times.append(extend_time)
+ self.circuits[circ_event.circ_id].last_extended_at = circ_event.arrived_at
+
+ if circ_event.status == 'BUILT':
+ circ = self.circuits[circ_event.circ_id]
+ buildtime = reduce(lambda x,y:x+y,circ.extend_times,0.0)
+ self.buildtimesfile.write(str(circ.circ_id) + '\t' + str(buildtime) + '\n')
+ self.buildtimesfile.flush()
+
+ outstr = self.circ_event_str(now,circ_event)
+ self.detailfile.write(outstr)
+ self.detailfile.flush()
+
+ # check to see if done gathering data
+ if circ_event.status == 'BUILT': self.circ_built += 1
+ else:
+ #eventstr =
+ #if circ_event.circ_id in self.othercircs.keys():
+ if circ_event.circ_id not in self.othercircs.keys():
+ self.othercircs[circ_event.circ_id] = []
+ self.othercircs[circ_event.circ_id] += [self.circ_event_str(now,circ_event)]
+ StatsHandler.circ_status_event(self,circ_event)
+
+def getdata(filename,ncircuits):
+ """ starts stat gathering thread """
+
+ s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
+ s.connect((control_host,control_port))
+ c = Connection(s)
+ c.authenticate() # also launches thread...
+ h = StatsGatherer(c,__selmgr,filename,ncircuits)
+ c.set_event_handler(h)
+
+ c.set_events([TorCtl.EVENT_TYPE.STREAM,
+ TorCtl.EVENT_TYPE.BW,
+ TorCtl.EVENT_TYPE.NS,
+ TorCtl.EVENT_TYPE.CIRC,
+ TorCtl.EVENT_TYPE.STREAM_BW,
+ TorCtl.EVENT_TYPE.NEWDESC], True)
+ return c
+
+def setargs():
+ ncircuits = ""
+ dirname = ""
+ filename = ""
+ if len(sys.argv[1:]) < 3:
+ usage()
+ sys.exit(2)
+ try:
+ opts,args = getopt.getopt(sys.argv[1:],"p:n:d:")
+ except getopt.GetoptError,err:
+ print str(err)
+ usage()
+ ncircuits=None
+ percentile=None
+ dirname=""
+ for o,a in opts:
+ if o == '-n':
+ if a.isdigit(): ncircuits = int(a)
+ else: usage()
+ elif o == '-d': dirname = a #directory where output files go
+ elif o == '-p':
+ if a.isdigit(): percentile = int(a)
+ else: usage()
+ else:
+ assert False, "Bad option"
+ return ncircuits,percentile,dirname
+
+def usage():
+ print 'usage: statscontroller.py [-p <#percentile>] -n <# circuits> -d <output dir name>'
+ sys.exit(1)
+
+
+def guardslice(p,ncircuits,dirname):
+
+ print 'Making new directory:',dirname
+ if not os.path.isdir(dirname):
+ os.mkdir(dirname)
+ else:
+ print 'Directory',dirname,'exists, not making a new one.'
+
+ print 'Guard percentiles:',p,'to',p+5
+ print '#Circuits',ncircuits
+
+ basefile_name = dirname + '/' + str(p) + '-' + str(p+5) + '.' + str(ncircuits)
+ aggfile_name = basefile_name + '.agg'
+
+ __selmgr.percent_fast = p+5
+ __selmgr.percent_skip = p
+
+ c = getdata(basefile_name,ncircuits)
+
+ for i in xrange(0,ncircuits):
+ print 'Building circuit',i
+ try:
+ # XXX: hrmm.. race conditions on the path_selectior members
+ # for the event handler thread?
+ # Probably only if streams end up coming in during this test..
+ circ = c.build_circuit(__selmgr.pathlen,__selmgr.path_selector)
+ c._handler.circuits[circ.circ_id] = circ
+ except TorCtl.ErrorReply,e:
+ plog("NOTICE","Error building circuit: " + str(e.args))
+
+ while True:
+ time.sleep(1)
+ if c._handler.circ_built + c._handler.circ_failed >= ncircuits:
+ print 'Done gathering stats for slice',p,'to',p+5,'on',ncircuits
+ print c._handler.circ_built,'built',c._handler.circ_failed,'failed'
+ break
+ c._handler.write_stats(aggfile_name)
+
+def main():
+ ncircuits,p,dirname = setargs()
+
+ if p is None:
+ # do all
+ for p in xrange(0,100,5):
+ guardslice(p,ncircuits,dirname)
+ else:
+ guardslice(p,ncircuits,dirname)
+if __name__ == '__main__':
+ main()
Copied: torflow/trunk/CircuitAnalysis/numpy_pareto.py (from rev 17873, torflow/trunk/tools/BTAnalysis/numpy_pareto.py)
===================================================================
--- torflow/trunk/CircuitAnalysis/numpy_pareto.py (rev 0)
+++ torflow/trunk/CircuitAnalysis/numpy_pareto.py 2009-01-05 16:04:10 UTC (rev 17913)
@@ -0,0 +1,59 @@
+#!/usr/bin/python
+import numpy
+import pylab
+import matplotlib
+
+
+def loadbuildtimes():
+ f = open('40k_r1/45-50.40000.buildtimes')
+ vals = []
+ for line in f:
+ line = line.split('\t')
+ vals += [float(line[1].strip())*1000]
+ vals.sort()
+ vals.reverse()
+ return vals
+
+
+def pareto(x,k,Xm):
+ return k*(Xm**k)/(x**(k+1))
+
+#get buildtime data (in ms)
+Z = loadbuildtimes()
+
+# plot histogram.
+# args: values, number of bins, normalize y/n, width of bars
+
+pylab.hist(Z,len(Z) / 100.0, normed=True, width=5)
+
+#pareto parameters (taken from output of ./shufflebt.py buildtimes)
+#Resolution of histogram: 100 ms
+#Mean: 5746.8020777, mode: 1600
+#ParK: 0.918058347945
+#ModeN: 32775 vs integrated: 32394.9483089
+#successful runs: 41712
+
+k = 0.687880881456
+Xm = 1800
+n = 28921
+
+
+# args to a range: x start, x end
+X = pylab.arange(Xm, max(Z), 1) # max(Z), 0.1) # x values from 1 to max(Z) in increments of 0.1 (can adjust this to look at different parts of the graph)
+Y = map(lambda x: pareto(x,k,Xm), X) #pareto(x) (units: #measurements with value x)
+
+# verify sanity by integrating scaled distribution:
+modeNint = numpy.trapz(map(lambda x: n*pareto(x, k, Xm),
+ xrange(Xm,200000)))
+
+print modeNint
+
+print n*pareto(Xm, k, Xm)
+
+#draw pareto curve
+# X values plotted against Y values, will appear as blue circles b:blue o:circle
+pylab.plot(X,Y,'b-')
+
+#save figure
+pylab.savefig('paretofig.png')
+
Copied: torflow/trunk/CircuitAnalysis/shufflebt.py (from rev 17873, torflow/trunk/tools/BTAnalysis/shufflebt.py)
===================================================================
--- torflow/trunk/CircuitAnalysis/shufflebt.py (rev 0)
+++ torflow/trunk/CircuitAnalysis/shufflebt.py 2009-01-05 16:04:10 UTC (rev 17913)
@@ -0,0 +1,337 @@
+#!/usr/bin/env python
+# shufflebt.py
+# (c) Fallon Chen 2008
+# Shuffles a list of build times and produces a pdf of n of those buildtimes,
+# which are put into res (defaults to 100)ms blocks.
+# Requires gnuplot 4.2 and a version coreutils that provides sort -R
+# "usage: shufflebt.py [-n <number of circuits>] [-s] [-g] [-k <k value>] [-d outdirname] <list of filenames>"
+# if outdir is not specified, the script will write files to the current directory
+# if a directory is given instead of a list of filenames, all files postfixed with '.buildtimes' will be processed
+import getopt,sys,os
+import popen2
+import math,copy
+from scipy.integrate import *
+from numpy import trapz
+import numpy
+import pylab
+import matplotlib
+
+class Stats:
+ def __init__(self,file):
+ self.f = open(file)
+ self.values = []
+ for line in self.f:
+ line = line.split('\t')
+ self.values += [float(line[1]) * 1000]
+
+ self.f.close()
+ self.buckets = {}
+ def mean(self):
+ # Borrowed from TorUtil
+ if len(self.values) > 0:
+ sum = reduce(lambda x,y: x+y,self.values,0.0)
+ return sum/len(self.values)
+ else:
+ return 0.0
+ def stddev(self):
+ # Borrowed from TorUtil
+ if len(self.values) > 1:
+ mean = self.mean()
+ sum = reduce(lambda x,y: x + ((y-mean)**2.0),self.values,0.0)
+ s = math.sqrt(sum/(len(self.values)-1))
+ return s
+ else:
+ return 0.0
+ def median(self):
+ if len(self.values) > 0:
+ values = copy.copy(self.values)
+ values.sort()
+ return values[(len(values) - 1)/2]
+ else:
+ return 0.0
+
+ def mode(self): # Requires makehistogram runs first
+ counts = {}
+ greatest_val = 0
+ greatest_idx = 0
+ for v in self.buckets.keys():
+ if self.buckets[v] > greatest_val:
+ greatest_idx = v
+ greatest_val = self.buckets[v]
+ return greatest_idx
+
+
+ def pyhist(self,res,histname):
+ bins = len(self.values) / res
+ print 'bins:',bins
+ x = matplotlib.numerix.arange(1,7000, 0.01)
+ S = pypareto(x,0.918058347945, 1600.0, 32775.0)
+ #pylab.hist(self.values,bins=bins,normed=False, width=1)
+ #(n,bins) = numpy.histogram(self.values,bins=bins,normed=False)
+ #pylab.plot(bins,n )
+ pylab.plot(x,S, 'bo')
+ #pylab.show()
+ pylab.savefig(histname + '.png')
+
+ # XXX: This doesn't seem to work for small #s of circuits
+ def makehistogram(self,res,histname):
+ #res = res /1000.0 # convert ms to s
+ values = copy.copy(self.values)
+ values.sort()
+ count = 0
+ i = 1
+ self.buckets = {}
+ for v in values:
+ if v < res * i: count += 1
+ else:
+ count += 1
+ self.buckets[int(res * i)] = count
+ #self.buckets[int(res * i * 10)] = count
+ i += 1
+ count = 0
+ f = open(histname,'w')
+ f.write('#build time <\t#circuits\n')
+ sortedkeys = self.buckets.keys()
+ sortedkeys.sort()
+ for b in sortedkeys:
+ towrite = str(b) + '\t' + str(self.buckets[b]) + '\n'
+ f.write(towrite)
+ f.close()
+
+ def paretoK(self, Xm):
+ n = 0
+ log_sum = 0
+ X = min(self.values)
+ for x in self.values:
+ if x < Xm: continue
+ n += 1
+ log_sum += math.log(x)
+ return n/(log_sum - n*math.log(Xm))
+
+ # Calculate the mean beyond a mode value
+ def modeMean(self, Xm):
+ n = 0
+ tot = 0
+ for x in self.values:
+ if x < Xm: continue
+ n += 1
+ tot += x
+ return tot/n
+
+ def modeN(self, Xm):
+ n = 0
+ for x in self.values:
+ if x < Xm: continue
+ n += 1
+ return n
+
+ def maxlikelihood(self,k):
+ # theta estimator for gamma PDF
+ # maxlikelihood estimator
+ # theta = sum(values) / N*k
+ return 10*sum(self.values)/(k * len(self.values))
+
+ def bayesian(self,k):
+ # bayesian estimator for gamma PDF
+ # y = sum(values)
+ # theta = y/(Nk - 1) +/- y^2/((Nk-1)^2(Nk -2))
+ y = sum(self.values) * 10
+ N = len(self.values)
+ mean = y/(N*k - 1)
+ sdev = (y*y)/((N*k - 1)* (N*k - 1) * (N*k - 2))
+ plus = mean + sdev
+ minus = mean - sdev
+ return plus,minus
+
+## Functions that return a gnuplot function string for a given distribution
+def gamma(k,theta, N,fname):
+ # gnuplot string for gamma PDF
+ # g(x,k,B) = (x**(k - 1) * B**k * exp(-B*x))/gamma(k)
+ B = 1.0/theta
+
+ ps = fname + '(x) = '+str(N)+'*((x**' + str(k-1) + ')*(' +str(B**k)+ ')*(exp(-' + str(B) +'*x)))' +'/gamma('+str(k)+')\n'
+ return ps
+
+def pareto(k,Xm,N,fname):
+ # gnuplot string for shifted, normalized exponential PDF
+ # g(x,k,B) = (N * k*(Xm**k)/x**(k+1)))
+ ps = fname+'(x)=(x<='+str(Xm)+') ? 0 : (('+str((N*k)*(Xm**k))+')/((x)**('+str(k+1)+')))\n'
+ #ps = fname+'(x)='+str(N*k*(Xm**k))+'/x**('+str(k+1)+')\n'
+ return ps
+
+def pypareto(x, k,Xm):
+ # gnuplot string for shifted, normalized exponential PDF
+ # g(x,k,B) = (N * k*(Xm**k)/x**(k+1)))
+ if x<Xm: return 0
+ else: return ((((k)*(Xm**k)))/((x)**((k+1))))
+
+def exp(mean,shift,N,fname):
+ # gnuplot string for normalized exponential PDF
+ # g(x,k,B) = N * l*exp(-l*(x-shift))
+ l = 1.0/mean
+ ps = fname+'(x)=(x<'+str(shift)+')?0:('+str(N*l)+'*exp(-abs('+str(l)+'*(x-'+str(shift)+'))))\n'
+ return ps
+
+def shiftedExp(mean,shift,N,fname):
+ # gnuplot string for shifted, normalized exponential PDF
+ # g(x,k,B) = N * l*exp(-l*(x-shift))/(1+(1-exp(-l*shift)))
+ l = 1.0/mean
+ ps = fname+'(x)='+str(N*l)+'*exp(-abs('+str(l)+'*(x-'+str(shift)+')))/(1+(1-exp(-'+str(l*shift)+')))\n'
+ return ps
+
+def poisson(u,N,fname):
+ ps = fname + "(x) = " + str(N) + "*(" + str(u) + "**x)*exp(-"+str(u)+")/gamma(x + 1)\n"
+ return ps
+
+def normal(u,d,N,fname):
+ ps = fname + "(x)="+str(int(N)/d)+"*(exp(-((x-"+str(u)+ ")**2)/"+str(2*d*d)+"))/sqrt(2*pi)\n"
+ return ps
+
+
+def usage():
+ print "usage: shufflebt.py [-n <number of circuits>] [-s] [-g] [-k <k value>] [-d outdirname] [-r <res in ms>] <list of filenames>"
+ sys.exit(1)
+
+def intermediate_filename(infile,shuffle,truncate,outdir):
+
+ if not shuffle and not truncate: return os.path.abspath(infile)
+
+ intermediate = [os.path.join(os.path.abspath(outdir),os.path.basename(infile))]
+ if truncate: intermediate.append(str(truncate))
+ if shuffle:
+ intermediate.append('shuffled')
+ return '.'.join(intermediate)
+
+def histogram_basefilename(infile,shuffle,truncate,res,outdir):
+ name = [os.path.join(os.path.abspath(outdir),os.path.basename(infile))]
+
+ if truncate: name.append(str(truncate))
+ if shuffle: name.append('shuffled')
+ name.append('res' + str(res))
+ return '.'.join(name)
+
+def getargs():
+ # [-n <truncate to # circuits>] [-s] <list of filenames>
+ k = 3
+ res = 100
+ sort =False
+ truncate = None
+ graph = False
+ outdirname = "." # will write to current directory if not specified
+ filenames = []
+ if len(sys.argv) < 2: usage()
+ else:
+ arglen = len(sys.argv[1:])
+ i = 0
+ while (arglen - i) > 0:
+ if sys.argv[i+1] == '-s': sort = True
+ elif sys.argv[i+1] == '-n':
+ if not sys.argv[i + 2].isdigit(): usage()
+ truncate = sys.argv[i+2]
+ i += 1
+ elif sys.argv[i + 1] == '-g': graph = True
+ elif sys.argv[i + 1] == '-k':
+ k = float(sys.argv[i + 2])
+ i += 1
+ elif sys.argv[i+1] == '-d':
+ outdirname = sys.argv[i + 2]
+ i += 1
+ elif sys.argv[i+1] == '-r':
+ res = float(sys.argv[i+2])
+ i += 1
+ else:
+ filenames += [sys.argv[i+1]]
+ i += 1
+
+
+ return sort, truncate,graph,outdirname,filenames,k,res
+
+
+def shuffle(sort,truncate,filename,newfile):
+ if not sort and truncate is None: return
+ sortlocation = '/usr/local/bin/sort' #peculiarity of fallon's system
+ #sortlocation = 'sort'
+ if sort and truncate:
+ cmd = sortlocation + ' -R ' + filename + ' | head -n ' + truncate + ' > ' + newfile
+ elif sort and not truncate:
+ cmd = sortlocation + ' -R ' + filename + ' > ' + newfile
+ elif not sort and truncate:
+ cmd = 'cat ' + filename + ' | head -n ' + truncate + ' > ' + newfile
+
+ p = popen2.Popen4(cmd)
+ p.wait()
+
+if __name__ == "__main__":
+ sort, truncate,graph,dirname,filenames,k,res = getargs()
+
+ # make new directory
+ print 'Making new directory:',dirname
+ if not os.path.isdir(dirname):
+ os.mkdir(dirname)
+ else:
+ print 'Dir exists, not making a new one'
+
+ for filename in filenames:
+ if os.path.isdir(filename):
+ # shallow add of files in dir
+ for f in os.listdir(filename):
+ if f[-11:] == '.buildtimes':
+ filenames += [os.path.join(filename,f)]
+ filenames.remove(filename)
+
+ for filename in filenames:
+ print 'Processing',filename
+ print '------------------------------'
+ if not os.path.exists(filename):
+ print filename,'is not a valid path'
+ continue
+# if truncate and sort or truncate and not sort:
+# newfile = os.path.join(dirname, os.path.basename(filename) + '.' + truncate + '.shuffled')
+# elif sort and not truncate:
+# newfile = os.path.join(dirname , os.path.basename(filename) + '.shuffled')
+# else:
+# newfile = filename
+ newfile = intermediate_filename(filename,sort,truncate,dirname)
+ # shuffle, create new file
+ shuffle(sort,truncate,filename,newfile)
+
+ # create histogram from file
+ s = Stats(newfile)
+ histfilename = histogram_basefilename(filename,sort,truncate,res,dirname)
+ s.makehistogram(res,histfilename + '.hist')
+ mean = s.mean()
+ stddev = s.stddev()
+ median = s.median()
+ mode = s.mode() # relies on s.makehistogram for buckets
+ parK = s.paretoK(mode)
+ modeN = s.modeN(mode)
+ modeMean = s.modeMean(mode)
+ # verify sanity by integrating scaled distribution:
+ modeNint = trapz(map(lambda x: modeN* pypareto(x, parK, mode),
+ xrange(1,200000)))
+
+ print 'Resolution of histogram:',res,'ms'
+ print 'Mean: '+str(mean)+', mode: '+str(mode)
+ print 'ParK: '+str(parK)
+ print 'ModeN: '+str(modeN)+" vs integrated: "+str(modeNint)
+ print '#successful runs:',len(s.values)
+ # get stats
+
+ if graph:
+ # plot histogram
+ # args: values, # bins, normalize y/n, width of bars
+ pylab.hist(s.values,len(s.values) / res, normed=True,width=5)
+
+ #plot Pareto curve
+ X = pylab.arange(mode, max(s.values), 1)
+ Y = map(lambda x: pypareto(x, parK, mode), X)
+ n = len(s.values)
+
+
+ pylab.plot(X,Y,'b-')
+
+ #save figure
+ pylab.savefig(histfilename + '.png')
+ pylab.clf()
+
+
Deleted: torflow/trunk/guardslices.py
===================================================================
--- torflow/trunk/guardslices.py 2009-01-05 14:32:58 UTC (rev 17912)
+++ torflow/trunk/guardslices.py 2009-01-05 16:04:10 UTC (rev 17913)
@@ -1,212 +0,0 @@
-#!/usr/bin/env python
-# uses metatroller to collect circuit build times for 5% slices of guard nodes
-# [OUTPUT] one directory, with three files: StatsHandler aggregate stats file, file with all circuit events (for detailed reference), file with just buildtimes
-
-import socket,sys,time,getopt,os
-from TorCtl.TorUtil import meta_port,meta_host,control_port,control_host
-from TorCtl.StatsSupport import StatsHandler
-from TorCtl import PathSupport, TorCtl
-__selmgr = PathSupport.SelectionManager(
- pathlen=3,
- order_exits=True,
- percent_fast=80,
- percent_skip=0,
- min_bw=1024,
- use_all_exits=True,
- uniform=True,
- use_exit=None,
- use_guards=True,
- restrict_guards=True)
-
-class Connection(PathSupport.Connection):
-
- """ thread quits when required number of circuits found, otherwise identical"""
- def __init__(self,s):
- PathSupport.Connection.__init__(self,s)
- def _loop(self):
- while 1:
- try:
- isEvent, reply = self._read_reply()
- except:
- self._err(sys.exc_info())
- return
-
- if isEvent:
- if self._handler is not None:
- self._eventQueue.put((time.time(), reply))
- else:
- cb = self._queue.get() # atomic..
- cb(reply)
-
- if self._handler is not None:
- if self._handler.circ_failed + self._handler.circ_built >= self._handler.nstats:
- print 'Finished gathering',self._handler.circ_failed + self._handler.circ_built,'circuits'
- print self._handler.circ_failed,'failed',self._handler.circ_built,'built'
- return
-
-class StatsGatherer(StatsHandler):
- def __init__(self,c, selmgr,basefile_name,nstats):
- StatsHandler.__init__(self,c, selmgr)
-
- self.detailfile = open(basefile_name + '.detail','w')
- self.buildtimesfile = open(basefile_name + '.buildtimes','w')
- self.circ_built = 0
- self.nstats = nstats
-
- # sometimes relevant CircEvents occur before the circ_id is
- # added to self.circuits, which means they get discarded
- # we track them in self.othercircs: a dictionary of list of events
- self.othercircs = {}
-
- def circ_event_str(self,now,circ_event):
- """ returns an string summarizing the circuit event"""
- output = [circ_event.event_name, str(circ_event.circ_id),
- circ_event.status]
- if circ_event.path:
- output.append(",".join(circ_event.path))
- if circ_event.reason:
- output.append("REASON=" + circ_event.reason)
- if circ_event.remote_reason:
- output.append("REMOTE_REASON=" + circ_event.remote_reason)
- output = [now]+ output
- outstr = ' '.join(output) + '\n'
- return outstr
-
- def add_missed_events(self,circ_id):
- """ if there are events for a circuit that were missed, add them"""
- if circ_id in self.othercircs:
- for e_str in self.othercircs[circ_id]:
- self.detailfile.write(e_str)
- self.detailfile.flush()
- # now in self.circuits, so can delete it from self.othercircs
- del self.othercircs[circ_id]
-
-
- def circ_status_event(self, circ_event):
- """ handles circuit status event """
- now = time.time()
- now = '%3.10f' % now
-
- if circ_event.circ_id in self.circuits.keys():
- self.add_missed_events(circ_event.circ_id)
- if circ_event.status == 'EXTENDED':
- extend_time = circ_event.arrived_at-self.circuits[circ_event.circ_id].last_extended_at
- self.circuits[circ_event.circ_id].extend_times.append(extend_time)
- self.circuits[circ_event.circ_id].last_extended_at = circ_event.arrived_at
-
- if circ_event.status == 'BUILT':
- circ = self.circuits[circ_event.circ_id]
- buildtime = reduce(lambda x,y:x+y,circ.extend_times,0.0)
- self.buildtimesfile.write(str(circ.circ_id) + '\t' + str(buildtime) + '\n')
- self.buildtimesfile.flush()
-
- outstr = self.circ_event_str(now,circ_event)
- self.detailfile.write(outstr)
- self.detailfile.flush()
-
- # check to see if done gathering data
- if circ_event.status == 'BUILT': self.circ_built += 1
- else:
- #eventstr =
- #if circ_event.circ_id in self.othercircs.keys():
- if circ_event.circ_id not in self.othercircs.keys():
- self.othercircs[circ_event.circ_id] = []
- self.othercircs[circ_event.circ_id] += [self.circ_event_str(now,circ_event)]
- StatsHandler.circ_status_event(self,circ_event)
-
-def getdata(filename,ncircuits):
- """ starts stat gathering thread """
-
- s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
- s.connect((control_host,control_port))
- c = Connection(s)
- c.authenticate() # also launches thread...
- h = StatsGatherer(c,__selmgr,filename,ncircuits)
- c.set_event_handler(h)
-
- c.set_events([TorCtl.EVENT_TYPE.STREAM,
- TorCtl.EVENT_TYPE.BW,
- TorCtl.EVENT_TYPE.NS,
- TorCtl.EVENT_TYPE.CIRC,
- TorCtl.EVENT_TYPE.STREAM_BW,
- TorCtl.EVENT_TYPE.NEWDESC], True)
- return c
-def setargs():
- ncircuits = ""
- dirname = ""
- filename = ""
- if len(sys.argv[1:]) < 3:
- usage()
- sys.exit(2)
- try:
- opts,args = getopt.getopt(sys.argv[1:],"p:n:d:")
- except getopt.GetoptError,err:
- print str(err)
- usage()
- ncircuits=None
- percentile=None
- dirname=""
- for o,a in opts:
- if o == '-n':
- if a.isdigit(): ncircuits = int(a)
- else: usage()
- elif o == '-d': dirname = a #directory where output files go
- elif o == '-p':
- if a.isdigit(): percentile = int(a)
- else: usage()
- else:
- assert False, "Bad option"
- return ncircuits,percentile,dirname
-
-def usage():
- print 'usage: statscontroller.py [-p <#percentile>] -n <# circuits> -d <output dir name>'
- sys.exit(1)
-
-
-def guardslice(p,ncircuits,dirname):
-
- print 'Making new directory:',dirname
- if not os.path.isdir(dirname):
- os.mkdir(dirname)
- else:
- print 'Directory',dirname,'exists, not making a new one.'
-
- print 'Guard percentiles:',p,'to',p+5
- print '#Circuits',ncircuits
-
- basefile_name = dirname + '/' + str(p) + '-' + str(p+5) + '.' + str(ncircuits)
- aggfile_name = basefile_name + '.agg'
-
- __selmgr.percent_fast = p+5
- __selmgr.percent_skip = p
-
- c = getdata(basefile_name,ncircuits)
-
- for i in range(0,ncircuits):
- print 'Building circuit',i
- try:
- circ = c.build_circuit(__selmgr.pathlen,__selmgr.path_selector)
- c._handler.circuits[circ.circ_id] = circ
- except TorCtl.ErrorReply,e:
- plog("NOTICE","Error building circuit: " + str(e.args))
-
- while True:
- if c._handler.circ_built + c._handler.circ_failed >= ncircuits:
- print 'Done gathering stats for slice',p,'to',p+5,'on',ncircuits
- print c._handler.circ_built,'built',c._handler.circ_failed,'failed'
- break
- c._handler.write_stats(aggfile_name)
-
-def main():
-
-
- ncircuits,p,dirname = setargs()
-
- if p is None:
- # do all
- for p in range(0,100,5):
- guardslice(p,ncircuits,dirname)
- else:
- guardslice(p,ncircuits,dirname)
-if __name__ == '__main__':
- main()
Copied: torflow/trunk/moniTor.py (from rev 17873, torflow/trunk/tools/moniTor.py)
===================================================================
--- torflow/trunk/moniTor.py (rev 0)
+++ torflow/trunk/moniTor.py 2009-01-05 16:04:10 UTC (rev 17913)
@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+#
+#
+# This is a "top-like" interface for Tor information
+# It's goal at the start is to just tell you basic information
+# In the future, you may be able to control Tor with it.
+#
+# See this for some of the original ideas:
+# http://archives.seul.org/or/dev/Jan-2008/msg00005.html
+#
+# A typical output of moniTor could look like this (with some fake data
+# for the purpose of this example):
+#
+# ~ Name/ID: gabelmoo 6833 3D07 61BC F397 A587 A0C0 B963 E4A9 E99E C4D3
+# ~ Version: 0.2.0.15-alpha-dev (r13077) on Linux x86_64
+# ~ Config: /home/tor/gabelmoo/torrc, Exit policy: no exit allowed
+# ~ IP address: 88.198.7.215, OR port: 443, Dir port: 80
+#
+# ~ CPU: 9.0% this tor, 3.4% other processes, 87.6% idle
+# ~ Mem: 49.9% this tor, 2.0% other processes, 48.1% free
+# ~ Connections: 1090 OR conns, 320 Dir conns
+# ~ Bandwidth: 1.2 MB/s current, 1.3 MB/s avg
+#
+# ~ Recent events (see also /home/tor/gabelmoo/monitor.log):
+# ~ 14:30:01 [warn] Consensus does not include configured authority 'moria
+# ~ 14:30:01 [warn] Consensus does not include configured authority 'ides'
+# ~ 14:30:01 [warn] 0 unknown, 0 missing key, 2 good, 0 bad, 1 no signatur
+# ~ 14:30:01 [warn] Not enough info to publish pending consensus
+#
+
+
+__author__ = "Jacob Appelbaum"
+__version__ = "0.1-2008_01_16"
+__copyright__ = "http://www.torproject.org/Jacob Appelbaum 2008"
+
+import curses
+import time
+import sys
+import socket
+
+# Hack.. Can also set PYTHONPATH..
+# http://docs.python.org/tut/node8.html#searchPath
+sys.path.append('../')
+from TorCtl import TorCtl, TorUtil
+from TorCtl.TorCtl import *
+
+# Parse authenticate string from file here
+
+#moniTorConf = "/etc/moniTor.conf"
+#authSecret = open(moniTorConf).read().strip()
+authSecret = ""
+
+def parse_config():
+
+ #moniTorConf = "/etc/moniTor.conf"
+ #authSecret = open(moniTorConf).read().strip()
+ #authSecret = ""
+
+ return
+
+def create_oracle(host,port):
+ """ Create a useful TorCtl object
+ """
+ print "I'm going to connect to %s and connect to port %i" %(sh,sp)
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.connect((host,port))
+ oracle = Connection(s)
+ oracle_thread = oracle.launch_thread()
+ oracle.authenticate(authSecret)
+
+ return oracle, oracle_thread
+
+# Much like run_example from TorCtl
+def collect_status(oracle):
+ """ A basic loop for collecting static information from our TorCtl object
+ """
+ # add name/id, exit policy, or-port, dir-port
+
+ static_keys = ['version', 'config-file', 'address', 'fingerprint', 'exit-policy/default', 'accounting/enabled']
+ static_info = dict([(key, oracle.get_info(key)[key]) for key in static_keys])
+
+ # Dynamic information can be collected by using our returned socket
+ return static_info, static_keys
+
+if __name__ == '__main__':
+ if len(sys.argv) > 1:
+ print "Syntax: ",sys.argv[0]
+ sys.exit(1)
+ else:
+ sys.argv.append("localhost:9051")
+
+ parse_config()
+ sh,sp = parseHostAndPort(sys.argv[1])
+
+ torctl_oracle, torctl_oracle_thread = create_oracle(sh,sp)
+ static_info, static_keys, = collect_status(torctl_oracle)
+
+ # Number of connections, current bw
+ dynamic_keys = ['version', 'config-file', 'address', 'fingerprint']
+
+ torctl_oracle.set_event_handler(DebugEventHandler())
+ torctl_oracle.set_events([EVENT_TYPE.STREAM, EVENT_TYPE.CIRC,
+ EVENT_TYPE.NS, EVENT_TYPE.NEWDESC,
+ EVENT_TYPE.ORCONN, EVENT_TYPE.BW], True)
+
+
+ while True:
+ # Populate the dynamic info each run
+ dynamic_info = dict([(key, torctl_oracle.get_info(key)[key]) for key in dynamic_keys])
+
+ # Now we can draw a few interesting things to the screen
+ for key in static_info:
+ print key + " is " + static_info[key]
+
+ for key in dynamic_info:
+ print key + " is " + dynamic_info[key]
+
+ time.sleep(1)
+ # So ghetto, so ghetto!
+ os.system('clear')
+
Deleted: torflow/trunk/tools/moniTor.py
===================================================================
--- torflow/trunk/tools/moniTor.py 2009-01-05 14:32:58 UTC (rev 17912)
+++ torflow/trunk/tools/moniTor.py 2009-01-05 16:04:10 UTC (rev 17913)
@@ -1,121 +0,0 @@
-#!/usr/bin/env python
-#
-#
-# This is a "top-like" interface for Tor information
-# It's goal at the start is to just tell you basic information
-# In the future, you may be able to control Tor with it.
-#
-# See this for some of the original ideas:
-# http://archives.seul.org/or/dev/Jan-2008/msg00005.html
-#
-# A typical output of moniTor could look like this (with some fake data
-# for the purpose of this example):
-#
-# ~ Name/ID: gabelmoo 6833 3D07 61BC F397 A587 A0C0 B963 E4A9 E99E C4D3
-# ~ Version: 0.2.0.15-alpha-dev (r13077) on Linux x86_64
-# ~ Config: /home/tor/gabelmoo/torrc, Exit policy: no exit allowed
-# ~ IP address: 88.198.7.215, OR port: 443, Dir port: 80
-#
-# ~ CPU: 9.0% this tor, 3.4% other processes, 87.6% idle
-# ~ Mem: 49.9% this tor, 2.0% other processes, 48.1% free
-# ~ Connections: 1090 OR conns, 320 Dir conns
-# ~ Bandwidth: 1.2 MB/s current, 1.3 MB/s avg
-#
-# ~ Recent events (see also /home/tor/gabelmoo/monitor.log):
-# ~ 14:30:01 [warn] Consensus does not include configured authority 'moria
-# ~ 14:30:01 [warn] Consensus does not include configured authority 'ides'
-# ~ 14:30:01 [warn] 0 unknown, 0 missing key, 2 good, 0 bad, 1 no signatur
-# ~ 14:30:01 [warn] Not enough info to publish pending consensus
-#
-
-
-__author__ = "Jacob Appelbaum"
-__version__ = "0.1-2008_01_16"
-__copyright__ = "http://www.torproject.org/Jacob Appelbaum 2008"
-
-import curses
-import time
-import sys
-import socket
-
-# Hack.. Can also set PYTHONPATH..
-# http://docs.python.org/tut/node8.html#searchPath
-sys.path.append('../')
-from TorCtl import TorCtl, TorUtil
-from TorCtl.TorCtl import *
-
-# Parse authenticate string from file here
-
-#moniTorConf = "/etc/moniTor.conf"
-#authSecret = open(moniTorConf).read().strip()
-authSecret = ""
-
-def parse_config():
-
- #moniTorConf = "/etc/moniTor.conf"
- #authSecret = open(moniTorConf).read().strip()
- #authSecret = ""
-
- return
-
-def create_oracle(host,port):
- """ Create a useful TorCtl object
- """
- print "I'm going to connect to %s and connect to port %i" %(sh,sp)
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- s.connect((host,port))
- oracle = Connection(s)
- oracle_thread = oracle.launch_thread()
- oracle.authenticate(authSecret)
-
- return oracle, oracle_thread
-
-# Much like run_example from TorCtl
-def collect_status(oracle):
- """ A basic loop for collecting static information from our TorCtl object
- """
- # add name/id, exit policy, or-port, dir-port
-
- static_keys = ['version', 'config-file', 'address', 'fingerprint', 'exit-policy/default', 'accounting/enabled']
- static_info = dict([(key, oracle.get_info(key)[key]) for key in static_keys])
-
- # Dynamic information can be collected by using our returned socket
- return static_info, static_keys
-
-if __name__ == '__main__':
- if len(sys.argv) > 1:
- print "Syntax: ",sys.argv[0]
- sys.exit(1)
- else:
- sys.argv.append("localhost:9051")
-
- parse_config()
- sh,sp = parseHostAndPort(sys.argv[1])
-
- torctl_oracle, torctl_oracle_thread = create_oracle(sh,sp)
- static_info, static_keys, = collect_status(torctl_oracle)
-
- # Number of connections, current bw
- dynamic_keys = ['version', 'config-file', 'address', 'fingerprint']
-
- torctl_oracle.set_event_handler(DebugEventHandler())
- torctl_oracle.set_events([EVENT_TYPE.STREAM, EVENT_TYPE.CIRC,
- EVENT_TYPE.NS, EVENT_TYPE.NEWDESC,
- EVENT_TYPE.ORCONN, EVENT_TYPE.BW], True)
-
-
- while True:
- # Populate the dynamic info each run
- dynamic_info = dict([(key, torctl_oracle.get_info(key)[key]) for key in dynamic_keys])
-
- # Now we can draw a few interesting things to the screen
- for key in static_info:
- print key + " is " + static_info[key]
-
- for key in dynamic_info:
- print key + " is " + dynamic_info[key]
-
- time.sleep(1)
- # So ghetto, so ghetto!
- os.system('clear')
-
More information about the tor-commits
mailing list