[or-cvs] r19623: {torflow} Directory reorg, minor aggregator fixes, and stats output fi (in torflow/trunk/NetworkScanners/BwAuthority: . data data/scanner.1 data/scanner.2 data/scanner.3)
mikeperry at seul.org
mikeperry at seul.org
Wed Jun 3 20:18:39 UTC 2009
Author: mikeperry
Date: 2009-06-03 16:18:39 -0400 (Wed, 03 Jun 2009)
New Revision: 19623
Added:
torflow/trunk/NetworkScanners/BwAuthority/data/
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
Removed:
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
torflow/trunk/NetworkScanners/BwAuthority/scanner.1/
torflow/trunk/NetworkScanners/BwAuthority/scanner.2/
torflow/trunk/NetworkScanners/BwAuthority/scanner.3/
Modified:
torflow/trunk/NetworkScanners/BwAuthority/aggregate.py
torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/torrc
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/torrc
torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/torrc
torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh
Log:
Directory reorg, minor aggregator fixes, and stats output filter
fix to handle the fact that consensus updates can change our
percentiles during a scan and cause us to report unmeasured
nodes and not report measured nodes.
Modified: torflow/trunk/NetworkScanners/BwAuthority/aggregate.py
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/aggregate.py 2009-06-03 17:40:13 UTC (rev 19622)
+++ torflow/trunk/NetworkScanners/BwAuthority/aggregate.py 2009-06-03 20:18:39 UTC (rev 19623)
@@ -7,11 +7,11 @@
bw_files = {}
nodes = {}
-# XXX: Do not generate output file if we don't have a 'done' file
-# from each scanner.
# XXX: Alpha smoothing, here or in tor?
-# XXX: Consolidate scanner.1, scanner.2, scanner.3 into one dir
-# so it is easier to traverse them.
+# - Tor, because we want to be able to opine about stuff
+# we did not measure this round
+# - Or maybe that's an argument for reading the whole consensus
+# and doing it here.. but that will always be one behind..
def base10_round(bw_val):
# This keeps the first 3 decimal digits of the bw value only
@@ -79,23 +79,32 @@
self.ns_bw = int(re.search("[\s]*ns_bw=([\S]+)[\s]*", line).group(1))
def main(argv):
- for d in argv[1:-1]:
+ for da in argv[1:-1]:
# First, create a list of the most recent files in the
# scan dirs that are recent enough
- for root, dirs, files in os.walk(d):
- for f in files:
- if re.search("^bws-[\S]+-done-", f):
- fp = file(d+"/"+f, "r")
- ranks = fp.readline()
- timestamp = float(fp.readline())
- fp.close()
- if ranks not in bw_files or bw_files[ranks][0] < timestamp:
- bw_files[ranks] = (timestamp, d+"/"+f)
+ for root, dirs, f in os.walk(da):
+ for ds in dirs:
+ print ds
+ if re.match("^scanner.[\d+]$", ds):
+ print ds
+ for sr, sd, files in os.walk(da+"/"+ds+"/scan-data"):
+ for f in files:
+ if re.search("^bws-[\S]+-done-", f):
+ print sr+"/"+f
+ found_done = True
+ fp = file(sr+"/"+f, "r")
+ slicenum = sr+"/"+fp.readline()
+ timestamp = float(fp.readline())
+ fp.close()
+ if slicenum not in bw_files \
+ or bw_files[slicenum][0] < timestamp:
+ bw_files[slicenum] = (timestamp, sr+"/"+f)
+
for (t,f) in bw_files.itervalues():
fp = file(f, "r")
- fp.readline()
- fp.readline()
+ fp.readline() # slicenum
+ fp.readline() # timestamp
for l in fp.readlines():
try:
line = Line(l)
Modified: torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py 2009-06-03 17:40:13 UTC (rev 19622)
+++ torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py 2009-06-03 20:18:39 UTC (rev 19623)
@@ -71,6 +71,7 @@
nodes_per_slice = config.getint('BwAuthority', 'nodes_per_slice')
save_every = config.getint('BwAuthority', 'save_every')
circs_per_node = config.getint('BwAuthority', 'circs_per_node')
+ min_streams = config.getint('BwAuthority', 'min_streams')
out_dir = config.get('BwAuthority', 'out_dir')
tor_dir = config.get('BwAuthority', 'tor_dir')
max_fetch_time = config.getint('BwAuthority', 'max_fetch_time')
@@ -83,7 +84,7 @@
return (start_pct,stop_pct,nodes_per_slice,save_every,
circs_per_node,out_dir,max_fetch_time,tor_dir,
- sleep_start,sleep_stop)
+ sleep_start,sleep_stop,min_streams)
def choose_url(percentile):
for (pct, url) in urls:
@@ -106,16 +107,15 @@
self.add_event_listener(self.sql_consensus_listener)
self.add_event_listener(SQLSupport.StreamListener())
- def write_sql_stats(self, percent_skip, percent_fast, rfilename=None):
+ def write_sql_stats(self, stats_filter=None, rfilename=None):
if not rfilename:
rfilename="./data/stats/sql-"+time.strftime("20%y-%m-%d-%H:%M:%S")
cond = threading.Condition()
def notlambda(h):
cond.acquire()
SQLSupport.RouterStats.write_stats(file(rfilename, "w"),
- percent_skip, percent_fast,
- order_by=SQLSupport.RouterStats.sbw,
- recompute=True)
+ 0, 100, order_by=SQLSupport.RouterStats.sbw,
+ recompute=True, stat_clause=stats_filter)
cond.notify()
cond.release()
cond.acquire()
@@ -123,19 +123,17 @@
cond.wait()
cond.release()
- def write_strm_bws(self, percent_skip, percent_fast, rfilename=None):
+ def write_strm_bws(self, slice_num, stats_filter=None, rfilename=None):
if not rfilename:
rfilename="./data/stats/bws-"+time.strftime("20%y-%m-%d-%H:%M:%S")
cond = threading.Condition()
def notlambda(this):
cond.acquire()
f=file(rfilename, "w")
- f.write("low="+str(int(round((percent_skip*len(this.sorted_r))/100.0,0)))
- +" hi="+str(int(round((percent_fast*len(this.sorted_r))/100.0,0)))
- +"\n")
- SQLSupport.RouterStats.write_bws(f, percent_skip, percent_fast,
+ f.write("slicenum="+str(slice_num)+"\n")
+ SQLSupport.RouterStats.write_bws(f, 0, 100,
order_by=SQLSupport.RouterStats.sbw,
- recompute=False)
+ recompute=False, stat_clase=stats_filter)
f.close()
cond.notify()
cond.release()
@@ -240,8 +238,9 @@
def notlambda(this):
cond.acquire()
# TODO: Using the entry_gen router list is somewhat ghetto..
- if this.selmgr.path_selector.entry_gen.rstr_routers and \
- this.selmgr.path_selector.exit_gen.rstr_routers:
+ if (not this.selmgr.bad_restrictions) or \
+ (this.selmgr.path_selector.entry_gen.rstr_routers and \
+ this.selmgr.path_selector.exit_gen.rstr_routers):
for r in this.selmgr.path_selector.entry_gen.rstr_routers:
if r._generated[position] < count:
cond._finished = False
@@ -381,15 +380,17 @@
hi = str(round(stop_pct,1))
# Warning, don't remove the sql stats without changing the recompute
# param in write_strm_bws to True
- hdlr.write_sql_stats(start_pct, stop_pct, os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-"+str(successful)+"-"+race_time)
- hdlr.write_strm_bws(start_pct, stop_pct, os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-"+str(successful)+"-"+race_time)
+ hdlr.write_sql_stats(os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-"+str(successful)+"-"+race_time, stats_filter=SQLSupport.RouterStats.circ_chosen >= 1)
+ hdlr.write_strm_bws(os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-"+str(successful)+"-"+race_time, stats_filter=SQLSupport.RouterStats.strm_chosen >= 1)
+
plog('INFO', str(start_pct) + '-' + str(stop_pct) + '% ' + str(successful) + ' fetches took ' + str(attempt) + ' tries.')
def main(argv):
TorUtil.read_config(argv[1])
(start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir,
- max_fetch_time,tor_dir,sleep_start,sleep_stop) = read_config(argv[1])
+ max_fetch_time,tor_dir,sleep_start,sleep_stop,
+ min_streams) = read_config(argv[1])
try:
(c,hdlr) = setup_handler(out_dir, tor_dir+"/control_auth_cookie")
@@ -408,7 +409,7 @@
while True:
pct = start_pct
plog('INFO', 'Beginning time loop')
-
+ slice_num = 0
while pct < stop_pct:
pct_step = hdlr.rank_to_percent(nodes_per_slice)
hdlr.reset_stats()
@@ -424,14 +425,21 @@
lo = str(round(pct,1))
hi = str(round(pct+pct_step,1))
-
- # Warning, don't remove the sql stats without changing the recompute
+ # There may be a consensus change between the point of speed
+ # racing and the writing of stats causing a discrepency
+ # between the immediate, current consensus result used to determine
+ # termination and this average-based result :(
+ # We may need a filter based on circuit chosen and not percentage, or
+ # we may need to convert this to be most recent rank, and not avg
+ # (the latter still leaves a small race condition).
+ hdlr.write_sql_stats(os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"), stats_clause=SQLSupport.RouterStats.circ_chosen >= 1)
+ # Warning, don't remove the sql stats call without changing the recompute
# param in write_strm_bws to True
- hdlr.write_sql_stats(pct, pct+pct_step, os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"))
- hdlr.write_strm_bws(pct, pct+pct_step, os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"))
+ hdlr.write_strm_bws(os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"), slice_num, stats_clause=SQLSupport.RouterStats.strm_closed >= min_streams)
plog('DEBUG', 'Wrote stats')
pct += pct_step
hdlr.save_sql_file(sql_file, os.getcwd()+"/"+out_dir+"/db-"+str(lo)+":"+str(hi)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S")+".sqlite")
+ slice_num += 1
def cleanup(c, f):
plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors="+f)
Property changes on: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1
___________________________________________________________________
Added: svn:mergeinfo
+
Deleted: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.1/bwauthority.cfg 2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg 2009-06-03 20:18:39 UTC (rev 19623)
@@ -1,20 +0,0 @@
-[TorCtl]
-loglevel=DEBUG
-tor_host = 127.0.0.1
-tor_port = 9110
-control_host = 127.0.0.1
-control_port = 9111
-control_pass =
-# XXX: Unused
-meta_host = 127.0.0.1
-meta_port = 9112
-
-[BwAuthority]
-out_dir = ./scanner.1/scan-data
-tor_dir = ./scanner.1/tor-data
-start_pct = 0
-stop_pct = 50
-save_every = 10
-nodes_per_slice = 50
-circs_per_node = 2
-max_fetch_time = 1800
Copied: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg (from rev 19617, torflow/trunk/NetworkScanners/BwAuthority/scanner.1/bwauthority.cfg)
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg 2009-06-03 20:18:39 UTC (rev 19623)
@@ -0,0 +1,23 @@
+[TorCtl]
+loglevel=DEBUG
+tor_host = 127.0.0.1
+tor_port = 9110
+control_host = 127.0.0.1
+control_port = 9111
+control_pass =
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9112
+
+[BwAuthority]
+out_dir = ./data/scanner.1/scan-data
+tor_dir = ./data/scanner.1/tor-data
+start_pct = 0
+stop_pct = 20
+save_every = 10
+nodes_per_slice = 50
+circs_per_node = 2
+min_streams = 1
+max_fetch_time = 1800
+sleep_start = 01:30
+sleep_stop = 05:30
Modified: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/torrc
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.1/torrc 2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/torrc 2009-06-03 20:18:39 UTC (rev 19623)
@@ -2,9 +2,9 @@
SocksPort 9110 # what port to advertise for application connections
ControlPort 9111
SafeLogging 0
-Log notice file ./scanner.1/tor.log
+Log notice file ./data/scanner.1/tor.log
FetchUselessDescriptors 1
-DataDirectory ./scanner.1/tor-data
+DataDirectory ./data/scanner.1/tor-data
CookieAuthentication 1
# XXX: This should be for test scans only. Our timeout values must match the
Property changes on: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2
___________________________________________________________________
Added: svn:mergeinfo
+
Deleted: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.2/bwauthority.cfg 2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg 2009-06-03 20:18:39 UTC (rev 19623)
@@ -1,20 +0,0 @@
-[TorCtl]
-loglevel=DEBUG
-tor_host = 127.0.0.1
-tor_port = 9120
-control_host = 127.0.0.1
-control_port = 9121
-control_pass =
-# XXX: Unused
-meta_host = 127.0.0.1
-meta_port = 9122
-
-[BwAuthority]
-out_dir = ./scanner.2/scan-data
-tor_dir = ./scanner.2/tor-data
-start_pct = 50
-stop_pct = 75
-save_every = 10
-nodes_per_slice = 50
-circs_per_node = 2
-max_fetch_time = 1800
Copied: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg (from rev 19617, torflow/trunk/NetworkScanners/BwAuthority/scanner.2/bwauthority.cfg)
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg 2009-06-03 20:18:39 UTC (rev 19623)
@@ -0,0 +1,23 @@
+[TorCtl]
+loglevel=DEBUG
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass =
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.2/scan-data
+tor_dir = ./data/scanner.2/tor-data
+start_pct = 20
+stop_pct = 60
+save_every = 10
+nodes_per_slice = 50
+circs_per_node = 2
+min_streams = 1
+max_fetch_time = 1800
+sleep_start = 01:30
+sleep_stop = 05:30
Modified: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/torrc
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.2/torrc 2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/torrc 2009-06-03 20:18:39 UTC (rev 19623)
@@ -2,9 +2,9 @@
SocksPort 9120 # what port to advertise for application connections
ControlPort 9121
SafeLogging 0
-Log notice file ./scanner.2/tor.log
+Log notice file ./data/scanner.2/tor.log
FetchUselessDescriptors 1
-DataDirectory ./scanner.2/tor-data
+DataDirectory ./data/scanner.2/tor-data
CookieAuthentication 1
# XXX: This should be for test scans only. Our timeout values must match the
Property changes on: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3
___________________________________________________________________
Added: svn:mergeinfo
+
Deleted: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.3/bwauthority.cfg 2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg 2009-06-03 20:18:39 UTC (rev 19623)
@@ -1,20 +0,0 @@
-[TorCtl]
-loglevel=DEBUG
-tor_host = 127.0.0.1
-tor_port = 9130
-control_host = 127.0.0.1
-control_port = 9131
-control_pass =
-# XXX: Unused
-meta_host = 127.0.0.1
-meta_port = 9132
-
-[BwAuthority]
-out_dir = ./scanner.3/scan-data
-tor_dir = ./scanner.3/tor-data
-start_pct =75
-stop_pct = 100
-save_every = 10
-nodes_per_slice = 50
-circs_per_node = 2
-max_fetch_time = 1800
Copied: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg (from rev 19617, torflow/trunk/NetworkScanners/BwAuthority/scanner.3/bwauthority.cfg)
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg 2009-06-03 20:18:39 UTC (rev 19623)
@@ -0,0 +1,23 @@
+[TorCtl]
+loglevel=DEBUG
+tor_host = 127.0.0.1
+tor_port = 9130
+control_host = 127.0.0.1
+control_port = 9131
+control_pass =
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9132
+
+[BwAuthority]
+out_dir = ./data/scanner.3/scan-data
+tor_dir = ./data/scanner.3/tor-data
+start_pct =60
+stop_pct = 100
+save_every = 10
+nodes_per_slice = 50
+circs_per_node = 2
+min_streams = 1
+max_fetch_time = 1800
+sleep_start = 01:30
+sleep_stop = 05:30
Modified: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/torrc
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.3/torrc 2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/torrc 2009-06-03 20:18:39 UTC (rev 19623)
@@ -2,9 +2,9 @@
SocksPort 9130 # what port to advertise for application connections
ControlPort 9131
SafeLogging 0
-Log notice file ./scanner.3/tor.log
+Log notice file ./data/scanner.3/tor.log
FetchUselessDescriptors 1
-DataDirectory ./scanner.3/tor-data
+DataDirectory ./data/scanner.3/tor-data
CookieAuthentication 1
# XXX: This should be for test scans only. Our timeout values must match the
Modified: torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh 2009-06-03 17:40:13 UTC (rev 19622)
+++ torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh 2009-06-03 20:18:39 UTC (rev 19623)
@@ -7,16 +7,16 @@
# git branch --track rs-format-fix mikeperry/rs-format-fix
# git checkout rs-format-fix
TOR_EXE=../../../tor.git/src/or/tor
-PYTHONPATH=../../../SQLAlchemy-0.5.4p2/lib
+#PYTHONPATH=../../../SQLAlchemy-0.5.4p2/lib
-for i in scanner.*
+for i in data/scanner.*
do
rm $i/scan-data/*
done
-$TOR_EXE -f ./scanner.1/torrc &
-$TOR_EXE -f ./scanner.2/torrc &
-$TOR_EXE -f ./scanner.3/torrc &
+$TOR_EXE -f ./data/scanner.1/torrc &
+$TOR_EXE -f ./data/scanner.2/torrc &
+$TOR_EXE -f ./data/scanner.3/torrc &
# If this is a fresh start, we should allow the tors time to download
# new descriptors.
@@ -24,8 +24,8 @@
export PYTHONPATH
-./bwauthority.py ./scanner.1/bwauthority.cfg >& ./scanner.1/bw.log &
-./bwauthority.py ./scanner.2/bwauthority.cfg >& ./scanner.2/bw.log &
-./bwauthority.py ./scanner.3/bwauthority.cfg >& ./scanner.3/bw.log &
+./bwauthority.py ./data/scanner.1/bwauthority.cfg >& ./data/scanner.1/bw.log &
+./bwauthority.py ./data/scanner.2/bwauthority.cfg >& ./data/scanner.2/bw.log &
+./bwauthority.py ./data/scanner.3/bwauthority.cfg >& ./data/scanner.3/bw.log &
More information about the tor-commits
mailing list