[or-cvs] r19589: {torflow} Fix race condition issue with commit() call. Also add additi (in torflow/trunk/NetworkScanners/BwAuthority: . scanner.1 scanner.2 scanner.3)
mikeperry at seul.org
mikeperry at seul.org
Sat May 30 04:40:13 UTC 2009
Author: mikeperry
Date: 2009-05-30 00:40:12 -0400 (Sat, 30 May 2009)
New Revision: 19589
Added:
torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh
torflow/trunk/NetworkScanners/BwAuthority/scanner.2/
torflow/trunk/NetworkScanners/BwAuthority/scanner.2/bwauthority.cfg
torflow/trunk/NetworkScanners/BwAuthority/scanner.2/scan-data/
torflow/trunk/NetworkScanners/BwAuthority/scanner.2/tor-data/
torflow/trunk/NetworkScanners/BwAuthority/scanner.2/torrc
torflow/trunk/NetworkScanners/BwAuthority/scanner.3/
torflow/trunk/NetworkScanners/BwAuthority/scanner.3/bwauthority.cfg
torflow/trunk/NetworkScanners/BwAuthority/scanner.3/scan-data/
torflow/trunk/NetworkScanners/BwAuthority/scanner.3/tor-data/
torflow/trunk/NetworkScanners/BwAuthority/scanner.3/torrc
Modified:
torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py
torflow/trunk/NetworkScanners/BwAuthority/scanner.1/bwauthority.cfg
torflow/trunk/NetworkScanners/BwAuthority/scanner.1/torrc
Log:
Fix race condition issue with commit() call. Also add
additional scan data directories and script for
parallelization.
Modified: torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py 2009-05-30 00:10:28 UTC (rev 19588)
+++ torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py 2009-05-30 04:40:12 UTC (rev 19589)
@@ -37,6 +37,7 @@
# Note these urls should be https due to caching considerations.
# If you really must make them http, be sure to change exit_ports to [80]
# below, or else the scan will not finish.
+# As the network balances, these can become more uniform in size
# cutoff percent URL
urls = [(10, "https://128.174.236.117/4096k"),
(20, "https://128.174.236.117/2048k"),
@@ -72,16 +73,17 @@
save_every = config.getint('BwAuthority', 'save_every')
circs_per_node = config.getint('BwAuthority', 'circs_per_node')
out_dir = config.get('BwAuthority', 'out_dir')
+ tor_dir = config.get('BwAuthority', 'tor_dir')
max_fetch_time = config.getint('BwAuthority', 'max_fetch_time')
return (start_pct,stop_pct,nodes_per_slice,save_every,
- circs_per_node,out_dir,max_fetch_time)
+ circs_per_node,out_dir,max_fetch_time,tor_dir)
def choose_url(percentile):
for (pct, url) in urls:
if percentile < pct:
- #return url
- return "https://86.59.21.36/torbrowser/dist/tor-im-browser-1.2.0_ru_split/tor-im-browser-1.2.0_ru_split.part01.exe"
+ return url
+ #return "https://86.59.21.36/torbrowser/dist/tor-im-browser-1.2.0_ru_split/tor-im-browser-1.2.0_ru_split.part01.exe"
raise PathSupport.NoNodesRemain("No nodes left for url choice!")
# Note: be careful writing functions for this class. Remember that
@@ -165,30 +167,24 @@
cond.release()
def commit(self):
- cond1 = threading.Condition()
- cond2 = threading.Condition()
+ plog("INFO", "Scanner committing jobs...")
+ cond = threading.Condition()
def notlambda2(this):
+ cond.acquire()
plog("INFO", "Commit done.")
- cond2.notify()
- cond2.release()
+ cond.notify()
+ cond.release()
def notlambda1(this):
- cond1.acquire()
- cond2.acquire()
plog("INFO", "Committing jobs...")
this.run_all_jobs = True
self.schedule_low_prio(notlambda2)
- cond1.notify()
- cond1.release()
- cond1.acquire()
- cond2.acquire()
+ cond.acquire()
self.schedule_immediate(notlambda1)
- cond1.wait()
- cond1.release()
- cond2.wait()
- cond2.release()
+ cond.wait()
+ cond.release()
plog("INFO", "Scanner commit done.")
def close_circuits(self):
@@ -284,22 +280,22 @@
def wait_for_consensus(self):
cond = threading.Condition()
- # XXX: We want to set a flag that says "Don't compute consensus db
- # until we get called again plz". Locks and semaphores are off limits
- # because this is the same thread+codepath
def notlambda(this):
if this.sql_consensus_listener.last_desc_at \
!= SQLSupport.ConsensusTrackerListener.CONSENSUS_DONE:
+ this.sql_consensus_listener.wait_for_signal = False
plog("INFO", "Waiting on consensus result: "+str(this.run_all_jobs))
this.schedule_low_prio(notlambda)
else:
cond.acquire()
+ this.sql_consensus_listener.wait_for_signal = True
cond.notify()
cond.release()
cond.acquire()
self.schedule_low_prio(notlambda)
cond.wait()
cond.release()
+ plog("INFO", "Consensus OK")
def http_request(address):
''' perform an http GET-request and return 1 for success or 0 for failure '''
@@ -380,10 +376,10 @@
def main(argv):
TorUtil.read_config(argv[1])
(start_pct,stop_pct,nodes_per_slice,save_every,
- circs_per_node,out_dir,max_fetch_time) = read_config(argv[1])
+ circs_per_node,out_dir,max_fetch_time,tor_dir) = read_config(argv[1])
try:
- (c,hdlr) = setup_handler()
+ (c,hdlr) = setup_handler(tor_dir+"/control_auth_cookie")
except Exception, e:
plog("WARN", "Can't connect to Tor: "+str(e))
@@ -418,7 +414,7 @@
hdlr.write_strm_bws(pct, pct+pct_step, os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"))
plog('DEBUG', 'Wrote stats')
pct += pct_step
- hdlr.save_sql_file(sql_file, "db-"+str(lo)+":"+str(hi)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S")+".sqlite")
+ hdlr.save_sql_file(sql_file, os.getcwd()+"/"+out_dir+"/db-"+str(lo)+":"+str(hi)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S")+".sqlite")
def cleanup(c, f):
plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors="+f)
@@ -428,13 +424,13 @@
except TorCtl.TorCtlClosed:
pass
-def setup_handler():
+def setup_handler(cookie_file):
plog('INFO', 'Connecting to Tor...')
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((control_host,control_port))
c = PathSupport.Connection(s)
#c.debug(file("control.log", "w", buffering=0))
- c.authenticate(control_pass)
+ c.authenticate_cookie(file(cookie_file, "r"))
h = BwScanHandler(c, __selmgr)
c.set_event_handler(h)
Added: torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh 2009-05-30 04:40:12 UTC (rev 19589)
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# This tor must have the w status line fix as well as the stream bw fix
+# Ie:
+# git remote add mikeperry git://git.torproject.org/~mikeperry/git/tor
+# git fetch mikeperry
+# git branch --track control-ns-wfix mikeperry/control-ns-wfix
+# git checkout control-ns-wfix
+TOR_EXE=../../../tor.git/src/or/tor
+
+$TOR_EXE -f ./scanner.1/torrc &
+$TOR_EXE -f ./scanner.2/torrc &
+$TOR_EXE -f ./scanner.3/torrc &
+
+./bwautority.py ./scanner.1/bwauthority.cfg >& ./scanner.1/bw.log &
+./bwautority.py ./scanner.2/bwauthority.cfg >& ./scanner.2/bw.log &
+./bwautority.py ./scanner.3/bwauthority.cfg >& ./scanner.3/bw.log &
+
+
Property changes on: torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: torflow/trunk/NetworkScanners/BwAuthority/scanner.1/bwauthority.cfg
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.1/bwauthority.cfg 2009-05-30 00:10:28 UTC (rev 19588)
+++ torflow/trunk/NetworkScanners/BwAuthority/scanner.1/bwauthority.cfg 2009-05-30 04:40:12 UTC (rev 19589)
@@ -1,18 +1,19 @@
[TorCtl]
loglevel=DEBUG
tor_host = 127.0.0.1
-tor_port = 9060
+tor_port = 9110
control_host = 127.0.0.1
-control_port = 9061
+control_port = 9111
control_pass =
# XXX: Unused
meta_host = 127.0.0.1
-meta_port = 9062
+meta_port = 9112
[BwAuthority]
out_dir = ./scanner.1/scan-data
+tor_dir = ./scanner.1/tor-data
start_pct = 0
-stop_pct = 15
+stop_pct = 50
save_every = 10
nodes_per_slice = 50
circs_per_node = 2
Modified: torflow/trunk/NetworkScanners/BwAuthority/scanner.1/torrc
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.1/torrc 2009-05-30 00:10:28 UTC (rev 19588)
+++ torflow/trunk/NetworkScanners/BwAuthority/scanner.1/torrc 2009-05-30 04:40:12 UTC (rev 19589)
@@ -1,10 +1,11 @@
## Configuration file for metatroller on alternate prot
-SocksPort 9060 # what port to advertise for application connections
-ControlPort 9061
+SocksPort 9110 # what port to advertise for application connections
+ControlPort 9111
SafeLogging 0
-Log notice stderr
+Log notice file ./scanner.1/tor.log
FetchUselessDescriptors 1
DataDirectory ./scanner.1/tor-data
+CookieAuthentication 1
# XXX: This should be for test scans only. Our timeout values must match the
# normal Tor clients.
Added: torflow/trunk/NetworkScanners/BwAuthority/scanner.2/bwauthority.cfg
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.2/bwauthority.cfg (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/scanner.2/bwauthority.cfg 2009-05-30 04:40:12 UTC (rev 19589)
@@ -0,0 +1,20 @@
+[TorCtl]
+loglevel=DEBUG
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass =
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./scanner.2/scan-data
+tor_dir = ./scanner.2/tor-data
+start_pct = 50
+stop_pct = 75
+save_every = 10
+nodes_per_slice = 50
+circs_per_node = 2
+max_fetch_time = 1800
Added: torflow/trunk/NetworkScanners/BwAuthority/scanner.2/torrc
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.2/torrc (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/scanner.2/torrc 2009-05-30 04:40:12 UTC (rev 19589)
@@ -0,0 +1,12 @@
+## Configuration file for metatroller on alternate prot
+SocksPort 9120 # what port to advertise for application connections
+ControlPort 9121
+SafeLogging 0
+Log notice file ./scanner.2/tor.log
+FetchUselessDescriptors 1
+DataDirectory ./scanner.2/tor-data
+CookieAuthentication 1
+
+# XXX: This should be for test scans only. Our timeout values must match the
+# normal Tor clients.
+SocksTimeout 45
Added: torflow/trunk/NetworkScanners/BwAuthority/scanner.3/bwauthority.cfg
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.3/bwauthority.cfg (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/scanner.3/bwauthority.cfg 2009-05-30 04:40:12 UTC (rev 19589)
@@ -0,0 +1,20 @@
+[TorCtl]
+loglevel=DEBUG
+tor_host = 127.0.0.1
+tor_port = 9130
+control_host = 127.0.0.1
+control_port = 9131
+control_pass =
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9132
+
+[BwAuthority]
+out_dir = ./scanner.3/scan-data
+tor_dir = ./scanner.3/tor-data
+start_pct =75
+stop_pct = 100
+save_every = 10
+nodes_per_slice = 50
+circs_per_node = 2
+max_fetch_time = 1800
Added: torflow/trunk/NetworkScanners/BwAuthority/scanner.3/torrc
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.3/torrc (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/scanner.3/torrc 2009-05-30 04:40:12 UTC (rev 19589)
@@ -0,0 +1,12 @@
+## Configuration file for metatroller on alternate prot
+SocksPort 9130 # what port to advertise for application connections
+ControlPort 9131
+SafeLogging 0
+Log notice file ./scanner.3/tor.log
+FetchUselessDescriptors 1
+DataDirectory ./scanner.3/tor-data
+CookieAuthentication 1
+
+# XXX: This should be for test scans only. Our timeout values must match the
+# normal Tor clients.
+SocksTimeout 45
More information about the tor-commits
mailing list