[tor-commits] [torflow/master] Add a 5th scanner for Unmeasured=1 nodes.
mikeperry at torproject.org
mikeperry at torproject.org
Mon Jun 1 05:37:43 UTC 2015
commit 4f272b1bf606df6e9788ad5bc60dbe3f9027eb23
Author: Mike Perry <mikeperry-git at torproject.org>
Date: Thu May 21 18:18:55 2015 -0700
Add a 5th scanner for Unmeasured=1 nodes.
---
NetworkScanners/BwAuthority/bwauthority_child.py | 36 ++++++++++++++++----
.../BwAuthority/data/scanner.1/bwauthority.cfg | 2 ++
.../BwAuthority/data/scanner.2/bwauthority.cfg | 2 ++
.../BwAuthority/data/scanner.3/bwauthority.cfg | 2 ++
.../BwAuthority/data/scanner.4/bwauthority.cfg | 2 ++
.../BwAuthority/data/scanner.5/bwauthority.cfg | 30 ++++++++++++++++
NetworkScanners/BwAuthority/run_scan.sh | 2 +-
7 files changed, 68 insertions(+), 8 deletions(-)
diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py
index 2584991..4bac13a 100755
--- a/NetworkScanners/BwAuthority/bwauthority_child.py
+++ b/NetworkScanners/BwAuthority/bwauthority_child.py
@@ -95,9 +95,13 @@ def read_config(filename):
pid_file = config.get('BwAuthority', 'pid_file')
db_url = config.get('BwAuthority', 'db_url')
+ only_unmeasured = config.getint('BwAuthority', 'only_unmeasured')
+ min_unmeasured = config.getint('BwAuthority', 'min_unmeasured')
+
return (start_pct,stop_pct,nodes_per_slice,save_every,
circs_per_node,out_dir,max_fetch_time,tor_dir,
- sleep_start,sleep_stop,min_streams,pid_file,db_url)
+ sleep_start,sleep_stop,min_streams,pid_file,db_url,only_unmeasured,
+ min_unmeasured)
def choose_url(percentile):
# TODO: Maybe we don't want to read the file *every* time?
@@ -205,7 +209,7 @@ class BwScanHandler(ScanSupport.SQLScanHandler):
def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir,
max_fetch_time, sleep_start_tp, sleep_stop_tp, slice_num,
- min_streams, sql_file):
+ min_streams, sql_file, only_unmeasured):
hdlr.set_pct_rstr(start_pct, stop_pct)
attempt = 0
@@ -214,8 +218,6 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir,
if hdlr.is_count_met(circs_per_node, successful): break
hdlr.wait_for_consensus()
-
-
# Check local time. Do not scan between 01:30 and 05:30 local time
lt = time.localtime()
sleep_start = time.mktime(lt[0:3]+sleep_start_tp+(0,0,0)+(lt[-1],))
@@ -239,7 +241,14 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir,
# stream.. however, we count it as 'successful' below
timer = threading.Timer(max_fetch_time, lambda: hdlr.close_streams(7))
timer.start()
- url = choose_url(start_pct)
+
+ # Always use median URL size for unmeasured nodes
+ # They may be too slow..
+ if only_unmeasured:
+ url = choose_url(50)
+ else:
+ url = choose_url(start_pct)
+
plog("DEBUG", "Launching stream request for url "+url+" in "+str(start_pct)+'-'+str(stop_pct) + '%')
ret = http_request(url)
timer.cancel()
@@ -297,7 +306,8 @@ def main(argv):
TorUtil.read_config(argv[1])
(start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir,
max_fetch_time,tor_dir,sleep_start,sleep_stop,
- min_streams,pid_file_name,db_url) = read_config(argv[1])
+ min_streams,pid_file_name,db_url,only_unmeasured,
+ min_unmeasured) = read_config(argv[1])
# make sure necessary out_dir directory exists
path = os.getcwd()+'/'+out_dir
@@ -332,9 +342,21 @@ def main(argv):
socket.socket = socks.socksocket
plog("INFO", "Set socks proxy to "+TorUtil.tor_host+":"+str(TorUtil.tor_port))
+ hdlr.schedule_selmgr(lambda s: setattr(s, "only_unmeasured", only_unmeasured))
+
hdlr.wait_for_consensus()
+ # We should go to sleep if there are less than 5 unmeasured nodes after
+ # consensus update
+ while min_unmeasured and hdlr.get_unmeasured() < min_unmeasured:
+ plog("NOTICE", "Less than "+str(min_unmeasured)+" unmeasured nodes ("+str(hdlr.get_unmeasured())+"). Sleeping for a bit")
+ time.sleep(3600) # Until next consensus arrives
+ plog("NOTICE", "Woke up from waiting for more unmeasured nodes. Getting consensus and checking again")
+ hdlr.wait_for_consensus()
+
pct_step = hdlr.rank_to_percent(nodes_per_slice)
+ plog("INFO", "Percent per slice is: "+str(pct_step))
+ if pct_step > 100: pct_step = 100
# check to see if we are done
if (slice_num * pct_step + start_pct > stop_pct):
@@ -344,7 +366,7 @@ def main(argv):
plog("DEBUG", "Starting slice number %s" % slice_num)
successful = speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node,
save_every, out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num,
- min_streams, sql_file)
+ min_streams, sql_file, only_unmeasured)
# For debugging memory leak..
#TorUtil.dump_class_ref_counts(referrer_depth=1)
diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
index 4cced6b..7f81a0b 100644
--- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
@@ -26,3 +26,5 @@ min_streams = 1
max_fetch_time = 600
sleep_start = 01:30
sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0
diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
index 53dc469..28cfaa6 100644
--- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
@@ -26,3 +26,5 @@ min_streams = 1
max_fetch_time = 300
sleep_start = 01:30
sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0
diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
index 6df68e1..7b71aa8 100644
--- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
@@ -26,3 +26,5 @@ min_streams = 1
max_fetch_time = 300
sleep_start = 01:30
sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0
diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg
index ff19b87..0dfea36 100644
--- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg
@@ -26,3 +26,5 @@ min_streams = 1
max_fetch_time = 300
sleep_start = 01:30
sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0
diff --git a/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg
new file mode 100644
index 0000000..468a62e
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg
@@ -0,0 +1,30 @@
+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9110
+control_host = 127.0.0.1
+control_port = 9111
+control_pass =
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9112
+
+[BwAuthority]
+out_dir = ./data/scanner.5/scan-data
+pid_file = ./data/scanner.5/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4
+tor_dir = ./data/tor
+start_pct = 0
+stop_pct = 100
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 360
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 1
+min_unmeasured = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.5/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.5/scan-data/.gitignore
new file mode 100644
index 0000000..e69de29
diff --git a/NetworkScanners/BwAuthority/run_scan.sh b/NetworkScanners/BwAuthority/run_scan.sh
index c21e871..7853647 100755
--- a/NetworkScanners/BwAuthority/run_scan.sh
+++ b/NetworkScanners/BwAuthority/run_scan.sh
@@ -1,7 +1,7 @@
#!/bin/sh
# Number of scanners to run.
-SCANNER_COUNT=4
+SCANNER_COUNT=5
# This tor must have the w status line fix as well as the stream bw fix
# Ie git master or 0.2.2.x
More information about the tor-commits
mailing list