commit 94dff967fd457c4922fe7838f5f5ef4c7b17eefe Author: Tom Ritter tom@ritter.vg Date: Thu May 21 13:01:07 2015 -0500

Double the scanners, double the tors

Also update aggregate.py to work with a two-tor system --- NetworkScanners/BwAuthority/aggregate.py | 6 ++-- .../BwAuthority/data/scanner.1/bwauthority.cfg | 4 +-- .../BwAuthority/data/scanner.2/bwauthority.cfg | 6 ++-- .../BwAuthority/data/scanner.3/bwauthority.cfg | 6 ++-- .../BwAuthority/data/scanner.4/bwauthority.cfg | 6 ++-- .../BwAuthority/data/scanner.5/bwauthority.cfg | 25 ++++++++-------- .../BwAuthority/data/scanner.6/bwauthority.cfg | 30 ++++++++++++++++++++ .../BwAuthority/data/scanner.7/bwauthority.cfg | 30 ++++++++++++++++++++ .../BwAuthority/data/scanner.8/bwauthority.cfg | 30 ++++++++++++++++++++ .../BwAuthority/data/scanner.9/bwauthority.cfg | 30 ++++++++++++++++++++ .../data/scanner.9/scan-data/.gitignore | 2 ++ NetworkScanners/BwAuthority/data/tor.1/torrc | 18 ++++++++++++ NetworkScanners/BwAuthority/data/tor.2/torrc | 18 ++++++++++++ NetworkScanners/BwAuthority/data/tor/torrc | 18 ------------ NetworkScanners/BwAuthority/run_scan.sh | 29 +++++++++++-------- 15 files changed, 203 insertions(+), 55 deletions(-)

diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 49ad3cc..6d35dba 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -362,7 +362,8 @@ def write_file_list(datadir):

def main(argv): TorUtil.read_config(argv[1]+"/scanner.1/bwauthority.cfg") - TorUtil.loglevel = "NOTICE" + TorUtil.loglevel = "DEBUG" + TorUtil.logfile = "data/aggregate-debug.log"

(branch, head) = TorUtil.get_git_version(PATH_TO_TORFLOW_REPO) plog('INFO', 'TorFlow Version: %s' % branch+' '+head) @@ -373,7 +374,7 @@ def main(argv): s.connect((TorUtil.control_host,TorUtil.control_port)) c = TorCtl.Connection(s) c.debug(file(argv[1]+"/aggregate-control.log", "w", buffering=0)) - c.authenticate_cookie(file(argv[1]+"/tor/control_auth_cookie", + c.authenticate_cookie(file(argv[1]+"/tor.1/control_auth_cookie", "r"))

ns_list = c.get_network_status() @@ -847,6 +848,7 @@ def main(argv): plog("INFO", "Avg network +pid_error=+"+str(sum(map(lambda n: n.pid_error, p_nodes))/len(p_nodes))) plog("INFO", "Avg network -pid_error="+str(sum(map(lambda n: n.pid_error, n_nodes))/len(n_nodes)))

+ plog("INFO", "Measured "+str(measured_pct) +"% of all tor nodes (" +str(measured_bw_pct)+"% of previous consensus bw).") diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg index 7f81a0b..ad9a4cd 100644 --- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg @@ -16,9 +16,9 @@ pid_file = ./data/scanner.1/bwauthority.pid db_url = #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 -tor_dir = ./data/tor +tor_dir = ./data/tor.1 start_pct = 0 -stop_pct = 10 +stop_pct = 12 save_every = 0 nodes_per_slice = 50 circs_per_node = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg index 28cfaa6..3e94e5e 100644 --- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg @@ -16,9 +16,9 @@ pid_file = ./data/scanner.2/bwauthority.pid db_url = #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan2 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan2 -tor_dir = ./data/tor -start_pct = 10 -stop_pct = 40 +tor_dir = ./data/tor.1 +start_pct = 12 +stop_pct = 25 save_every = 0 nodes_per_slice = 50 circs_per_node = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg index 7b71aa8..71c3285 100644 --- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg @@ -16,9 +16,9 @@ pid_file = ./data/scanner.3/bwauthority.pid db_url = #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan3 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan3 -tor_dir = ./data/tor -start_pct = 30 -stop_pct = 60 +tor_dir = ./data/tor.1 +start_pct = 25 +stop_pct = 37 save_every = 0 nodes_per_slice = 50 circs_per_node = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg index 0dfea36..9a676ec 100644 --- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg @@ -16,9 +16,9 @@ pid_file = ./data/scanner.4/bwauthority.pid db_url = #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4 -tor_dir = ./data/tor -start_pct = 60 -stop_pct = 100 +tor_dir = ./data/tor.1 +start_pct = 37 +stop_pct = 50 save_every = 0 nodes_per_slice = 50 circs_per_node = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg index 468a62e..1acb1a2 100644 --- a/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg @@ -1,30 +1,31 @@ [TorCtl] loglevel=WARN tor_host = 127.0.0.1 -tor_port = 9110 +tor_port = 9120 control_host = 127.0.0.1 -control_port = 9111 +control_port = 9121 control_pass = # XXX: Unused meta_host = 127.0.0.1 -meta_port = 9112 +meta_port = 9122

[BwAuthority] out_dir = ./data/scanner.5/scan-data pid_file = ./data/scanner.5/bwauthority.pid # if db_url is unset bwauthority will default to sqlite -db_url = -#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4 -#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4 -tor_dir = ./data/tor -start_pct = 0 -stop_pct = 100 +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan5 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan5 +tor_dir = ./data/tor.2 +start_pct = 50 +stop_pct = 62 save_every = 0 nodes_per_slice = 50 circs_per_node = 5 min_streams = 1 -max_fetch_time = 360 +max_fetch_time = 300 sleep_start = 01:30 sleep_stop = 01:30 -only_unmeasured = 1 -min_unmeasured = 5 +only_unmeasured = 0 +min_unmeasured = 0 + diff --git a/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg new file mode 100755 index 0000000..5bc3616 --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg @@ -0,0 +1,30 @@ +[TorCtl] +loglevel=WARN +tor_host = 127.0.0.1 +tor_port = 9120 +control_host = 127.0.0.1 +control_port = 9121 +control_pass = +# XXX: Unused +meta_host = 127.0.0.1 +meta_port = 9122 + +[BwAuthority] +out_dir = ./data/scanner.6/scan-data +pid_file = ./data/scanner.6/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan6 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan6 +tor_dir = ./data/tor.2 +start_pct = 62 +stop_pct = 75 +save_every = 0 +nodes_per_slice = 50 +circs_per_node = 5 +min_streams = 1 +max_fetch_time = 300 +sleep_start = 01:30 +sleep_stop = 01:30 +only_unmeasured = 0 +min_unmeasured = 0 diff --git a/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg new file mode 100755 index 0000000..2898b80 --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg @@ -0,0 +1,30 @@ +[TorCtl] +loglevel=WARN +tor_host = 127.0.0.1 +tor_port = 9120 +control_host = 127.0.0.1 +control_port = 9121 +control_pass = +# XXX: Unused +meta_host = 127.0.0.1 +meta_port = 9122 + +[BwAuthority] +out_dir = ./data/scanner.7/scan-data +pid_file = ./data/scanner.7/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan7 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan7 +tor_dir = ./data/tor.2 +start_pct = 75 +stop_pct = 87 +save_every = 0 +nodes_per_slice = 50 +circs_per_node = 5 +min_streams = 1 +max_fetch_time = 300 +sleep_start = 01:30 +sleep_stop = 01:30 +only_unmeasured = 0 +min_unmeasured = 0 diff --git a/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg new file mode 100755 index 0000000..56a5cfa --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg @@ -0,0 +1,30 @@ +[TorCtl] +loglevel=WARN +tor_host = 127.0.0.1 +tor_port = 9120 +control_host = 127.0.0.1 +control_port = 9121 +control_pass = +# XXX: Unused +meta_host = 127.0.0.1 +meta_port = 9122 + +[BwAuthority] +out_dir = ./data/scanner.8/scan-data +pid_file = ./data/scanner.8/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan8 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan8 +tor_dir = ./data/tor.2 +start_pct = 87 +stop_pct = 100 +save_every = 0 +nodes_per_slice = 50 +circs_per_node = 5 +min_streams = 1 +max_fetch_time = 300 +sleep_start = 01:30 +sleep_stop = 01:30 +only_unmeasured = 0 +min_unmeasured = 0 diff --git a/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg new file mode 100644 index 0000000..7e49dbb --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg @@ -0,0 +1,30 @@ +[TorCtl] +loglevel=WARN +tor_host = 127.0.0.1 +tor_port = 9120 +control_host = 127.0.0.1 +control_port = 9121 +control_pass = +# XXX: Unused +meta_host = 127.0.0.1 +meta_port = 9122 + +[BwAuthority] +out_dir = ./data/scanner.9/scan-data +pid_file = ./data/scanner.9/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan9 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan9 +tor_dir = ./data/tor.2 +start_pct = 0 +stop_pct = 100 +save_every = 0 +nodes_per_slice = 50 +circs_per_node = 5 +min_streams = 1 +max_fetch_time = 360 +sleep_start = 01:30 +sleep_stop = 01:30 +only_unmeasured = 1 +min_unmeasured = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore new file mode 100644 index 0000000..20f86df --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore @@ -0,0 +1,2 @@ +bws-* +sql-* \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/data/tor.1/torrc b/NetworkScanners/BwAuthority/data/tor.1/torrc new file mode 100644 index 0000000..933141e --- /dev/null +++ b/NetworkScanners/BwAuthority/data/tor.1/torrc @@ -0,0 +1,18 @@ +## Configuration file for metatroller on alternate prot +SocksPort 9110 # what port to advertise for application connections +ControlPort 9111 +SafeLogging 0 +Log notice file ./data/tor.1/tor.log +DataDirectory ./data/tor.1 +PidFile ./data/tor.1/tor.pid +CookieAuthentication 1 +FetchUselessDescriptors 1 +__LeaveStreamsUnattached 1 + +# Bad idea? Too much consensus update activity? +FetchDirInfoExtraEarly 1 +FetchDirInfoEarly 1 + +# Use CREATE cells for first hop, so we can still measure CPU overload via +# circ failure +FastFirstHopPK 0 diff --git a/NetworkScanners/BwAuthority/data/tor.2/torrc b/NetworkScanners/BwAuthority/data/tor.2/torrc new file mode 100755 index 0000000..209781f --- /dev/null +++ b/NetworkScanners/BwAuthority/data/tor.2/torrc @@ -0,0 +1,18 @@ +## Configuration file for metatroller on alternate prot +SocksPort 9120 # what port to advertise for application connections +ControlPort 9121 +SafeLogging 0 +Log notice file ./data/tor.2/tor.log +DataDirectory ./data/tor.2 +PidFile ./data/tor.2/tor.pid +CookieAuthentication 1 +FetchUselessDescriptors 1 +__LeaveStreamsUnattached 1 + +# Bad idea? Too much consensus update activity? +FetchDirInfoExtraEarly 1 +FetchDirInfoEarly 1 + +# Use CREATE cells for first hop, so we can still measure CPU overload via +# circ failure +FastFirstHopPK 0 diff --git a/NetworkScanners/BwAuthority/data/tor/torrc b/NetworkScanners/BwAuthority/data/tor/torrc deleted file mode 100644 index c64273b..0000000 --- a/NetworkScanners/BwAuthority/data/tor/torrc +++ /dev/null @@ -1,18 +0,0 @@ -## Configuration file for metatroller on alternate prot -SocksPort 9110 # what port to advertise for application connections -ControlPort 9111 -SafeLogging 0 -Log notice file ./data/tor/tor.log -DataDirectory ./data/tor -PidFile ./data/tor/tor.pid -CookieAuthentication 1 -FetchUselessDescriptors 1 -__LeaveStreamsUnattached 1 - -# Bad idea? Too much consensus update activity? -FetchDirInfoExtraEarly 1 -FetchDirInfoEarly 1 - -# Use CREATE cells for first hop, so we can still measure CPU overload via -# circ failure -FastFirstHopPK 0 diff --git a/NetworkScanners/BwAuthority/run_scan.sh b/NetworkScanners/BwAuthority/run_scan.sh index 8a71b1d..908b76e 100755 --- a/NetworkScanners/BwAuthority/run_scan.sh +++ b/NetworkScanners/BwAuthority/run_scan.sh @@ -1,7 +1,9 @@ #!/bin/sh

-# Number of scanners to run. -SCANNER_COUNT=5 +# Number of applications to run. +SCANNERS_PER_TOR_COUNT=4 +TOR_COUNT=2 +SCANNER_COUNT=$(($SCANNERS_PER_TOR_COUNT * $TOR_COUNT + 1))

# This tor must have the w status line fix as well as the stream bw fix # Ie git master or 0.2.2.x @@ -19,15 +21,17 @@ for n in `seq $SCANNER_COUNT`; do done

KILLED_TOR=false -if [ -f "./data/tor/tor.pid" ]; then - PID=`cat ./data/tor/tor.pid` - if kill -0 "$PID" 2>/dev/null; then # it is a running process and we may send signals to it - kill $PID - if [ $? -eq 0 ]; then - KILLED_TOR=true +for n in `seq $TOR_COUNT`; do + PIDFILE=./data/tor.${n}/tor.pid + if [ -f $PIDFILE ]; then + if kill -0 `head -1 $PIDFILE` 2>/dev/null; then # it is a running process and we may send signals to it + kill `head -1 $PIDFILE` + if [ $? -eq 0 ]; then + KILLED_TOR=true + fi fi fi -fi +done

sleep 5

@@ -35,9 +39,10 @@ sleep 5 # A more accurate resume could be implemented in bwauthority.py find data/scanner.* -name .svn -prune -o -type f -a ! -name '*-done-*' -a ! -name bwauthority.cfg -a ! -name .gitignore -exec rm {} +

-rm -f ./data/tor/tor.log - -$TOR_EXE -f ./data/tor/torrc & +for n in `seq $TOR_COUNT`; do + rm -f ./data/tor.${n}/tor.log + $TOR_EXE -f ./data/tor.${n}/torrc & +done

# If this is a fresh start, we should allow the tors time to download # new descriptors.