tor-commits June 2015

tor-commits@lists.torproject.org

22 participants
1870 discussions

[tor-messenger-build/master] Update xmpp-onion-js.patch
by sukhbir＠torproject.org 01 Jun '15

01 Jun '15

commit 4de2c48c5c0439089120647df0456e999a550cd8 Author: Sukhbir Singh <sukhbir(a)torproject.org> Date: Mon Jun 1 08:30:48 2015 -0400 Update xmpp-onion-js.patch --- projects/instantbird/xmpp-onion-js.patch | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/projects/instantbird/xmpp-onion-js.patch b/projects/instantbird/xmpp-onion-js.patch index d04c78b..e6365fc 100644 --- a/projects/instantbird/xmpp-onion-js.patch +++ b/projects/instantbird/xmpp-onion-js.patch @@ -1,8 +1,7 @@ diff --git a/im/content/accountWizard.js b/im/content/accountWizard.js -index 707f1c7..f27ae92 100644 --- a/im/content/accountWizard.js +++ b/im/content/accountWizard.js -@@ -7,6 +7,14 @@ Cu.import("resource:///modules/imServices.jsm"); +@@ -7,6 +7,14 @@ const PREF_EXTENSIONS_GETMOREPROTOCOLSURL = "extensions.getMoreProtocolsURL"; @@ -17,7 +16,7 @@ index 707f1c7..f27ae92 100644 var accountWizard = { onload: function aw_onload() { let topProtoList = document.getElementById("topprotolist"); -@@ -105,6 +113,21 @@ var accountWizard = { +@@ -105,6 +113,21 @@ return textbox; }, @@ -39,12 +38,12 @@ index 707f1c7..f27ae92 100644 showUsernamePage: function aw_showUsernamePage() { let proto = this.proto.id; if ("userNameBoxes" in this && this.userNameProto == proto) { -@@ -160,11 +183,20 @@ var accountWizard = { +@@ -160,11 +183,20 @@ document.getElementById("accountusername").next = next; }, + setOnionAddress: function aw_setOnionAddress() { -+ if (this.proto.id == "prpl-jabber") { ++ if (this.proto.id == "prpl-jabber" && this.onionAddress) { + document.getElementById("prpl-jabber-server").value = this.onionAddress; + } + }, @@ -61,7 +60,7 @@ index 707f1c7..f27ae92 100644 this.protoSpecOptId = id; /* FIXME -@@ -172,6 +204,7 @@ var accountWizard = { +@@ -172,6 +204,7 @@ !this.proto.newMailNotification; */ this.populateProtoSpecificBox(); @@ -69,12 +68,13 @@ index 707f1c7..f27ae92 100644 let proxyVisible = this.proto.usePurpleProxy; if (proxyVisible) { -@@ -405,7 +438,16 @@ var accountWizard = { +@@ -405,7 +438,17 @@ for (let i = 0; i < this.prefs.length; ++i) { let opt = this.prefs[i]; let label = bundle.getFormattedString("accountColon", [opt.opt.label]); - rows.appendChild(this.createSummaryRow(label, opt.value)); -+ if (this.onionAddress && label == "Server:") { ++ // Only append the label for the "Server" field. ++ if (this.onionAddress == opt.value) { + let wizardBundle = document.getElementById("topProtocolsBundle"); + let onionLabel = wizardBundle.getFormattedString("onionAddress.label", + [this.jabberDomain.value]);

1 0

[torflow/master] Double the scanners, double the tors
by mikeperry＠torproject.org 01 Jun '15

01 Jun '15

commit 94dff967fd457c4922fe7838f5f5ef4c7b17eefe Author: Tom Ritter <tom(a)ritter.vg> Date: Thu May 21 13:01:07 2015 -0500 Double the scanners, double the tors Also update aggregate.py to work with a two-tor system --- NetworkScanners/BwAuthority/aggregate.py | 6 ++-- .../BwAuthority/data/scanner.1/bwauthority.cfg | 4 +-- .../BwAuthority/data/scanner.2/bwauthority.cfg | 6 ++-- .../BwAuthority/data/scanner.3/bwauthority.cfg | 6 ++-- .../BwAuthority/data/scanner.4/bwauthority.cfg | 6 ++-- .../BwAuthority/data/scanner.5/bwauthority.cfg | 25 ++++++++-------- .../BwAuthority/data/scanner.6/bwauthority.cfg | 30 ++++++++++++++++++++ .../BwAuthority/data/scanner.7/bwauthority.cfg | 30 ++++++++++++++++++++ .../BwAuthority/data/scanner.8/bwauthority.cfg | 30 ++++++++++++++++++++ .../BwAuthority/data/scanner.9/bwauthority.cfg | 30 ++++++++++++++++++++ .../data/scanner.9/scan-data/.gitignore | 2 ++ NetworkScanners/BwAuthority/data/tor.1/torrc | 18 ++++++++++++ NetworkScanners/BwAuthority/data/tor.2/torrc | 18 ++++++++++++ NetworkScanners/BwAuthority/data/tor/torrc | 18 ------------ NetworkScanners/BwAuthority/run_scan.sh | 29 +++++++++++-------- 15 files changed, 203 insertions(+), 55 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 49ad3cc..6d35dba 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -362,7 +362,8 @@ def write_file_list(datadir): def main(argv): TorUtil.read_config(argv[1]+"/scanner.1/bwauthority.cfg") - TorUtil.loglevel = "NOTICE" + TorUtil.loglevel = "DEBUG" + TorUtil.logfile = "data/aggregate-debug.log" (branch, head) = TorUtil.get_git_version(PATH_TO_TORFLOW_REPO) plog('INFO', 'TorFlow Version: %s' % branch+' '+head) @@ -373,7 +374,7 @@ def main(argv): s.connect((TorUtil.control_host,TorUtil.control_port)) c = TorCtl.Connection(s) c.debug(file(argv[1]+"/aggregate-control.log", "w", buffering=0)) - c.authenticate_cookie(file(argv[1]+"/tor/control_auth_cookie", + c.authenticate_cookie(file(argv[1]+"/tor.1/control_auth_cookie", "r")) ns_list = c.get_network_status() @@ -847,6 +848,7 @@ def main(argv): plog("INFO", "Avg network +pid_error=+"+str(sum(map(lambda n: n.pid_error, p_nodes))/len(p_nodes))) plog("INFO", "Avg network -pid_error="+str(sum(map(lambda n: n.pid_error, n_nodes))/len(n_nodes))) + plog("INFO", "Measured "+str(measured_pct) +"% of all tor nodes (" +str(measured_bw_pct)+"% of previous consensus bw).") diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg index 7f81a0b..ad9a4cd 100644 --- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg @@ -16,9 +16,9 @@ pid_file = ./data/scanner.1/bwauthority.pid db_url = #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 -tor_dir = ./data/tor +tor_dir = ./data/tor.1 start_pct = 0 -stop_pct = 10 +stop_pct = 12 save_every = 0 nodes_per_slice = 50 circs_per_node = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg index 28cfaa6..3e94e5e 100644 --- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg @@ -16,9 +16,9 @@ pid_file = ./data/scanner.2/bwauthority.pid db_url = #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan2 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan2 -tor_dir = ./data/tor -start_pct = 10 -stop_pct = 40 +tor_dir = ./data/tor.1 +start_pct = 12 +stop_pct = 25 save_every = 0 nodes_per_slice = 50 circs_per_node = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg index 7b71aa8..71c3285 100644 --- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg @@ -16,9 +16,9 @@ pid_file = ./data/scanner.3/bwauthority.pid db_url = #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan3 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan3 -tor_dir = ./data/tor -start_pct = 30 -stop_pct = 60 +tor_dir = ./data/tor.1 +start_pct = 25 +stop_pct = 37 save_every = 0 nodes_per_slice = 50 circs_per_node = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg index 0dfea36..9a676ec 100644 --- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg @@ -16,9 +16,9 @@ pid_file = ./data/scanner.4/bwauthority.pid db_url = #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4 -tor_dir = ./data/tor -start_pct = 60 -stop_pct = 100 +tor_dir = ./data/tor.1 +start_pct = 37 +stop_pct = 50 save_every = 0 nodes_per_slice = 50 circs_per_node = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg index 468a62e..1acb1a2 100644 --- a/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg @@ -1,30 +1,31 @@ [TorCtl] loglevel=WARN tor_host = 127.0.0.1 -tor_port = 9110 +tor_port = 9120 control_host = 127.0.0.1 -control_port = 9111 +control_port = 9121 control_pass = # XXX: Unused meta_host = 127.0.0.1 -meta_port = 9112 +meta_port = 9122 [BwAuthority] out_dir = ./data/scanner.5/scan-data pid_file = ./data/scanner.5/bwauthority.pid # if db_url is unset bwauthority will default to sqlite -db_url = -#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4 -#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4 -tor_dir = ./data/tor -start_pct = 0 -stop_pct = 100 +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan5 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan5 +tor_dir = ./data/tor.2 +start_pct = 50 +stop_pct = 62 save_every = 0 nodes_per_slice = 50 circs_per_node = 5 min_streams = 1 -max_fetch_time = 360 +max_fetch_time = 300 sleep_start = 01:30 sleep_stop = 01:30 -only_unmeasured = 1 -min_unmeasured = 5 +only_unmeasured = 0 +min_unmeasured = 0 + diff --git a/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg new file mode 100755 index 0000000..5bc3616 --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg @@ -0,0 +1,30 @@ +[TorCtl] +loglevel=WARN +tor_host = 127.0.0.1 +tor_port = 9120 +control_host = 127.0.0.1 +control_port = 9121 +control_pass = +# XXX: Unused +meta_host = 127.0.0.1 +meta_port = 9122 + +[BwAuthority] +out_dir = ./data/scanner.6/scan-data +pid_file = ./data/scanner.6/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan6 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan6 +tor_dir = ./data/tor.2 +start_pct = 62 +stop_pct = 75 +save_every = 0 +nodes_per_slice = 50 +circs_per_node = 5 +min_streams = 1 +max_fetch_time = 300 +sleep_start = 01:30 +sleep_stop = 01:30 +only_unmeasured = 0 +min_unmeasured = 0 diff --git a/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg new file mode 100755 index 0000000..2898b80 --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg @@ -0,0 +1,30 @@ +[TorCtl] +loglevel=WARN +tor_host = 127.0.0.1 +tor_port = 9120 +control_host = 127.0.0.1 +control_port = 9121 +control_pass = +# XXX: Unused +meta_host = 127.0.0.1 +meta_port = 9122 + +[BwAuthority] +out_dir = ./data/scanner.7/scan-data +pid_file = ./data/scanner.7/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan7 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan7 +tor_dir = ./data/tor.2 +start_pct = 75 +stop_pct = 87 +save_every = 0 +nodes_per_slice = 50 +circs_per_node = 5 +min_streams = 1 +max_fetch_time = 300 +sleep_start = 01:30 +sleep_stop = 01:30 +only_unmeasured = 0 +min_unmeasured = 0 diff --git a/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg new file mode 100755 index 0000000..56a5cfa --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg @@ -0,0 +1,30 @@ +[TorCtl] +loglevel=WARN +tor_host = 127.0.0.1 +tor_port = 9120 +control_host = 127.0.0.1 +control_port = 9121 +control_pass = +# XXX: Unused +meta_host = 127.0.0.1 +meta_port = 9122 + +[BwAuthority] +out_dir = ./data/scanner.8/scan-data +pid_file = ./data/scanner.8/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan8 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan8 +tor_dir = ./data/tor.2 +start_pct = 87 +stop_pct = 100 +save_every = 0 +nodes_per_slice = 50 +circs_per_node = 5 +min_streams = 1 +max_fetch_time = 300 +sleep_start = 01:30 +sleep_stop = 01:30 +only_unmeasured = 0 +min_unmeasured = 0 diff --git a/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg new file mode 100644 index 0000000..7e49dbb --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg @@ -0,0 +1,30 @@ +[TorCtl] +loglevel=WARN +tor_host = 127.0.0.1 +tor_port = 9120 +control_host = 127.0.0.1 +control_port = 9121 +control_pass = +# XXX: Unused +meta_host = 127.0.0.1 +meta_port = 9122 + +[BwAuthority] +out_dir = ./data/scanner.9/scan-data +pid_file = ./data/scanner.9/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan9 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan9 +tor_dir = ./data/tor.2 +start_pct = 0 +stop_pct = 100 +save_every = 0 +nodes_per_slice = 50 +circs_per_node = 5 +min_streams = 1 +max_fetch_time = 360 +sleep_start = 01:30 +sleep_stop = 01:30 +only_unmeasured = 1 +min_unmeasured = 5 diff --git a/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore new file mode 100644 index 0000000..20f86df --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore @@ -0,0 +1,2 @@ +bws-* +sql-* \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/data/tor.1/torrc b/NetworkScanners/BwAuthority/data/tor.1/torrc new file mode 100644 index 0000000..933141e --- /dev/null +++ b/NetworkScanners/BwAuthority/data/tor.1/torrc @@ -0,0 +1,18 @@ +## Configuration file for metatroller on alternate prot +SocksPort 9110 # what port to advertise for application connections +ControlPort 9111 +SafeLogging 0 +Log notice file ./data/tor.1/tor.log +DataDirectory ./data/tor.1 +PidFile ./data/tor.1/tor.pid +CookieAuthentication 1 +FetchUselessDescriptors 1 +__LeaveStreamsUnattached 1 + +# Bad idea? Too much consensus update activity? +FetchDirInfoExtraEarly 1 +FetchDirInfoEarly 1 + +# Use CREATE cells for first hop, so we can still measure CPU overload via +# circ failure +FastFirstHopPK 0 diff --git a/NetworkScanners/BwAuthority/data/tor.2/torrc b/NetworkScanners/BwAuthority/data/tor.2/torrc new file mode 100755 index 0000000..209781f --- /dev/null +++ b/NetworkScanners/BwAuthority/data/tor.2/torrc @@ -0,0 +1,18 @@ +## Configuration file for metatroller on alternate prot +SocksPort 9120 # what port to advertise for application connections +ControlPort 9121 +SafeLogging 0 +Log notice file ./data/tor.2/tor.log +DataDirectory ./data/tor.2 +PidFile ./data/tor.2/tor.pid +CookieAuthentication 1 +FetchUselessDescriptors 1 +__LeaveStreamsUnattached 1 + +# Bad idea? Too much consensus update activity? +FetchDirInfoExtraEarly 1 +FetchDirInfoEarly 1 + +# Use CREATE cells for first hop, so we can still measure CPU overload via +# circ failure +FastFirstHopPK 0 diff --git a/NetworkScanners/BwAuthority/data/tor/torrc b/NetworkScanners/BwAuthority/data/tor/torrc deleted file mode 100644 index c64273b..0000000 --- a/NetworkScanners/BwAuthority/data/tor/torrc +++ /dev/null @@ -1,18 +0,0 @@ -## Configuration file for metatroller on alternate prot -SocksPort 9110 # what port to advertise for application connections -ControlPort 9111 -SafeLogging 0 -Log notice file ./data/tor/tor.log -DataDirectory ./data/tor -PidFile ./data/tor/tor.pid -CookieAuthentication 1 -FetchUselessDescriptors 1 -__LeaveStreamsUnattached 1 - -# Bad idea? Too much consensus update activity? -FetchDirInfoExtraEarly 1 -FetchDirInfoEarly 1 - -# Use CREATE cells for first hop, so we can still measure CPU overload via -# circ failure -FastFirstHopPK 0 diff --git a/NetworkScanners/BwAuthority/run_scan.sh b/NetworkScanners/BwAuthority/run_scan.sh index 8a71b1d..908b76e 100755 --- a/NetworkScanners/BwAuthority/run_scan.sh +++ b/NetworkScanners/BwAuthority/run_scan.sh @@ -1,7 +1,9 @@ #!/bin/sh -# Number of scanners to run. -SCANNER_COUNT=5 +# Number of applications to run. +SCANNERS_PER_TOR_COUNT=4 +TOR_COUNT=2 +SCANNER_COUNT=$(($SCANNERS_PER_TOR_COUNT * $TOR_COUNT + 1)) # This tor must have the w status line fix as well as the stream bw fix # Ie git master or 0.2.2.x @@ -19,15 +21,17 @@ for n in `seq $SCANNER_COUNT`; do done KILLED_TOR=false -if [ -f "./data/tor/tor.pid" ]; then - PID=`cat ./data/tor/tor.pid` - if kill -0 "$PID" 2>/dev/null; then # it is a running process and we may send signals to it - kill $PID - if [ $? -eq 0 ]; then - KILLED_TOR=true +for n in `seq $TOR_COUNT`; do + PIDFILE=./data/tor.${n}/tor.pid + if [ -f $PIDFILE ]; then + if kill -0 `head -1 $PIDFILE` 2>/dev/null; then # it is a running process and we may send signals to it + kill `head -1 $PIDFILE` + if [ $? -eq 0 ]; then + KILLED_TOR=true + fi fi fi -fi +done sleep 5 @@ -35,9 +39,10 @@ sleep 5 # A more accurate resume could be implemented in bwauthority.py find data/scanner.* -name .svn -prune -o -type f -a ! -name '*-done-*' -a ! -name bwauthority.cfg -a ! -name .gitignore -exec rm {} + -rm -f ./data/tor/tor.log - -$TOR_EXE -f ./data/tor/torrc & +for n in `seq $TOR_COUNT`; do + rm -f ./data/tor.${n}/tor.log + $TOR_EXE -f ./data/tor.${n}/torrc & +done # If this is a fresh start, we should allow the tors time to download # new descriptors.

1 0

[torflow/master] Fix .gitignores
by mikeperry＠torproject.org 01 Jun '15

01 Jun '15

commit 500b5525cd2d5cb953ae1ccb4b3bc66282b10c72 Author: Tom Ritter <tom(a)ritter.vg> Date: Thu May 21 12:55:52 2015 -0500 Fix .gitignores Make them actually ignore the data files that are generated, and prevent the .gitignores from being deleted by run_scan.sh --- NetworkScanners/BwAuthority/.gitignore | 18 ++++++++++++++++++ .../BwAuthority/data/scanner.1/scan-data/.gitignore | 2 ++ .../BwAuthority/data/scanner.2/scan-data/.gitignore | 2 ++ .../BwAuthority/data/scanner.3/scan-data/.gitignore | 2 ++ .../BwAuthority/data/scanner.4/scan-data/.gitignore | 2 ++ .../BwAuthority/data/scanner.5/scan-data/.gitignore | 2 ++ .../BwAuthority/data/scanner.6/scan-data/.gitignore | 2 ++ .../BwAuthority/data/scanner.7/scan-data/.gitignore | 2 ++ .../BwAuthority/data/scanner.8/scan-data/.gitignore | 2 ++ NetworkScanners/BwAuthority/run_scan.sh | 2 +- 10 files changed, 35 insertions(+), 1 deletion(-) diff --git a/NetworkScanners/BwAuthority/.gitignore b/NetworkScanners/BwAuthority/.gitignore index a0ea384..319a96f 100644 --- a/NetworkScanners/BwAuthority/.gitignore +++ b/NetworkScanners/BwAuthority/.gitignore @@ -1 +1,19 @@ local.cfg +bwscan.V3BandwidthsFile + +data/bwscan* +data/aggregate-control.log +data/aggregate-debug.log + +data/tor*/cached-certs +data/tor*/cached-consensus +data/tor*/cached-descriptors +data/tor*/cached-descriptors.new +data/tor*/control_auth_cookie +data/tor*/lock +data/tor*/state +data/tor*/tor.log +data/tor*/tor.pid + +data/scanner.*/bw.log +data/scanner.*/bwauthority.pid \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/data/scanner.1/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.1/scan-data/.gitignore index e69de29..20f86df 100644 --- a/NetworkScanners/BwAuthority/data/scanner.1/scan-data/.gitignore +++ b/NetworkScanners/BwAuthority/data/scanner.1/scan-data/.gitignore @@ -0,0 +1,2 @@ +bws-* +sql-* \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/data/scanner.2/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.2/scan-data/.gitignore index e69de29..20f86df 100644 --- a/NetworkScanners/BwAuthority/data/scanner.2/scan-data/.gitignore +++ b/NetworkScanners/BwAuthority/data/scanner.2/scan-data/.gitignore @@ -0,0 +1,2 @@ +bws-* +sql-* \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/data/scanner.3/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.3/scan-data/.gitignore index e69de29..20f86df 100644 --- a/NetworkScanners/BwAuthority/data/scanner.3/scan-data/.gitignore +++ b/NetworkScanners/BwAuthority/data/scanner.3/scan-data/.gitignore @@ -0,0 +1,2 @@ +bws-* +sql-* \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/data/scanner.4/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.4/scan-data/.gitignore index e69de29..20f86df 100644 --- a/NetworkScanners/BwAuthority/data/scanner.4/scan-data/.gitignore +++ b/NetworkScanners/BwAuthority/data/scanner.4/scan-data/.gitignore @@ -0,0 +1,2 @@ +bws-* +sql-* \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/data/scanner.5/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.5/scan-data/.gitignore index e69de29..20f86df 100644 --- a/NetworkScanners/BwAuthority/data/scanner.5/scan-data/.gitignore +++ b/NetworkScanners/BwAuthority/data/scanner.5/scan-data/.gitignore @@ -0,0 +1,2 @@ +bws-* +sql-* \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/data/scanner.6/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.6/scan-data/.gitignore new file mode 100644 index 0000000..20f86df --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.6/scan-data/.gitignore @@ -0,0 +1,2 @@ +bws-* +sql-* \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/data/scanner.7/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.7/scan-data/.gitignore new file mode 100644 index 0000000..20f86df --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.7/scan-data/.gitignore @@ -0,0 +1,2 @@ +bws-* +sql-* \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/data/scanner.8/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.8/scan-data/.gitignore new file mode 100644 index 0000000..20f86df --- /dev/null +++ b/NetworkScanners/BwAuthority/data/scanner.8/scan-data/.gitignore @@ -0,0 +1,2 @@ +bws-* +sql-* \ No newline at end of file diff --git a/NetworkScanners/BwAuthority/run_scan.sh b/NetworkScanners/BwAuthority/run_scan.sh index d2674ec..8a71b1d 100755 --- a/NetworkScanners/BwAuthority/run_scan.sh +++ b/NetworkScanners/BwAuthority/run_scan.sh @@ -33,7 +33,7 @@ sleep 5 # FIXME: We resume in a ghetto way by saving the bws-*done* files. # A more accurate resume could be implemented in bwauthority.py -find data/scanner.* -name .svn -prune -o -type f -a ! -name '*-done-*' -a ! -name bwauthority.cfg -exec rm {} + +find data/scanner.* -name .svn -prune -o -type f -a ! -name '*-done-*' -a ! -name bwauthority.cfg -a ! -name .gitignore -exec rm {} + rm -f ./data/tor/tor.log

1 0

[torflow/master] Stop listening for consensus events once we have it.
by mikeperry＠torproject.org 01 Jun '15

01 Jun '15

commit af5fa45ca82d29011676aa97703d77b403e6cf77 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Tue May 26 17:31:15 2015 -0700 Stop listening for consensus events once we have it. --- NetworkScanners/BwAuthority/bwauthority_child.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 13d4080..1c5a0c1 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -226,20 +226,7 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir, successful = 0 while True: if hdlr.is_count_met(circs_per_node, successful): break - hdlr.wait_for_consensus() - - # Check local time. Do not scan between 01:30 and 05:30 local time - lt = time.localtime() - sleep_start = time.mktime(lt[0:3]+sleep_start_tp+(0,0,0)+(lt[-1],)) - sleep_stop = time.mktime(lt[0:3]+sleep_stop_tp+(0,0,0)+(lt[-1],)) t0 = time.time() - if sleep_start <= t0 and t0 <= sleep_stop: - plog("NOTICE", "It's bedtime. Sleeping for "+str(round((sleep_stop-t0)/3600.0,1))+"h") - try: - time.sleep(sleep_stop - t0) - except: - pass - t0 = time.time() hdlr.new_exit() attempt += 1 @@ -364,6 +351,13 @@ def main(argv): plog("NOTICE", "Woke up from waiting for more unmeasured nodes. Getting consensus and checking again") hdlr.wait_for_consensus() + # Now that we have the consensus, we shouldn't need to listen + # for new consensus events. + c.set_events([TorCtl.EVENT_TYPE.STREAM, + TorCtl.EVENT_TYPE.BW, + TorCtl.EVENT_TYPE.CIRC, + TorCtl.EVENT_TYPE.STREAM_BW], True) + pct_step = hdlr.rank_to_percent(nodes_per_slice) plog("INFO", "Percent per slice is: "+str(pct_step)) if pct_step > 100: pct_step = 100

1 0

[torflow/master] Workaround strange issue with unmeasured flag parsing.
by mikeperry＠torproject.org 01 Jun '15

01 Jun '15

commit a06038c682c9861c651bad798d27cb5096bf6db6 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Sun May 31 19:17:52 2015 -0700 Workaround strange issue with unmeasured flag parsing. It seems like consensus/newdesc events are causing TorCtl to reset the unmeasured flag when we shouldn't be. That is a TorCtl bug, but since there may also be memory bloat issues with leaving the bwauthority_child processes around anyway, just exit and let the parent restart the slice with a clean consensus. --- NetworkScanners/BwAuthority/bwauthority.py | 6 ++++++ NetworkScanners/BwAuthority/bwauthority_child.py | 17 +++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index e190e7d..6a04542 100755 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -12,11 +12,14 @@ from signal import signal, SIGTERM, SIGKILL # exit code to indicate scan completion # make sure to update this in bwauthority_child.py as well STOP_PCT_REACHED = 9 +RESTART_SLICE = 1 # path to git repos (.git) PATH_TO_TORFLOW_REPO = '../../.git/' PATH_TO_TORCTL_REPO = '../../.git/modules/TorCtl/' +p = None + def main(argv): (branch, head) = get_git_version(PATH_TO_TORFLOW_REPO) plog('INFO', 'TorFlow Version: %s' % branch+' '+head) @@ -30,6 +33,8 @@ def main(argv): p.wait() if (p.returncode == 0): slice_num += 1 + elif (p.returncode == RESTART_SLICE): + plog('INFO', 'restarting slice_num '+str(slice_num)) elif (p.returncode == STOP_PCT_REACHED): plog('INFO', 'restarting from slice 0') slice_num = 0 @@ -60,6 +65,7 @@ if __name__ == '__main__': try: main(s_argv) except KeyboardInterrupt: + global p p.kill() plog('INFO', "Ctrl + C was pressed. Exiting ... ") except Exception, e: diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 1c5a0c1..a42a9b5 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -72,6 +72,7 @@ __selmgr = PathSupport.SelectionManager( # exit code to indicate scan completion # make sure to update this in bwauthority.py as well STOP_PCT_REACHED = 9 +RESTART_SLICE = 1 def read_config(filename): config = ConfigParser.SafeConfigParser() @@ -343,14 +344,6 @@ def main(argv): hdlr.wait_for_consensus() - # We should go to sleep if there are less than 5 unmeasured nodes after - # consensus update - while min_unmeasured and hdlr.get_unmeasured() < min_unmeasured: - plog("NOTICE", "Less than "+str(min_unmeasured)+" unmeasured nodes ("+str(hdlr.get_unmeasured())+"). Sleeping for a bit") - time.sleep(3600) # Until next consensus arrives - plog("NOTICE", "Woke up from waiting for more unmeasured nodes. Getting consensus and checking again") - hdlr.wait_for_consensus() - # Now that we have the consensus, we shouldn't need to listen # for new consensus events. c.set_events([TorCtl.EVENT_TYPE.STREAM, @@ -358,6 +351,14 @@ def main(argv): TorCtl.EVENT_TYPE.CIRC, TorCtl.EVENT_TYPE.STREAM_BW], True) + # We should go to sleep if there are less than 5 unmeasured nodes after + # consensus update + if min_unmeasured and hdlr.get_unmeasured() < min_unmeasured: + plog("NOTICE", "Less than "+str(min_unmeasured)+" unmeasured nodes ("+str(hdlr.get_unmeasured())+"). Sleeping for a bit") + time.sleep(3600) # Until next consensus arrives + plog("NOTICE", "Woke up from waiting for more unmeasured nodes. Requesting slice restart.") + sys.exit(RESTART_SLICE) + pct_step = hdlr.rank_to_percent(nodes_per_slice) plog("INFO", "Percent per slice is: "+str(pct_step)) if pct_step > 100: pct_step = 100

1 0

[torflow/master] Create script to Use virtualenv to setup everything.
by mikeperry＠torproject.org 01 Jun '15

01 Jun '15

commit 3d87c6884a8cb59df356653093794a6ac425b820 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Wed May 20 17:38:03 2015 -0700 Create script to Use virtualenv to setup everything. This will help us pin versions to eliminate bitrot issues. --- NetworkScanners/BwAuthority/README.BwAuthorities | 174 ++++++++-------------- NetworkScanners/BwAuthority/aggregate.py | 2 +- NetworkScanners/BwAuthority/alpha_test.py | 2 +- NetworkScanners/BwAuthority/bwauthority_child.py | 2 +- NetworkScanners/BwAuthority/cron.sh | 8 +- NetworkScanners/BwAuthority/install-debs.sh | 27 ++++ NetworkScanners/BwAuthority/run_scan.sh | 9 ++ NetworkScanners/BwAuthority/setup.sh | 65 ++++++++ 8 files changed, 174 insertions(+), 115 deletions(-) diff --git a/NetworkScanners/BwAuthority/README.BwAuthorities b/NetworkScanners/BwAuthority/README.BwAuthorities index dec3ed2..fcf1f54 100644 --- a/NetworkScanners/BwAuthority/README.BwAuthorities +++ b/NetworkScanners/BwAuthority/README.BwAuthorities @@ -3,7 +3,10 @@ How to Run a Bandwidth-Measuring Directory Authority -0. Run a Directory Authority +0. Run a Directory Authority or Find One + +A Directory Authority is not required to run the bw scanners, but it is +required if you want to submit results for the consensus. See http://git.torproject.org/checkout/tor/master/doc/v3-authority-howto.txt @@ -12,74 +15,68 @@ your authority. You can get it with: git clone git://git.torproject.org/git/tor.git tor.git +You can also submit your results to an existing bandwidth authority. +Basically, this will involve placing the bwscan.V3BandwidthsFile output on a +webserver or SSH host that a bw authority can use to download that file. See +Section 4 for more details. + -1. Find a machine with 10Mbit+ downstream +1. Find a machine with 100Mbit+ downstream This can be the same as your directory authority, but it does not have -to be. You will not need the 10Mbit continuously, but it should be +to be. You will not need the 100Mbit continuously, but it should be available on demand, as some of the faster nodes actually do have this much slack capacity. You can test your capacity by hitting the current test server directly: # wget --no-check-certificate https://38.229.70.2/64M -The machine will require around 4-5Gbytes/day. - -2. Set up TorCtl - -You can add TorCtl (pytorctl.git) as a git submodule by running the add_torctl.sh script in -the root of torflow.git. BwAuthority expects pytorctl to be checked out into the root of -torflow as TorCtl. - - -3. Compile Tor for your authority and your scanner - -No special configure script options are needed, but again, both -need to be running the master branch from tor git. - - -4. Install Dependencies - -4.1. Dependencies from your distribution's package manager +2. Installation and setup -In Debian-based systems, the following packages are required: +The bandwidth authorities are sensitive to exact component versions. There are +two ways to set them up with the versions they need: use our scripts to +prepare a virtualenv, or run through the setup manually. - $ sudo apt-get install python2.6 libpython2.6-dev libsqlite3-dev +2.1. Scripted virtualenv setup -If you want to use postgres support, you should also install python-psycopg2. +The easiest and most reliable setup method is to use the setup.sh script +to install a python 2.6 virtual environment. This script will download all +of the dependencies and install them for you, but it will require that you +have a copy of python2.6 installed and in your path. +There is also a install-debs.sh script for Debian and Ubuntu systems that will +handle python2.6 and some additional package dependency installation for you. -4.2. Python Dependencies +2.2. Manual setup +You really should at least look at the virtualenv setup.sh script before +trying this, but if you insist, here are the step by step instructions. -4.2.1. Using Pip or Peep: +2.2.1. Set up TorCtl -First, ensure that you've got a *recent* version of pip. If you already have -pip, do: +You need to add TorCtl (pytorctl.git) as a git submodule by running the +add_torctl.sh script in the root of torflow.git. BwAuthority expects pytorctl +to be checked out into the root of torflow as TorCtl. - $ pip install --upgrade pip +2.2.2. Set up Tor -Next, if you'd like to verify the correctness of the downloaded dependencies -with SHA-256 (rather than MD5, which is pip's default), do: +The bandwidth authorities expect a tor binary in a tor.git repository along +side the current torflow checkout. Here is how you would set that up: - $ pip install peep + cd ../../../ + git clone https://git.torproject.org/tor.git tor.git + cd tor.git + git checkout release-0.2.4 + ./autogen.sh + ./configure --disable-asciidoc + make -j4 -Finally, do: +2.2.3. Install Python Dependencies - $ peep install -r .../NetworkScanners/BwAuthority/requirements.txt +The Bandwidth Authorities use the SQLAlchemy is 0.7.2 and Elixir 0.7.1. -(Or, if you didn't install peep, do `$ pip install -r requirements.txt`.) - - -4.2.2. The Tedious Way - -The latest version of SQLAlchemy is 0.7.2 and the latest version of Elixir -is 0.7.1 at the time of writing. While TorFlow is written to be compatible -with 0.4.x and 0.5.x and 0.6.x of SQLAlchemy, 0.5.5 was noted for -problems parsing postgres database URLS, 0.4.8 seems to exhibit odd object persistence bugs. - -If your distribution does not provide 0.7.x or newer, you will likely want to +If your distribution does not provide 0.7.x, you will likely want to download that tarball from: http://pypi.python.org/pypi/SQLAlchemy/ @@ -88,7 +85,7 @@ Untar it in the same directory that contains the TorFlow checkout and your git checkout (for peace of mind, you will want all three in the same place). -If your distribution does not provide Elixir 0.7.x or above, do the +If your distribution does not provide Elixir 0.7.x, do the same with Elixir: http://pypi.python.org/pypi/Elixir/ @@ -102,50 +99,13 @@ Elixir-0.7.1.tar.gz SQLAlchemy-0.7.2.tar.gz torflow-trunk Both these libraries also depend upon python-pysqlite2, which should be a package for your distribution (you want 2.3.x for SQLite 3.x). - -5. Enable voting on bandwidths in your authority torrc - -The new configuration option is V3BandwidthsFile. It specifies the -file containing your measured results, which we will configure -in the later steps. Pick a location accessible by your Tor -directory authority process and any rsync user you may have. - -I recommend /var/lib/tor.scans/bwscan. If you try to use -/var/lib/tor, tor will reset your permissions and exclude -any other users from writing the file there. - - -6. Create a new user capable of writing the bwscan file - -You will need to run the scanning scripts as a separate user. That's -because the scripts run commands like 'killall tor' and expect it not -to affect any other tor processes. - -The new user should have write access to your bwscan dir from step 5. - -# useradd bwscanner -# chown toruser:bwscanner /var/lib/tor.scans/ -# chmod 770 /var/lib/tor.scans/ - - -7. Spot-check ./run_scan.sh - -This is the script that will launch the scanners. By default, it -launches four in parallel, and expects the git checkout to be in -../../../tor.git/, and the SQLAlchemy extraction to be in -../../../SQLAlchemy-0.7.x - -Again, note that this is the same directory that contains the -torflow checkout directory. - - -8. Set up a cron job to submit results +2.2.4. Set up a cron job to submit results The provided cron.sh script is meant to be used in a cron job to aggregate the results and provide them to your directory authority at least every four hours, but more often is better. -Because cron.sh is likely to be updated by SVN, you're going to want to +Because cron.sh is likely to be updated by git, you're going to want to make your own copy before you install the cron job: # cp cron.sh cron-mine.sh @@ -176,7 +136,22 @@ will require the most bandwidth, and ./data/scanner.4 will require the least. -9. PROFIT! +3. Enable voting on bandwidths in your authority torrc + +The Bandwidth Authorities can be run without a directory authority, but for +their results to count, they must be paired with a working dirauth. + +The dirauth-side configuration option is V3BandwidthsFile. It specifies the +file containing your measured results, which we will configure in the later +steps. Pick a location accessible by your Tor directory authority process and +any rsync user you may have. + +I recommend /var/lib/tor.scans/bwscan. If you try to use /var/lib/tor, tor +will reset your permissions and exclude any other users from writing the file +there. + + +4. PROFIT! That's all there is to it. No '????' step needed! @@ -185,8 +160,8 @@ That's all there is to it. No '????' step needed! Appendix A: Creating the HTTPS scanning server The scanner server will need approx 30-40Mbit of upstream available, and will -need to serve https via a fixed IP. SSL is needed to avoid HTTP content -caches at the various exit nodes. Self-signed certs are OK. +need to serve https via a fixed IP. SSL is needed to avoid HTTP content caches +at the various exit nodes. Self-signed certs are OK. The server will consume around 12-15Gbytes/day. @@ -202,26 +177,3 @@ for i in 512 256 128 64 32 16; do done -Appendix B: Configuring PostgreSQL backend - -To use postgres instead of sqlite: - -1. Install postgresql: -sudo apt-get install postgresql postgresql-common postgresql-client-common - -2. Create role: -sudo -u postgres psql - CREATE USER bwscanner WITH PASSWORD 'password'; - -3. Create databases: -sudo -u postgres createdb BwScan1 -O bwscanner -sudo -u postgres createdb BwScan2 -O bwscanner -sudo -u postgres createdb BwScan3 -O bwscanner -sudo -u postgres createdb BwScan4 -O bwscanner - -4. Update bwauthority.cfg files -comment out the lines beginning with db_url= -uncomment the line: -#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 - -5. ./run_scan.sh diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 77e3cdc..cbd6657 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python import os import re import math diff --git a/NetworkScanners/BwAuthority/alpha_test.py b/NetworkScanners/BwAuthority/alpha_test.py index bd13800..6273d08 100755 --- a/NetworkScanners/BwAuthority/alpha_test.py +++ b/NetworkScanners/BwAuthority/alpha_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python import sys diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index e8d2c19..3bf7e03 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # 2009 Mike Perry, Karsten Loesing diff --git a/NetworkScanners/BwAuthority/cron.sh b/NetworkScanners/BwAuthority/cron.sh index e424a8f..d304031 100755 --- a/NetworkScanners/BwAuthority/cron.sh +++ b/NetworkScanners/BwAuthority/cron.sh @@ -1,12 +1,18 @@ #!/bin/sh -SCANNER_DIR=~/code/tor/torflow/NetworkScanners/BwAuthority +SCANNER_DIR=$(dirname "$0") +SCANNER_DIR=$(readlink -f "$SCANNER_DIR") TIMESTAMP=`date +%Y%m%d-%H%M` ARCHIVE=$SCANNER_DIR/data/bwscan.${TIMESTAMP} OUTPUT=$SCANNER_DIR/bwscan.V3BandwidthsFile cd $SCANNER_DIR # Needed for import to work properly. +if [ -f bwauthenv/bin/activate ] +then + echo "Using virtualenv..." + . bwauthenv/bin/activate +fi $SCANNER_DIR/aggregate.py $SCANNER_DIR/data $OUTPUT if [ $? = 0 ] diff --git a/NetworkScanners/BwAuthority/install-debs.sh b/NetworkScanners/BwAuthority/install-debs.sh new file mode 100755 index 0000000..8357d0b --- /dev/null +++ b/NetworkScanners/BwAuthority/install-debs.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +if [ ! $(dpkg -s python2.6 python2.6-dev 2>/dev/null >/dev/null) ] +then + echo "We need python2.6 to be in the path. Press enter to try to install it." + echo "or control-c and find your own way to install it and re-run this script" + echo + echo -n "Hit enter to install python2.6: " + read + sudo apt-get install python2.6 python2.6-dev + if [ $? -ne 0 ] + then + echo + echo "Your distribution does not natively provide python2.6." + echo "Press enter to try to install from a ppa, or control-c to install on your own" + echo + echo -n "Hit enter to install from ppa:fkrull/deadsnakes: " + read + sudo apt-get install software-properties-common + sudo add-apt-repository ppa:fkrull/deadsnakes + sudo apt-get update + sudo apt-get install python2.6 python2.6-dev + fi +fi + +sudo apt-get install libsqlite3-dev python-virtualenv +sudo apt-get install autoconf2.13 automake make libevent-dev diff --git a/NetworkScanners/BwAuthority/run_scan.sh b/NetworkScanners/BwAuthority/run_scan.sh index c6b552f..c21e871 100755 --- a/NetworkScanners/BwAuthority/run_scan.sh +++ b/NetworkScanners/BwAuthority/run_scan.sh @@ -49,8 +49,17 @@ else sleep 500 fi +if [ -f bwauthenv/bin/activate ] +then + echo "Using virtualenv..." + . bwauthenv/bin/activate +fi + [ -z "$PYTHONPATH" ] || export PYTHONPATH for n in `seq $SCANNER_COUNT`; do nice -n 20 ./bwauthority.py ./data/scanner.${n}/bwauthority.cfg \ > ./data/scanner.${n}/bw.log 2>&1 & done + +echo "Launched $SCANNER_COUNT bandwidth scanners. Job listing: " +jobs -l diff --git a/NetworkScanners/BwAuthority/setup.sh b/NetworkScanners/BwAuthority/setup.sh new file mode 100755 index 0000000..206f4a2 --- /dev/null +++ b/NetworkScanners/BwAuthority/setup.sh @@ -0,0 +1,65 @@ +#!/bin/bash -e + +SCANNER_DIR=$(dirname "$0") +SCANNER_DIR=$(readlink -f "$SCANNER_DIR") + +# 1. Install python2.6 if needed +if [ -z "$(which python2.6)" ] +then + echo "We need python2.6 to be in the path." + echo "If you are on a Debian or Ubuntu system, you can try ./install-debs.sh" + exit 1 +fi + +if [ -z "$(which virtualenv)" ] +then + echo "We need virtualenv to be in the path. If you are on a debian system, try:" + echo " sudo apt-get install libsqlite3-dev python-virtualenv" + exit 1 +fi + +# 2. Ensure TorCtl submodule is added +pushd ../../ +./add_torctl.sh +popd + +# 3. Compile tor 0.2.6 +if [ ! -x ../../../tor/src/or/tor ] +then + pushd ../../../ + git clone https://git.torproject.org/tor.git tor + cd tor + git checkout release-0.2.6 + ./autogen.sh + ./configure --disable-asciidoc + make -j4 + popd +fi + +# 4. Initialize virtualenv +if [ ! -f bwauthenv/bin/activate ] +then + virtualenv -p python2.6 bwauthenv +fi +source bwauthenv/bin/activate + +# 5. Install new pip and peep +pip install --upgrade https://pypi.python.org/packages/source/p/pip/pip-6.1.1.tar.gz#sha256=89f3b… +pip install https://pypi.python.org/packages/source/p/peep/peep-2.4.1.tar.gz#sha256=2a8… +peep install -r $SCANNER_DIR/requirements.txt + +# 6. Prepare cron script +cp cron.sh cron-mine.sh +echo -e "45 0-23 * * * $SCANNER_DIR/cron-mine.sh" | crontab +echo -e "@reboot $SCANNER_DIR/run_scan.sh\n`crontab -l`" | crontab +echo "Prepared crontab. Current crontab: " +crontab -l + +# 7. Inform user what to do +echo +echo "If we got this far, everything should be ready!" +echo +echo "Start the scan with ./run_scan.sh" +echo "You can manually run ./cron-mine.sh manually to check results" +echo "Detailed logs are in ./data/scanner.*/bw.log." +echo "Progress can also be inferred from files in ./data/scanner.*/scan-data"

1 0

[torflow/master] Log the success counts for each slice.
by mikeperry＠torproject.org 01 Jun '15

01 Jun '15

commit bb2fcc0efeaa3e24b35d9b6241173d6eaaa8f6c2 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Wed May 20 19:04:20 2015 -0700 Log the success counts for each slice. --- NetworkScanners/BwAuthority/bwauthority_child.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 3bf7e03..2584991 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -290,6 +290,8 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir, plog('DEBUG', 'Wrote stats') #hdlr.save_sql_file(sql_file, os.getcwd()+"/"+out_dir+"/bw-db-"+str(lo)+":"+str(hi)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S")+".sqlite") + return successful + def main(argv): plog("DEBUG", "Child Process Spawning...") TorUtil.read_config(argv[1]) @@ -340,8 +342,8 @@ def main(argv): sys.exit(STOP_PCT_REACHED) plog("DEBUG", "Starting slice number %s" % slice_num) - speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, save_every, out_dir, - max_fetch_time, sleep_start, sleep_stop, slice_num, + successful = speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, + save_every, out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, min_streams, sql_file) # For debugging memory leak.. @@ -352,6 +354,12 @@ def main(argv): #circ_measure(hdlr, pct, pct+pct_step, circs_per_node, save_every, # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) + + # XXX: Hack this to return a codelen double the slice size on failure? + plog("NOTICE", "Slice success count: "+str(successful)) + if successful == 0: + plog("NOTICE", "Slice success count is ZERO!") + sys.exit(0) def ignore_streams(c,hdlr):

1 0

[torflow/master] Turn on code to add exits if few are present in a slice.
by mikeperry＠torproject.org 01 Jun '15

01 Jun '15

commit 238f61666b31be5095d614f5c7ae9c77dc3c5486 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Tue May 26 15:18:11 2015 -0700 Turn on code to add exits if few are present in a slice. --- NetworkScanners/BwAuthority/bwauthority_child.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 4bac13a..4b4ac6f 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -65,7 +65,8 @@ __selmgr = PathSupport.SelectionManager( use_exit=None, use_guards=False, exit_ports=[443], - order_by_ratio=True) # XXX: may be a poor idea for PID control? + order_by_ratio=True, # XXX: may be a poor idea for PID control? + min_exits=10) # exit code to indicate scan completion # make sure to update this in bwauthority.py as well

1 0

[torflow/master] Add a pip/peep requirements.txt file for running a Bandwidth Authority.
by mikeperry＠torproject.org 01 Jun '15

01 Jun '15

commit de0f7ed5c5a8e79eabc824d4074fac6ead31efb3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed May 20 06:57:18 2015 +0000 Add a pip/peep requirements.txt file for running a Bandwidth Authority. --- NetworkScanners/BwAuthority/requirements.txt | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/NetworkScanners/BwAuthority/requirements.txt b/NetworkScanners/BwAuthority/requirements.txt new file mode 100644 index 0000000..1cfee33 --- /dev/null +++ b/NetworkScanners/BwAuthority/requirements.txt @@ -0,0 +1,29 @@ +--no-use-wheel --no-index + +# This is pip/peep requirements.txt file specifying the pinned Python +# dependencies for running a Bandwidth Authority. +# +# To use this file, first ensure that you've got a *recent* version +# of pip. If you already have pip, do: +# +# $ pip install --upgrade pip +# +# Next, if you'd like to verify the correctness of the downloaded dependencies +# with SHA-256 (rather than MD5, which is pip's default), do: +# +# $ pip install peep +# +# Finally, to use this file, do: +# +# $ peep install -r requirements.txt +# +# (Or, if you didn't install peep, do `$ pip install -r requirements.txt`.) + +# sha256: NUKQvRsYu2nhGUay2RP9mxXCpKp94uhWXSmUktt3NyE +pysqlite==2.6.0 + +# sha256: tYgFyp-PUSPmCfJTqRboeh4-rERqCq4Sjt5jBIXI9AY +SQLAlchemy==0.7.2 + +# sha256: p-9DfyW1ROT3T7Mjb8Q80l9db-tgN918ZpMQRtdUOek +Elixir==0.7.1

1 0

[torflow/master] Add section to README.BwAuthorities with pip/peep instructions.
by mikeperry＠torproject.org 01 Jun '15

01 Jun '15

commit 412fd5c95d5552c26c489898603d32679c6b7807 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed May 20 06:58:13 2015 +0000 Add section to README.BwAuthorities with pip/peep instructions. --- NetworkScanners/BwAuthority/README.BwAuthorities | 38 ++++++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/NetworkScanners/BwAuthority/README.BwAuthorities b/NetworkScanners/BwAuthority/README.BwAuthorities index 576fead..dec3ed2 100644 --- a/NetworkScanners/BwAuthority/README.BwAuthorities +++ b/NetworkScanners/BwAuthority/README.BwAuthorities @@ -38,7 +38,41 @@ No special configure script options are needed, but again, both need to be running the master branch from tor git. -4. Download SQLAlchemy 0.7.x-latest and Elixir 0.7.x-latest +4. Install Dependencies + + +4.1. Dependencies from your distribution's package manager + +In Debian-based systems, the following packages are required: + + $ sudo apt-get install python2.6 libpython2.6-dev libsqlite3-dev + +If you want to use postgres support, you should also install python-psycopg2. + + +4.2. Python Dependencies + + +4.2.1. Using Pip or Peep: + +First, ensure that you've got a *recent* version of pip. If you already have +pip, do: + + $ pip install --upgrade pip + +Next, if you'd like to verify the correctness of the downloaded dependencies +with SHA-256 (rather than MD5, which is pip's default), do: + + $ pip install peep + +Finally, do: + + $ peep install -r .../NetworkScanners/BwAuthority/requirements.txt + +(Or, if you didn't install peep, do `$ pip install -r requirements.txt`.) + + +4.2.2. The Tedious Way The latest version of SQLAlchemy is 0.7.2 and the latest version of Elixir is 0.7.1 at the time of writing. While TorFlow is written to be compatible @@ -68,8 +102,6 @@ Elixir-0.7.1.tar.gz SQLAlchemy-0.7.2.tar.gz torflow-trunk Both these libraries also depend upon python-pysqlite2, which should be a package for your distribution (you want 2.3.x for SQLite 3.x). -If you want to use postgres support, you should also install python-psycopg2. - 5. Enable voting on bandwidths in your authority torrc

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits June 2015