- tor-commits - lists.torproject.org

[torflow/master] Clean up imports
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit fdfa2e7ad6576effd429274e0369cdb2cc461ddb Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 19:04:17 2011 -0700 Clean up imports --- NetworkScanners/BwAuthority/bwauthority.py | 21 ++++++++------------- 1 files changed, 8 insertions(+), 13 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index b1a6197..62f0658 100755 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -1,14 +1,11 @@ #!/usr/bin/env python -import sys -import subprocess -import ConfigParser -import os -import traceback -sys.path.append("../../") -from TorCtl import TorUtil -from TorCtl.TorUtil import plog -from signal import signal, SIGTERM +from sys import argv as s_argv +from sys import path from sys import exit +from subprocess import Popen +path.append("../../") +from TorCtl.TorUtil import plog as plog +from signal import signal, SIGTERM # exit code to indicate scan completion @@ -20,7 +17,7 @@ def main(argv): while True: plog('INFO', 'Beginning time loop') global p - p = subprocess.Popen(["python", "bwauthority_child.py", argv[1], str(slice_num)]) + p = Popen(["python", "bwauthority_child.py", argv[1], str(slice_num)]) p.wait() if (p.returncode == 0): slice_num += 1 @@ -38,11 +35,9 @@ def sigterm_handler(signum, frame): if __name__ == '__main__': signal(SIGTERM, sigterm_handler) try: - main(sys.argv) + main(s_argv) except KeyboardInterrupt: p.kill() plog('INFO', "Ctrl + C was pressed. Exiting ... ") - traceback.print_exc() except Exception, e: plog('ERROR', "An unexpected error occured.") - traceback.print_exc()

1 0

[torflow/master] Return code should be positive, see sys.exit()
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit f4346a667801d7dee3c602cf02fe419ed1693031 Author: aagbsn <aagbsn(a)extc.org> Date: Sat Jun 18 16:04:39 2011 -0700 Return code should be positive, see sys.exit() --- NetworkScanners/BwAuthority/bwauthority.py | 2 +- NetworkScanners/BwAuthority/bwauthority_child.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index 62f0658..12eeab0 100755 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -10,7 +10,7 @@ from signal import signal, SIGTERM # exit code to indicate scan completion # make sure to update this in bwauthority_child.py as well -STOP_PCT_REACHED = -9 +STOP_PCT_REACHED = 9 def main(argv): slice_num = 0 diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 4b09f9f..e8a0a68 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -76,7 +76,7 @@ __selmgr = PathSupport.SelectionManager( # exit code to indicate scan completion # make sure to update this in bwauthority.py as well -STOP_PCT_REACHED = -9 +STOP_PCT_REACHED = 9 def read_config(filename): config = ConfigParser.SafeConfigParser() @@ -309,6 +309,7 @@ def main(argv): plog('INFO', 'stop_pct: %s reached. Exiting with %s' % (stop_pct, STOP_PCT_REACHED)) sys.exit(STOP_PCT_REACHED) + plog("DEBUG", "Starting slice number %s" % slice_num) speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, save_every, out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, min_streams, sql_file)

1 0

[torflow/master] Update submodule.
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit cb4db2317437621a22531e84b8686fc28785f2bf Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 24 12:29:28 2011 -0700 Update submodule. --- TorCtl | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/TorCtl b/TorCtl index af306b5..d0f593f 160000 --- a/TorCtl +++ b/TorCtl @@ -1 +1 @@ -Subproject commit af306b53a668168ae92a6c919d4013b81d7e61ad +Subproject commit d0f593f2ae245e951323ff7fc3a65fa6a1f16b47

1 0

[torflow/master] Update bw file percentages again; remove python 2.5ism.
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 5cd1f2be2e370176bf13ce44ad2033367cba032a Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 24 12:28:00 2011 -0700 Update bw file percentages again; remove python 2.5ism. --- NetworkScanners/BwAuthority/bwauthority_child.py | 17 ++++++++++------- 1 files changed, 10 insertions(+), 7 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index e8a0a68..c342f2e 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -50,13 +50,14 @@ user_agent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.37 # TODO: We'll also want to try to prefer pairing unmeasured nodes # together then, and use a different url set for them. # cutoff percent URL -urls = [(5, "https://38.229.70.2/16M"), # fbw 1499k..500k - (10, "https://38.229.70.2/8M"), # fbw 500k..350k - (20, "https://38.229.70.2/4M"), # fbw 350k..200k - (30, "https://38.229.70.2/2M"), # fbw 200k..128k - (50, "https://38.229.70.2/512k"), # fbw 128k..50k - (80, "https://38.229.70.2/256k"), # fbw 50k..26k - (100, "https://38.229.70.2/128k")] # fbw 26k..10k +urls = [(5, "https://38.229.70.2/8M"), # fbw 2000k..451k + (10, "https://38.229.70.2/4M"), # fbw 451k..275k + (20, "https://38.229.70.2/2M"), # fbw 275k..150k + (30, "https://38.229.70.2/1M"), # fbw 150k..100k + (40, "https://38.229.70.2/1M"), # fbw 100k..75k + (50, "https://38.229.70.2/512k"), # fbw 75k..50k + (80, "https://38.229.70.2/256k"), # fbw 50k..25k + (100, "https://38.229.70.2/128k")] # fbw 25k..10k # Do NOT modify this object directly after it is handed to PathBuilder @@ -376,6 +377,8 @@ if __name__ == '__main__': except KeyboardInterrupt: plog('INFO', "Ctrl + C was pressed. Exiting ... ") traceback.print_exc() + except SystemExit, e: + sys.exit(e) except Exception, e: plog('ERROR', "An unexpected error occured.") traceback.print_exc()

1 0

[torflow/master] Update submodule..
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 9f756249cf53985a944cb27bf7101d4ff7f369c3 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 24 12:33:36 2011 -0700 Update submodule.. --- TorCtl | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/TorCtl b/TorCtl index d0f593f..dc1cb9c 160000 --- a/TorCtl +++ b/TorCtl @@ -1 +1 @@ -Subproject commit d0f593f2ae245e951323ff7fc3a65fa6a1f16b47 +Subproject commit dc1cb9ce39fd5c9c5ea50f542642130a44e71fee

1 0

[torflow/master] bwauthority parent should exit when child is killed
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 0e32c07900b8722d67b338b00fd0c79d746b16a3 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 23 11:39:58 2011 -0700 bwauthority parent should exit when child is killed --- NetworkScanners/BwAuthority/bwauthority.py | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index 12eeab0..9125044 100755 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -5,7 +5,7 @@ from sys import exit from subprocess import Popen path.append("../../") from TorCtl.TorUtil import plog as plog -from signal import signal, SIGTERM +from signal import signal, SIGTERM, SIGKILL # exit code to indicate scan completion @@ -24,6 +24,12 @@ def main(argv): elif (p.returncode == STOP_PCT_REACHED): plog('INFO', 'restarting from slice 0') slice_num = 0 + elif (abs(p.returncode) == SIGKILL): + plog('WARN', 'Child process recieved SIGKILL, exiting') + exit() + elif (abs(p.returncode) == SIGTERM): + plog('WARN', 'Child process recieved SIGTERM, exiting') + exit() else: plog('WARN', 'Child process returned %s' % p.returncode)

1 0

[torflow/master] added db_url config parameter
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 83348a9cc59fe035026917b41585f101ea169a73 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 16 19:30:33 2011 -0700 added db_url config parameter --- NetworkScanners/BwAuthority/bwauthority.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index 0d57dfe..bcc9a49 100644 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -14,7 +14,7 @@ def main(argv): TorUtil.read_config(argv[1]) (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, max_fetch_time,tor_dir,sleep_start,sleep_stop, - min_streams,pid_file_name) = bwauthority_child.read_config(argv[1]) + min_streams,pid_file_name,db_url) = bwauthority_child.read_config(argv[1]) if pid_file_name: pidfd = file(pid_file_name, 'w')

1 0

[torflow/master] added db_url to configs
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 28dc35752bbf70ba4b417b4f7490d51a55c127f9 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 16 19:49:38 2011 -0700 added db_url to configs --- .../BwAuthority/data/scanner.1/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.2/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.3/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.4/bwauthority.cfg | 2 ++ 4 files changed, 8 insertions(+), 0 deletions(-) diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg index a90ce97..bd3a0ca 100644 --- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.1/scan-data pid_file = ./data/scanner.1/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct = 0 stop_pct = 12 diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg index 959f1e6..ea2472d 100644 --- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.2/scan-data pid_file = ./data/scanner.2/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct = 12 stop_pct = 35 diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg index 71e1f6f..526a912 100644 --- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.3/scan-data pid_file = ./data/scanner.3/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct =35 stop_pct = 60 diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg index 5a6b500..5a7c7a7 100644 --- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.4/scan-data pid_file = ./data/scanner.4/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct =60 stop_pct = 100

1 0

[torflow/master] Removed conflicting tor configuration options
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 77ed1ab8de9a264f0f885ea266227894c47056e0 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 10:44:42 2011 -0700 Removed conflicting tor configuration options __LeaveStreamsUnattached and FetchUselessDescriptors should now be set in torrc rather than by bwauthority to avoid a race between child processes --- NetworkScanners/BwAuthority/bwauthority_child.py | 17 ++++------------- 1 files changed, 4 insertions(+), 13 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 4d968ac..a433c0f 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -267,6 +267,7 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir, #hdlr.save_sql_file(sql_file, os.getcwd()+"/"+out_dir+"/bw-db-"+str(lo)+":"+str(hi)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S")+".sqlite") def main(argv): + plog("DEBUG", "Child Process Spawning...") TorUtil.read_config(argv[1]) (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, max_fetch_time,tor_dir,sleep_start,sleep_stop, @@ -319,14 +320,8 @@ def main(argv): # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) sys.exit(0) -def cleanup(c, f): - plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors="+f) - try: - # XXX: Remember __LeaveStreamsUnattached and use saved value! - c.set_option("__LeaveStreamsUnattached", "0") - c.set_option("FetchUselessDescriptors", f) - except TorCtl.TorCtlClosed: - pass +def cleanup(): + plog("DEBUG", "Child Process Exiting...") def setup_handler(out_dir, cookie_file): plog('INFO', 'Connecting to Tor at '+TorUtil.control_host+":"+str(TorUtil.control_port)) @@ -335,7 +330,6 @@ def setup_handler(out_dir, cookie_file): c = PathSupport.Connection(s) #c.debug(file(out_dir+"/control.log", "w", buffering=0)) c.authenticate_cookie(file(cookie_file, "r")) - #f = c.get_option("__LeaveStreamsUnattached")[0] h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector) @@ -349,10 +343,7 @@ def setup_handler(out_dir, cookie_file): TorCtl.EVENT_TYPE.CIRC, TorCtl.EVENT_TYPE.STREAM_BW], True) - c.set_option("__LeaveStreamsUnattached", "1") - f = c.get_option("FetchUselessDescriptors")[0][1] - c.set_option("FetchUselessDescriptors", "1") - atexit.register(cleanup, *(c, f)) + atexit.register(cleanup) return (c,h) def usage(argv):

1 0

[torflow/master] Ignore existing streams
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 2d4f6fd409fab70dbe7cb35ff4093c46200330a2 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 12:17:34 2011 -0700 Ignore existing streams --- NetworkScanners/BwAuthority/bwauthority_child.py | 13 ++++++++++++- 1 files changed, 12 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index a433c0f..dc777b0 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -22,6 +22,7 @@ import threading import ConfigParser import sqlalchemy import sets +import re sys.path.append("../../") @@ -285,7 +286,7 @@ def main(argv): except Exception, e: traceback.print_exc() plog("WARN", "Can't connect to Tor: "+str(e)) - + if db_url: hdlr.attach_sql_listener(db_url) sql_file = None @@ -320,6 +321,14 @@ def main(argv): # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) sys.exit(0) +def ignore_streams(c,hdlr): + for stream in c.get_info("stream-status")['stream-status'].rstrip("\n").split("\n"): + f = re.match("(?P<sid>\d*)\s(?P<status>\S*)\s(?P<cid>\d*)\s(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>\d{1,5})",stream).groupdict() + s = PathSupport.Stream(int(f['sid']), f['host'], int(f['port']), 0) + plog("DEBUG", "Ignoring foreign stream: %s" % f['sid']) + s.ignored = True + hdlr.streams[s.strm_id] = s + def cleanup(): plog("DEBUG", "Child Process Exiting...") @@ -333,6 +342,8 @@ def setup_handler(out_dir, cookie_file): h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector) + # ignore existing streams + ignore_streams(c,h) c.set_event_handler(h) #c.set_periodic_timer(2.0, "PULSE")

1 0