- tor-commits - lists.torproject.org

[torflow/master] Update submodule.
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit cb4db2317437621a22531e84b8686fc28785f2bf Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 24 12:29:28 2011 -0700 Update submodule. --- TorCtl | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/TorCtl b/TorCtl index af306b5..d0f593f 160000 --- a/TorCtl +++ b/TorCtl @@ -1 +1 @@ -Subproject commit af306b53a668168ae92a6c919d4013b81d7e61ad +Subproject commit d0f593f2ae245e951323ff7fc3a65fa6a1f16b47

1 0

[torflow/master] Update bw file percentages again; remove python 2.5ism.
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 5cd1f2be2e370176bf13ce44ad2033367cba032a Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 24 12:28:00 2011 -0700 Update bw file percentages again; remove python 2.5ism. --- NetworkScanners/BwAuthority/bwauthority_child.py | 17 ++++++++++------- 1 files changed, 10 insertions(+), 7 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index e8a0a68..c342f2e 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -50,13 +50,14 @@ user_agent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.37 # TODO: We'll also want to try to prefer pairing unmeasured nodes # together then, and use a different url set for them. # cutoff percent URL -urls = [(5, "https://38.229.70.2/16M"), # fbw 1499k..500k - (10, "https://38.229.70.2/8M"), # fbw 500k..350k - (20, "https://38.229.70.2/4M"), # fbw 350k..200k - (30, "https://38.229.70.2/2M"), # fbw 200k..128k - (50, "https://38.229.70.2/512k"), # fbw 128k..50k - (80, "https://38.229.70.2/256k"), # fbw 50k..26k - (100, "https://38.229.70.2/128k")] # fbw 26k..10k +urls = [(5, "https://38.229.70.2/8M"), # fbw 2000k..451k + (10, "https://38.229.70.2/4M"), # fbw 451k..275k + (20, "https://38.229.70.2/2M"), # fbw 275k..150k + (30, "https://38.229.70.2/1M"), # fbw 150k..100k + (40, "https://38.229.70.2/1M"), # fbw 100k..75k + (50, "https://38.229.70.2/512k"), # fbw 75k..50k + (80, "https://38.229.70.2/256k"), # fbw 50k..25k + (100, "https://38.229.70.2/128k")] # fbw 25k..10k # Do NOT modify this object directly after it is handed to PathBuilder @@ -376,6 +377,8 @@ if __name__ == '__main__': except KeyboardInterrupt: plog('INFO', "Ctrl + C was pressed. Exiting ... ") traceback.print_exc() + except SystemExit, e: + sys.exit(e) except Exception, e: plog('ERROR', "An unexpected error occured.") traceback.print_exc()

1 0

[torflow/master] Update submodule..
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 9f756249cf53985a944cb27bf7101d4ff7f369c3 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 24 12:33:36 2011 -0700 Update submodule.. --- TorCtl | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/TorCtl b/TorCtl index d0f593f..dc1cb9c 160000 --- a/TorCtl +++ b/TorCtl @@ -1 +1 @@ -Subproject commit d0f593f2ae245e951323ff7fc3a65fa6a1f16b47 +Subproject commit dc1cb9ce39fd5c9c5ea50f542642130a44e71fee

1 0

[torflow/master] bwauthority parent should exit when child is killed
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 0e32c07900b8722d67b338b00fd0c79d746b16a3 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 23 11:39:58 2011 -0700 bwauthority parent should exit when child is killed --- NetworkScanners/BwAuthority/bwauthority.py | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index 12eeab0..9125044 100755 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -5,7 +5,7 @@ from sys import exit from subprocess import Popen path.append("../../") from TorCtl.TorUtil import plog as plog -from signal import signal, SIGTERM +from signal import signal, SIGTERM, SIGKILL # exit code to indicate scan completion @@ -24,6 +24,12 @@ def main(argv): elif (p.returncode == STOP_PCT_REACHED): plog('INFO', 'restarting from slice 0') slice_num = 0 + elif (abs(p.returncode) == SIGKILL): + plog('WARN', 'Child process recieved SIGKILL, exiting') + exit() + elif (abs(p.returncode) == SIGTERM): + plog('WARN', 'Child process recieved SIGTERM, exiting') + exit() else: plog('WARN', 'Child process returned %s' % p.returncode)

1 0

[torflow/master] added db_url config parameter
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 83348a9cc59fe035026917b41585f101ea169a73 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 16 19:30:33 2011 -0700 added db_url config parameter --- NetworkScanners/BwAuthority/bwauthority.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index 0d57dfe..bcc9a49 100644 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -14,7 +14,7 @@ def main(argv): TorUtil.read_config(argv[1]) (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, max_fetch_time,tor_dir,sleep_start,sleep_stop, - min_streams,pid_file_name) = bwauthority_child.read_config(argv[1]) + min_streams,pid_file_name,db_url) = bwauthority_child.read_config(argv[1]) if pid_file_name: pidfd = file(pid_file_name, 'w')

1 0

[torflow/master] added db_url to configs
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 28dc35752bbf70ba4b417b4f7490d51a55c127f9 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 16 19:49:38 2011 -0700 added db_url to configs --- .../BwAuthority/data/scanner.1/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.2/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.3/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.4/bwauthority.cfg | 2 ++ 4 files changed, 8 insertions(+), 0 deletions(-) diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg index a90ce97..bd3a0ca 100644 --- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.1/scan-data pid_file = ./data/scanner.1/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct = 0 stop_pct = 12 diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg index 959f1e6..ea2472d 100644 --- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.2/scan-data pid_file = ./data/scanner.2/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct = 12 stop_pct = 35 diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg index 71e1f6f..526a912 100644 --- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.3/scan-data pid_file = ./data/scanner.3/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct =35 stop_pct = 60 diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg index 5a6b500..5a7c7a7 100644 --- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.4/scan-data pid_file = ./data/scanner.4/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct =60 stop_pct = 100

1 0

[torflow/master] Removed conflicting tor configuration options
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 77ed1ab8de9a264f0f885ea266227894c47056e0 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 10:44:42 2011 -0700 Removed conflicting tor configuration options __LeaveStreamsUnattached and FetchUselessDescriptors should now be set in torrc rather than by bwauthority to avoid a race between child processes --- NetworkScanners/BwAuthority/bwauthority_child.py | 17 ++++------------- 1 files changed, 4 insertions(+), 13 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 4d968ac..a433c0f 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -267,6 +267,7 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir, #hdlr.save_sql_file(sql_file, os.getcwd()+"/"+out_dir+"/bw-db-"+str(lo)+":"+str(hi)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S")+".sqlite") def main(argv): + plog("DEBUG", "Child Process Spawning...") TorUtil.read_config(argv[1]) (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, max_fetch_time,tor_dir,sleep_start,sleep_stop, @@ -319,14 +320,8 @@ def main(argv): # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) sys.exit(0) -def cleanup(c, f): - plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors="+f) - try: - # XXX: Remember __LeaveStreamsUnattached and use saved value! - c.set_option("__LeaveStreamsUnattached", "0") - c.set_option("FetchUselessDescriptors", f) - except TorCtl.TorCtlClosed: - pass +def cleanup(): + plog("DEBUG", "Child Process Exiting...") def setup_handler(out_dir, cookie_file): plog('INFO', 'Connecting to Tor at '+TorUtil.control_host+":"+str(TorUtil.control_port)) @@ -335,7 +330,6 @@ def setup_handler(out_dir, cookie_file): c = PathSupport.Connection(s) #c.debug(file(out_dir+"/control.log", "w", buffering=0)) c.authenticate_cookie(file(cookie_file, "r")) - #f = c.get_option("__LeaveStreamsUnattached")[0] h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector) @@ -349,10 +343,7 @@ def setup_handler(out_dir, cookie_file): TorCtl.EVENT_TYPE.CIRC, TorCtl.EVENT_TYPE.STREAM_BW], True) - c.set_option("__LeaveStreamsUnattached", "1") - f = c.get_option("FetchUselessDescriptors")[0][1] - c.set_option("FetchUselessDescriptors", "1") - atexit.register(cleanup, *(c, f)) + atexit.register(cleanup) return (c,h) def usage(argv):

1 0

[torflow/master] Ignore existing streams
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 2d4f6fd409fab70dbe7cb35ff4093c46200330a2 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 12:17:34 2011 -0700 Ignore existing streams --- NetworkScanners/BwAuthority/bwauthority_child.py | 13 ++++++++++++- 1 files changed, 12 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index a433c0f..dc777b0 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -22,6 +22,7 @@ import threading import ConfigParser import sqlalchemy import sets +import re sys.path.append("../../") @@ -285,7 +286,7 @@ def main(argv): except Exception, e: traceback.print_exc() plog("WARN", "Can't connect to Tor: "+str(e)) - + if db_url: hdlr.attach_sql_listener(db_url) sql_file = None @@ -320,6 +321,14 @@ def main(argv): # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) sys.exit(0) +def ignore_streams(c,hdlr): + for stream in c.get_info("stream-status")['stream-status'].rstrip("\n").split("\n"): + f = re.match("(?P<sid>\d*)\s(?P<status>\S*)\s(?P<cid>\d*)\s(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>\d{1,5})",stream).groupdict() + s = PathSupport.Stream(int(f['sid']), f['host'], int(f['port']), 0) + plog("DEBUG", "Ignoring foreign stream: %s" % f['sid']) + s.ignored = True + hdlr.streams[s.strm_id] = s + def cleanup(): plog("DEBUG", "Child Process Exiting...") @@ -333,6 +342,8 @@ def setup_handler(out_dir, cookie_file): h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector) + # ignore existing streams + ignore_streams(c,h) c.set_event_handler(h) #c.set_periodic_timer(2.0, "PULSE")

1 0

[torflow/master] add sample db_url strings
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 45db027b45d71f926dc59237c1b4d7bdd1e31739 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 16 23:02:42 2011 -0700 add sample db_url strings --- NetworkScanners/BwAuthority/bwauthority_child.py | 1 + .../BwAuthority/data/scanner.1/bwauthority.cfg | 4 +++- .../BwAuthority/data/scanner.2/bwauthority.cfg | 4 +++- .../BwAuthority/data/scanner.3/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.4/bwauthority.cfg | 2 ++ 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 6f61c63..4d968ac 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -287,6 +287,7 @@ def main(argv): if db_url: hdlr.attach_sql_listener(db_url) + sql_file = None else: plog("INFO", "db_url not found in config. Defaulting to sqlite") sql_file = os.getcwd()+'/'+out_dir+'/bwauthority.sqlite' diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg index bd3a0ca..8f9394f 100644 --- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg @@ -13,7 +13,9 @@ meta_port = 9112 out_dir = ./data/scanner.1/scan-data pid_file = ./data/scanner.1/bwauthority.pid # if db_url is unset bwauthority will default to sqlite -db_url = +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 tor_dir = ./data/tor start_pct = 0 stop_pct = 12 diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg index ea2472d..42a6a5d 100644 --- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg @@ -13,7 +13,9 @@ meta_port = 9112 out_dir = ./data/scanner.2/scan-data pid_file = ./data/scanner.2/bwauthority.pid # if db_url is unset bwauthority will default to sqlite -db_url = +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 tor_dir = ./data/tor start_pct = 12 stop_pct = 35 diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg index 526a912..a2ea7ac 100644 --- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg @@ -14,6 +14,8 @@ out_dir = ./data/scanner.3/scan-data pid_file = ./data/scanner.3/bwauthority.pid # if db_url is unset bwauthority will default to sqlite db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 tor_dir = ./data/tor start_pct =35 stop_pct = 60 diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg index 5a7c7a7..20ebe5a 100644 --- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg @@ -14,6 +14,8 @@ out_dir = ./data/scanner.4/scan-data pid_file = ./data/scanner.4/bwauthority.pid # if db_url is unset bwauthority will default to sqlite db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 tor_dir = ./data/tor start_pct =60 stop_pct = 100

1 0

[torflow/master] oops, missing offset
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 6c66500dcee8c53a75881514f7558e21d146f26c Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 16 20:16:32 2011 -0700 oops, missing offset --- NetworkScanners/BwAuthority/bwauthority_child.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 60f7a09..6f61c63 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -301,7 +301,7 @@ def main(argv): pct_step = hdlr.rank_to_percent(nodes_per_slice) # check to see if we are done - if (slice_num * pct_step > stop_pct): + if (slice_num * pct_step + start_pct > stop_pct): sys.exit(STOP_PCT_REACHED) speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, save_every, out_dir,

1 0