- tor-commits - lists.torproject.org

[torflow/master] added db_url config parameter
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 83348a9cc59fe035026917b41585f101ea169a73 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 16 19:30:33 2011 -0700 added db_url config parameter --- NetworkScanners/BwAuthority/bwauthority.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index 0d57dfe..bcc9a49 100644 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -14,7 +14,7 @@ def main(argv): TorUtil.read_config(argv[1]) (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, max_fetch_time,tor_dir,sleep_start,sleep_stop, - min_streams,pid_file_name) = bwauthority_child.read_config(argv[1]) + min_streams,pid_file_name,db_url) = bwauthority_child.read_config(argv[1]) if pid_file_name: pidfd = file(pid_file_name, 'w')

1 0

[torflow/master] added db_url to configs
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 28dc35752bbf70ba4b417b4f7490d51a55c127f9 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 16 19:49:38 2011 -0700 added db_url to configs --- .../BwAuthority/data/scanner.1/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.2/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.3/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.4/bwauthority.cfg | 2 ++ 4 files changed, 8 insertions(+), 0 deletions(-) diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg index a90ce97..bd3a0ca 100644 --- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.1/scan-data pid_file = ./data/scanner.1/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct = 0 stop_pct = 12 diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg index 959f1e6..ea2472d 100644 --- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.2/scan-data pid_file = ./data/scanner.2/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct = 12 stop_pct = 35 diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg index 71e1f6f..526a912 100644 --- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.3/scan-data pid_file = ./data/scanner.3/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct =35 stop_pct = 60 diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg index 5a6b500..5a7c7a7 100644 --- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg @@ -12,6 +12,8 @@ meta_port = 9112 [BwAuthority] out_dir = ./data/scanner.4/scan-data pid_file = ./data/scanner.4/bwauthority.pid +# if db_url is unset bwauthority will default to sqlite +db_url = tor_dir = ./data/tor start_pct =60 stop_pct = 100

1 0

[torflow/master] Removed conflicting tor configuration options
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 77ed1ab8de9a264f0f885ea266227894c47056e0 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 10:44:42 2011 -0700 Removed conflicting tor configuration options __LeaveStreamsUnattached and FetchUselessDescriptors should now be set in torrc rather than by bwauthority to avoid a race between child processes --- NetworkScanners/BwAuthority/bwauthority_child.py | 17 ++++------------- 1 files changed, 4 insertions(+), 13 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 4d968ac..a433c0f 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -267,6 +267,7 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir, #hdlr.save_sql_file(sql_file, os.getcwd()+"/"+out_dir+"/bw-db-"+str(lo)+":"+str(hi)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S")+".sqlite") def main(argv): + plog("DEBUG", "Child Process Spawning...") TorUtil.read_config(argv[1]) (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, max_fetch_time,tor_dir,sleep_start,sleep_stop, @@ -319,14 +320,8 @@ def main(argv): # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) sys.exit(0) -def cleanup(c, f): - plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors="+f) - try: - # XXX: Remember __LeaveStreamsUnattached and use saved value! - c.set_option("__LeaveStreamsUnattached", "0") - c.set_option("FetchUselessDescriptors", f) - except TorCtl.TorCtlClosed: - pass +def cleanup(): + plog("DEBUG", "Child Process Exiting...") def setup_handler(out_dir, cookie_file): plog('INFO', 'Connecting to Tor at '+TorUtil.control_host+":"+str(TorUtil.control_port)) @@ -335,7 +330,6 @@ def setup_handler(out_dir, cookie_file): c = PathSupport.Connection(s) #c.debug(file(out_dir+"/control.log", "w", buffering=0)) c.authenticate_cookie(file(cookie_file, "r")) - #f = c.get_option("__LeaveStreamsUnattached")[0] h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector) @@ -349,10 +343,7 @@ def setup_handler(out_dir, cookie_file): TorCtl.EVENT_TYPE.CIRC, TorCtl.EVENT_TYPE.STREAM_BW], True) - c.set_option("__LeaveStreamsUnattached", "1") - f = c.get_option("FetchUselessDescriptors")[0][1] - c.set_option("FetchUselessDescriptors", "1") - atexit.register(cleanup, *(c, f)) + atexit.register(cleanup) return (c,h) def usage(argv):

1 0

[torflow/master] Ignore existing streams
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 2d4f6fd409fab70dbe7cb35ff4093c46200330a2 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 12:17:34 2011 -0700 Ignore existing streams --- NetworkScanners/BwAuthority/bwauthority_child.py | 13 ++++++++++++- 1 files changed, 12 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index a433c0f..dc777b0 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -22,6 +22,7 @@ import threading import ConfigParser import sqlalchemy import sets +import re sys.path.append("../../") @@ -285,7 +286,7 @@ def main(argv): except Exception, e: traceback.print_exc() plog("WARN", "Can't connect to Tor: "+str(e)) - + if db_url: hdlr.attach_sql_listener(db_url) sql_file = None @@ -320,6 +321,14 @@ def main(argv): # out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num, sql_file) sys.exit(0) +def ignore_streams(c,hdlr): + for stream in c.get_info("stream-status")['stream-status'].rstrip("\n").split("\n"): + f = re.match("(?P<sid>\d*)\s(?P<status>\S*)\s(?P<cid>\d*)\s(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>\d{1,5})",stream).groupdict() + s = PathSupport.Stream(int(f['sid']), f['host'], int(f['port']), 0) + plog("DEBUG", "Ignoring foreign stream: %s" % f['sid']) + s.ignored = True + hdlr.streams[s.strm_id] = s + def cleanup(): plog("DEBUG", "Child Process Exiting...") @@ -333,6 +342,8 @@ def setup_handler(out_dir, cookie_file): h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector) + # ignore existing streams + ignore_streams(c,h) c.set_event_handler(h) #c.set_periodic_timer(2.0, "PULSE")

1 0

[torflow/master] add sample db_url strings
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 45db027b45d71f926dc59237c1b4d7bdd1e31739 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 16 23:02:42 2011 -0700 add sample db_url strings --- NetworkScanners/BwAuthority/bwauthority_child.py | 1 + .../BwAuthority/data/scanner.1/bwauthority.cfg | 4 +++- .../BwAuthority/data/scanner.2/bwauthority.cfg | 4 +++- .../BwAuthority/data/scanner.3/bwauthority.cfg | 2 ++ .../BwAuthority/data/scanner.4/bwauthority.cfg | 2 ++ 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 6f61c63..4d968ac 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -287,6 +287,7 @@ def main(argv): if db_url: hdlr.attach_sql_listener(db_url) + sql_file = None else: plog("INFO", "db_url not found in config. Defaulting to sqlite") sql_file = os.getcwd()+'/'+out_dir+'/bwauthority.sqlite' diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg index bd3a0ca..8f9394f 100644 --- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg @@ -13,7 +13,9 @@ meta_port = 9112 out_dir = ./data/scanner.1/scan-data pid_file = ./data/scanner.1/bwauthority.pid # if db_url is unset bwauthority will default to sqlite -db_url = +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 tor_dir = ./data/tor start_pct = 0 stop_pct = 12 diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg index ea2472d..42a6a5d 100644 --- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg @@ -13,7 +13,9 @@ meta_port = 9112 out_dir = ./data/scanner.2/scan-data pid_file = ./data/scanner.2/bwauthority.pid # if db_url is unset bwauthority will default to sqlite -db_url = +db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 tor_dir = ./data/tor start_pct = 12 stop_pct = 35 diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg index 526a912..a2ea7ac 100644 --- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg @@ -14,6 +14,8 @@ out_dir = ./data/scanner.3/scan-data pid_file = ./data/scanner.3/bwauthority.pid # if db_url is unset bwauthority will default to sqlite db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 tor_dir = ./data/tor start_pct =35 stop_pct = 60 diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg index 5a7c7a7..20ebe5a 100644 --- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg @@ -14,6 +14,8 @@ out_dir = ./data/scanner.4/scan-data pid_file = ./data/scanner.4/bwauthority.pid # if db_url is unset bwauthority will default to sqlite db_url = +#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1 +#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1 tor_dir = ./data/tor start_pct =60 stop_pct = 100

1 0

[torflow/master] oops, missing offset
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 6c66500dcee8c53a75881514f7558e21d146f26c Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 16 20:16:32 2011 -0700 oops, missing offset --- NetworkScanners/BwAuthority/bwauthority_child.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 60f7a09..6f61c63 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -301,7 +301,7 @@ def main(argv): pct_step = hdlr.rank_to_percent(nodes_per_slice) # check to see if we are done - if (slice_num * pct_step > stop_pct): + if (slice_num * pct_step + start_pct > stop_pct): sys.exit(STOP_PCT_REACHED) speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, save_every, out_dir,

1 0

[torflow/master] added log entry when scan completes
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 86a70ea65da0a82ef40fb06e1ad3090ec199bd07 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 13:39:16 2011 -0700 added log entry when scan completes --- NetworkScanners/BwAuthority/bwauthority.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index bcc9a49..52385f0 100644 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -30,6 +30,7 @@ def main(argv): if (p.returncode == 0): slice_num += 1 elif (p.returncode == bwauthority_child.STOP_PCT_REACHED): + plog('INFO', 'stop_pct reached, restarting from slice 0') slice_num = 0 else: plog('WARN', 'Child process returned %s' % p.returncode)

1 0

[torflow/master] Log when stop_pct reached
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit f7940f720b324e2823a142a2ee49eb01f826a7f5 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 16:41:05 2011 -0700 Log when stop_pct reached --- NetworkScanners/BwAuthority/bwauthority.py | 2 +- NetworkScanners/BwAuthority/bwauthority_child.py | 1 + 2 files changed, 2 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index 52385f0..ad53663 100644 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -30,7 +30,7 @@ def main(argv): if (p.returncode == 0): slice_num += 1 elif (p.returncode == bwauthority_child.STOP_PCT_REACHED): - plog('INFO', 'stop_pct reached, restarting from slice 0') + plog('INFO', 'restarting from slice 0') slice_num = 0 else: plog('WARN', 'Child process returned %s' % p.returncode) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index ee87829..0de7998 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -305,6 +305,7 @@ def main(argv): # check to see if we are done if (slice_num * pct_step + start_pct > stop_pct): + plog('INFO', 'stop_pct: %s reached. Exiting with %s' % (stop_pct, STOP_PCT_REACHED)) sys.exit(STOP_PCT_REACHED) speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node, save_every, out_dir,

1 0

[torflow/master] Case for NO streams added
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 7a44c16b4b9fdb2603bfed0f9e0a691c8b8dd304 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 12:20:50 2011 -0700 Case for NO streams added --- NetworkScanners/BwAuthority/bwauthority_child.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index dc777b0..ee87829 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -323,7 +323,11 @@ def main(argv): def ignore_streams(c,hdlr): for stream in c.get_info("stream-status")['stream-status'].rstrip("\n").split("\n"): - f = re.match("(?P<sid>\d*)\s(?P<status>\S*)\s(?P<cid>\d*)\s(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>\d{1,5})",stream).groupdict() + m = re.match("(?P<sid>\d*)\s(?P<status>\S*)\s(?P<cid>\d*)\s(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>\d{1,5})",stream) + if m: + f = m.groupdict() + else: + return # no streams s = PathSupport.Stream(int(f['sid']), f['host'], int(f['port']), 0) plog("DEBUG", "Ignoring foreign stream: %s" % f['sid']) s.ignored = True

1 0

[torflow/master] pruned unused code
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 40c5d11015ef2a8dab4bef93b8a256538059e0e4 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 18:48:42 2011 -0700 pruned unused code --- NetworkScanners/BwAuthority/bwauthority.py | 17 +++++------------ NetworkScanners/BwAuthority/bwauthority_child.py | 1 + 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority.py b/NetworkScanners/BwAuthority/bwauthority.py index ad53663..cee2044 100644 --- a/NetworkScanners/BwAuthority/bwauthority.py +++ b/NetworkScanners/BwAuthority/bwauthority.py @@ -7,20 +7,13 @@ import traceback sys.path.append("../../") from TorCtl import TorUtil from TorCtl.TorUtil import plog -import bwauthority_child -def main(argv): - TorUtil.read_config(argv[1]) - (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir, - max_fetch_time,tor_dir,sleep_start,sleep_stop, - min_streams,pid_file_name,db_url) = bwauthority_child.read_config(argv[1]) - - if pid_file_name: - pidfd = file(pid_file_name, 'w') - pidfd.write('%d\n' % os.getpid()) - pidfd.close() +# exit code to indicate scan completion +# make sure to update this in bwauthority_child.py as well +STOP_PCT_REACHED = -9 +def main(argv): slice_num = 0 while True: plog('INFO', 'Beginning time loop') @@ -29,7 +22,7 @@ def main(argv): p.wait() if (p.returncode == 0): slice_num += 1 - elif (p.returncode == bwauthority_child.STOP_PCT_REACHED): + elif (p.returncode == STOP_PCT_REACHED): plog('INFO', 'restarting from slice 0') slice_num = 0 else: diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 0de7998..4b09f9f 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -75,6 +75,7 @@ __selmgr = PathSupport.SelectionManager( exit_ports=[443]) # exit code to indicate scan completion +# make sure to update this in bwauthority.py as well STOP_PCT_REACHED = -9 def read_config(filename):

1 0