October 2011 - tor-commits - lists.torproject.org

[stem/master] Running tor instances for integ tests
by atagar＠torproject.org 16 Oct '11

16 Oct '11

commit bdbf352ad7f56aec17ce0156eaf53c9a6ac1fe68 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Oct 16 01:17:38 2011 -0700 Running tor instances for integ tests First draft for the startup/shutdown of integration tests. This... - makes a test directory - generates a torrc - starts a tor instance, waiting until its bootstrap completes (timing out if it gets stuck) - runs tests (not done yet - those are next) - shuts down the tor instance This'll certainly go through some refactoring to better modularize, but it's functional as-is. --- run_tests.py | 143 +++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 files changed, 123 insertions(+), 20 deletions(-) diff --git a/run_tests.py b/run_tests.py index 78f392d..9e4487c 100755 --- a/run_tests.py +++ b/run_tests.py @@ -4,13 +4,18 @@ Runs unit and integration tests. """ +import os import sys +import time import getopt +import signal +import tempfile import unittest +import subprocess import test.unit.message import test.unit.version -from stem.util import enum, term +from stem.util import enum, system, term OPT = "uit:h" OPT_EXPANDED = ["unit", "integ", "targets=", "help"] @@ -46,6 +51,57 @@ Runs tests for the stem library. %s """ +# Number of seconds before we time out our attempt to start a tor instance +TOR_INIT_TIMEOUT = 20 + +def init_tor_process(torrc_dst): + """ + Initializes and returns a tor process. This blocks until initialization + completes or we error out. + + Arguments: + torrc_dst (str) - path for a torrc configuration to use + + Returns: + subprocess.Popen instance for the instantiated tor process + + Raises: + OSError if we either fail to create the tor process or reached a timeout without success + """ + + start_time = time.time() + + # starts a tor subprocess, raising an OSError if it fails + tor_process = subprocess.Popen(["tor", "-f", torrc_dst], stdout = subprocess.PIPE, stderr = subprocess.PIPE) + + # time ourselves out if we reach TOR_INIT_TIMEOUT + def timeout_handler(signum, frame): + # terminates the uninitialized tor process and raise on timeout + tor_process.kill() + raise OSError("unable to start tor: reached a %i second timeout without success" % TOR_INIT_TIMEOUT) + + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(TOR_INIT_TIMEOUT) + + print term.format("Starting tor...", term.Color.BLUE, term.Attr.BOLD) + + while True: + init_line = tor_process.stdout.readline().strip() + + # this will provide empty results if the process is terminated + if not init_line: + tor_process.kill() # ... but best make sure + raise OSError("tor process terminated") + + print term.format(" %s" % init_line, term.Color.BLUE) + + # return the process if we're done with bootstrapping + if init_line.endswith("Bootstrapped 100%: Done."): + print term.format(" done (%i seconds)" % (time.time() - start_time), term.Color.BLUE, term.Attr.BOLD) + print + + return tor_process + if __name__ == '__main__': run_unit_tests = False run_integ_tests = False @@ -97,35 +153,82 @@ if __name__ == '__main__': for name, test_class in UNIT_TESTS: print "%s\n%s\n%s\n" % (DIVIDER, name, DIVIDER) - #print name suite = unittest.TestLoader().loadTestsFromTestCase(test_class) unittest.TextTestRunner(verbosity=2).run(suite) print - #import test.unit - #suite = unittest.TestLoader().loadTestsFromTestCase(test.unit.version.TestVerionFunctions) - #suite = unittest.TestLoader().discover("test/unit/", "*.py") - #suite.addTests(unittest.loader.loadTestsFromTestCase(test.unit.message.TestMessageFunctions)) - - #suite = unittest.TestLoader() - #suite.loadTestsFromTestCase(test.unit.message.TestMessageFunctions) - #suite.loadTestsFromTestCase(test.unit.version.TestVerionFunctions) - #unittest.TextTestRunner(verbosity=2).run(suite) - print if run_integ_tests: + # TODO: check if there's already a tor instance running + print "%s\n%s\n%s\n" % (DIVIDER, "INTEGRATION TESTS".center(70), DIVIDER) - for target in integ_targets: - runner, description = TARGET_ATTR[target] + print term.format("Setting up a test instance...", term.Color.BLUE, term.Attr.BOLD) + + # makes a temporary directory for the runtime resources of our integ test + test_dir = tempfile.mktemp("-stem-integ") + + try: + os.makedirs(test_dir) + print term.format(" created test directory: %s" % test_dir, term.Color.BLUE, term.Attr.BOLD) + except OSError, exc: + print term.format("Unable to make testing directory: %s" % exc, term.Color.RED, term.Attr.BOLD) + sys.exit(1) + + # makes a basic torrc for the integration tests to run against + torrc_contents = """# basic integration testing configuration +DataDirectory %s +ControlPort 9051 +""" % test_dir + + # writes our testing torrc + torrc_dst = os.path.join(test_dir, "torrc") + try: + torrc_file = open(torrc_dst, "w") + torrc_file.write(torrc_contents) + torrc_file.close() - print "Configuration: %s - %s" % (target, description) + print term.format(" wrote torrc: %s" % torrc_dst, term.Color.BLUE, term.Attr.BOLD) - if runner: - pass # TODO: implement - else: - print " %s" % term.format("Unimplemented", term.Color.RED, term.Attr.BOLD) + for line in torrc_contents.split("\n"): + print term.format(" %s" % line.strip(), term.Color.BLUE) + except Exception, exc: + print term.format("Unable to write testing torrc: %s" % exc, term.Color.RED, term.Attr.BOLD) + sys.exit(1) + + # starts a tor instance + try: + tor_process = init_tor_process(torrc_dst) + except OSError, exc: + print term.format("Unable to start a tor instance: %s" % exc, term.Color.RED, term.Attr.BOLD) + sys.exit(1) - print "" + print term.format("Running tests...", term.Color.BLUE, term.Attr.BOLD) + print + + # TODO: run tests + + print term.format("Shutting down tor...", term.Color.BLUE, term.Attr.BOLD) + tor_process.kill() + print term.format(" done", term.Color.BLUE, term.Attr.BOLD) + + + + + + # TODO: we might do target selection later but for now we should simply + # work with a single simple tor instance and see how it works out + # + #for target in integ_targets: + # runner, description = TARGET_ATTR[target] + # + # print "Configuration: %s - %s" % (target, description) + # + # if runner: + # pass # TODO: implement + # else: + # print " %s" % term.format("Unimplemented", term.Color.RED, term.Attr.BOLD) + # + # print ""

1 0

[torflow/master] Add --help Option
by mikeperry＠torproject.org 16 Oct '11

16 Oct '11

commit 28a9c344d69a571ad5c5205a43f7314781617e47 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Sat Oct 15 23:15:23 2011 +0530 Add --help Option Using --help gives the list of options required for SoaT. --- NetworkScanners/ExitAuthority/soat.py | 41 +++++++++++++++++--------------- 1 files changed, 22 insertions(+), 19 deletions(-) diff --git a/NetworkScanners/ExitAuthority/soat.py b/NetworkScanners/ExitAuthority/soat.py index 2cb2931..f647495 100755 --- a/NetworkScanners/ExitAuthority/soat.py +++ b/NetworkScanners/ExitAuthority/soat.py @@ -85,7 +85,7 @@ except ImportError: from os import getcwd, path print "TorCtl not found in %s. Did you run torflow.git/add_torctl.sh?" % path.abspath(getcwd()+'../..') print "Exiting..." - exit() + exit() sys.path.insert(0,"../libs") # Make our SocksiPy use our socket @@ -622,7 +622,7 @@ class Test: # Yes, this is a hack, and yes, it will bias results # away from the filter, but hey, at least it will still run. self._pickle_revision = 1 - + for addr in self.successes.keys(): if type(self.successes[addr]) == int: self.successes[addr] = set(xrange(0,self.successes[addr])) @@ -1202,7 +1202,7 @@ class BaseHTTPTest(Test): if self.compare(address,loaded_filetype,second_req) != COMPARE_EQUAL: return (address, False, second_req.code, '') - return (address, True, req.code, loaded_filetype) + return (address, True, req.code, loaded_filetype) def check_http(self, address, filetype, dynamic = False): ''' check whether a http connection to a given address is molested ''' @@ -1310,7 +1310,7 @@ class BaseHTTPTest(Test): # Reload direct content and try again new_req = http_request(address, my_cookie_jar, self.headers) sha1sum_new = sha(new_req.content) - + # If a new direct load somehow fails, then we're out of luck if not (200 <= new_req.code < 300): plog("WARN", "Failed to re-frech "+address+" outside of Tor. Did our network fail?") @@ -1322,7 +1322,7 @@ class BaseHTTPTest(Test): # Try our comparison again dynamic = self.compare(address,filetype,new_req) - + if dynamic == COMPARE_EQUAL: # The content has not actually changed, so our exit node is screwing with us. result = HttpTestResult(self.node_map[exit_node[1:]], @@ -1373,7 +1373,7 @@ class BaseHTTPTest(Test): def address_to_failed_prefix(self, address): return http_failed_dir + self._address_to_filename(address) - + def save_compare_data(self, address, filetype, req): context = self. address_to_context(address) @@ -1382,7 +1382,7 @@ class BaseHTTPTest(Test): f.close() lines = req.content.split('\n') - + hashes = [] working_hash = sha() for l in lines: @@ -1407,12 +1407,12 @@ class BaseHTTPTest(Test): context = self. address_to_context(address) new_linelist = req.content.split('\n') - + f = open(context + '.content') old_content = f.read() f.close() - + old_hashes = SnakePickler.load(context + '.hashes') if len(new_linelist) > len(old_hashes): @@ -1465,7 +1465,7 @@ class BaseHTTPTest(Test): context = self.address_to_context(address) f = open(context + '.content') old_content = f.read() - f.close() + f.close() return sha(old_content) # TODO move these somewhere sensible @@ -1589,7 +1589,7 @@ class BaseSSLTest(Test): if ssl_domain.cert_changed: plog("NOTICE", "Fully dynamic certificate host "+address) - result = SSLTestResult("NoExit", "NotStored!", address, ssl_file_name, + result = SSLTestResult("NoExit", "NotStored!", address, ssl_file_name, TEST_INCONCLUSIVE, INCONCLUSIVE_DYNAMICSSL) if self.rescan_nodes: @@ -1714,11 +1714,11 @@ class FixedTargetHTTPTest(FixedTargetTest, BaseHTTPTest): utargets = [t for t in targets if self._is_useable_url(t, ['http'])] FixedTargetTest.__init__(self, utargets) - def get_targets(self): + def get_targets(self): ret = [] for targ in self.fixed_targets: addr, succ, code, ftype = self.first_load(targ, False) - if succ: + if succ: ret.append([addr,ftype]) return ret @@ -2655,19 +2655,19 @@ def usage(): print '--target=<ip or url>' print '--loglevel=<DEBUG|INFO|NOTICE|WARN|ERROR|NONE>' print '' - - + + # main logic def main(argv): # make sure we have something to test for if len(argv) < 2: usage() return - + TorUtil.read_config(data_dir+"/torctl.cfg") - opts = ['ssl','rescan', 'pernode=', 'resume=','http','ssh','smtp','pop','imap','dns','dnsrebind','policies','exit=','target=','loglevel='] - + opts = ['ssl','rescan', 'pernode=', 'resume=','http','ssh','smtp','pop','imap','dns','dnsrebind','policies','exit=','target=','loglevel=','help'] + # make sure the arguments are correct try: flags, trailer = getopt.getopt(argv[1:], [], opts) @@ -2675,7 +2675,7 @@ def main(argv): print msg usage() return - + # get specific test types do_resume = False do_rescan = ('--rescan','') in flags @@ -2691,6 +2691,9 @@ def main(argv): fixed_exits=[] fixed_targets=[] for flag in flags: + if flag[0] == "--help": + usage() + return if flag[0] == "--exit": fixed_exits.append(flag[1]) if flag[0] == "--target":

1 0

[torflow/master] Add --help option for snakeinspector.py
by mikeperry＠torproject.org 16 Oct '11

16 Oct '11

commit fe8acab9265870133bd76b2da9b52c1788461b10 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Sat Oct 15 23:34:32 2011 +0530 Add --help option for snakeinspector.py Use --help to view usage. --- NetworkScanners/ExitAuthority/snakeinspector.py | 13 ++++++++----- 1 files changed, 8 insertions(+), 5 deletions(-) diff --git a/NetworkScanners/ExitAuthority/snakeinspector.py b/NetworkScanners/ExitAuthority/snakeinspector.py index f34826e..608df54 100755 --- a/NetworkScanners/ExitAuthority/snakeinspector.py +++ b/NetworkScanners/ExitAuthority/snakeinspector.py @@ -28,7 +28,7 @@ except ImportError: from os import getcwd, path print "TorCtl not found in %s. Did you run torflow.git/add_torctl.sh?" % path.abspath(getcwd()+'../..') print "Exiting..." - exit() + exit() TorCtl.TorUtil.loglevel="INFO" @@ -36,7 +36,8 @@ if TorCtl.TorUtil.loglevels[TorCtl.TorUtil.loglevel] > TorCtl.TorUtil.loglevels[ # Kill stderr (jsdiffer and exception noise) if our loglevel is above INFO sys.stderr = file("/dev/null", "w") -def usage(argv): +def usage(argv=None): + print "Usage: "+argv[0]+" with 0 or more of the following filters: " print " --dir <datadir>" print " --file <.result file>" @@ -86,17 +87,19 @@ class SIConf(object): def getargs(self, argv): try: - opts,args = getopt.getopt(argv[1:],"d:f:x:r:n:a:b:t:p:o:s:Fmcv", + opts,args = getopt.getopt(argv[1:],"d:f:x:r:n:a:b:t:p:o:s:h:Fmcv", ["dir=", "file=", "exit=", "reason=", "resultfilter=", "proto=", "verbose", "statuscode=", "siterate=", "exitrate=", "sortby=", "noreason=", "after=", "before=", "finishedafter=", "finishedbefore=", "croninterval=", "falsepositives", - "email", "confirmed"]) + "email", "confirmed","help"]) except getopt.GetoptError,err: print str(err) usage(argv) for o,a in opts: - if o == '-d' or o == '--dir': + if o == '-h' or o == '--help': + usage(argv) + elif o == '-d' or o == '--dir': self.use_dir = a elif o == '-f' or o == '--file': self.use_file = a

1 0

[torflow/master] Added usage() function
by mikeperry＠torproject.org 16 Oct '11

16 Oct '11

commit 059298f7cf0d3a96a74b966552ebe06992e8dfee Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Sun Oct 9 16:12:50 2011 +0530 Added usage() function --- NetworkScanners/ExitAuthority/soat.py | 18 ++++++++++-------- 1 files changed, 10 insertions(+), 8 deletions(-) diff --git a/NetworkScanners/ExitAuthority/soat.py b/NetworkScanners/ExitAuthority/soat.py index bafe43f..7e43edb 100755 --- a/NetworkScanners/ExitAuthority/soat.py +++ b/NetworkScanners/ExitAuthority/soat.py @@ -2638,12 +2638,7 @@ def setup_handler(out_dir, cookie_file, fixed_exits=[]): atexit.register(cleanup, *(c, l, f)) return (c,h) - -# main logic -def main(argv): - # make sure we have something to test for - if len(argv) < 2: - print '' +def usage(): print 'Please provide at least one test option:' print '--pernode=<n>' print '--resume=<n>' @@ -2660,8 +2655,15 @@ def main(argv): print '--target=<ip or url>' print '--loglevel=<DEBUG|INFO|NOTICE|WARN|ERROR|NONE>' print '' - return - + + +# main logic +def main(argv): + # make sure we have something to test for + if len(argv) < 2: + usage() + return + TorUtil.read_config(data_dir+"/torctl.cfg") opts = ['ssl','rescan', 'pernode=', 'resume=','http','ssh','smtp','pop','imap','dns','dnsrebind','policies','exit=','target=','loglevel=']

1 0

[torflow/master] Check cmdline arguments
by mikeperry＠torproject.org 16 Oct '11

16 Oct '11

commit b5e88e2785aadbbfe50aae87521a436e8e1a4b32 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Sun Oct 9 18:08:55 2011 +0530 Check cmdline arguments Make sure the commandline arguments are correct before proceding. If not, show error message and correct usage. --- NetworkScanners/ExitAuthority/soat.py | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diff --git a/NetworkScanners/ExitAuthority/soat.py b/NetworkScanners/ExitAuthority/soat.py index 7e43edb..2cb2931 100755 --- a/NetworkScanners/ExitAuthority/soat.py +++ b/NetworkScanners/ExitAuthority/soat.py @@ -2667,7 +2667,14 @@ def main(argv): TorUtil.read_config(data_dir+"/torctl.cfg") opts = ['ssl','rescan', 'pernode=', 'resume=','http','ssh','smtp','pop','imap','dns','dnsrebind','policies','exit=','target=','loglevel='] - flags, trailer = getopt.getopt(argv[1:], [], opts) + + # make sure the arguments are correct + try: + flags, trailer = getopt.getopt(argv[1:], [], opts) + except getopt.error, msg: + print msg + usage() + return # get specific test types do_resume = False @@ -2704,7 +2711,6 @@ def main(argv): plog("ERROR", "Unknown loglevel: "+flag[1]) sys.exit(0) - plog("DEBUG", "Read tor config. Got Socks proxy: "+str(TorUtil.tor_port)) # Make logs go to disk so resumes are less painful

1 0

[translation-tools/master] Enable all translations, add tsum (user manual)
by sebastian＠torproject.org 16 Oct '11

16 Oct '11

commit 307cf2f49e6fee72298af85643e9b6859606255c Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Oct 16 02:53:33 2011 +0200 Enable all translations, add tsum (user manual) --- config | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) diff --git a/config b/config index 5ddf287..855119f 100644 --- a/config +++ b/config @@ -7,8 +7,6 @@ TRANSLATION_REPO=/srv/translation.torproject.org/translations/ TX=/srv/translation.torproject.org/transifex-client/bin/tx -#PROJECTS="bridgedb gettor orbot vidalia vidalia_help vidalia_installer" - -PROJECTS="bridgedb orbot" +PROJECTS="bridgedb gettor orbot tsum vidalia vidalia_help vidalia_installer" PIDFILE=/srv/translation.torproject.org/run/update_translations.pid

1 0

[translation/tsum] Remove the README from the tsum branch
by sebastian＠torproject.org 16 Oct '11

16 Oct '11

commit 54a0c62fd4c4ce22bc636266db1754a0c9a15dee Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Oct 16 02:48:16 2011 +0200 Remove the README from the tsum branch --- README | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/README b/README deleted file mode 100644 index 7ad811e..0000000 --- a/README +++ /dev/null @@ -1 +0,0 @@ -This repository holds all translations, with one branch per project.

1 0

[arm/master] Skip validation warnings for DirReqStatistics
by atagar＠torproject.org 15 Oct '11

15 Oct '11

commit 1f347f33b2e6e2c5f18cb0658b4c5f77e870a280 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Oct 15 16:18:17 2011 -0700 Skip validation warnings for DirReqStatistics In new tor versions the DirReqStatistics is enabled by default and disabled when geoip lookups are unavailable. This casuses arm to warn that the torrc differs from tor's state. Discarding this validation warning instead. Tested by checking that validation didn't complain when I SETCONF this option. Caught by Sebastian. https://trac.torproject.org/projects/tor/ticket/4237 --- src/util/torConfig.py | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diff --git a/src/util/torConfig.py b/src/util/torConfig.py index 51bb656..477ad9e 100644 --- a/src/util/torConfig.py +++ b/src/util/torConfig.py @@ -580,6 +580,14 @@ def validate(contents = None): # checks if any custom options are missing from the torrc for option in customOptions: + # In new versions the 'DirReqStatistics' option is true by default and + # disabled on startup if geoip lookups are unavailable. If this option is + # missing then that's most likely the reason. + # + # https://trac.torproject.org/projects/tor/ticket/4237 + + if option == "DirReqStatistics": continue + if not option in seenOptions: issuesFound.append((None, ValidationError.MISSING, option))

1 0

[arm/master] fix: grammatical error with torrc mismatch message
by atagar＠torproject.org 15 Oct '11

15 Oct '11

commit 38af67c1dbe9e06d0375811b0df1afaa0bffd4a7 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Oct 15 13:43:43 2011 -0700 fix: grammatical error with torrc mismatch message Caught by Sebastian: https://trac.torproject.org/projects/tor/ticket/4237 --- src/util/torConfig.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/util/torConfig.py b/src/util/torConfig.py index 1d54196..51bb656 100644 --- a/src/util/torConfig.py +++ b/src/util/torConfig.py @@ -832,7 +832,7 @@ class Torrc(): log.log(CONFIG["log.torrc.validation.unnecessaryTorrcEntries"], msg) if mismatchLines or missingOptions: - msg = "The torrc differ from what tor's using. You can issue a sighup to reload the torrc values by pressing x." + msg = "The torrc differs from what tor's using. You can issue a sighup to reload the torrc values by pressing x." if mismatchLines: if len(mismatchLines) > 1:

1 0

[arm/master] Adding OSX support for the getPwd utility
by atagar＠torproject.org 15 Oct '11

15 Oct '11

commit 92ac1930143e51689396563877087d1070a246bd Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Oct 15 14:03:32 2011 -0700 Adding OSX support for the getPwd utility BSD platforms lack either pwdx or proc contents with this information. However, this is available via lsof. Using this for BSD platforms. This was tested by running sysTools.getPwd() on OSX - hopefully it should work on Free/OpenBSD too. Fix is thanks to Sebastian. https://trac.torproject.org/projects/tor/ticket/4236 --- src/util/sysTools.py | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diff --git a/src/util/sysTools.py b/src/util/sysTools.py index 98361e5..4bb0667 100644 --- a/src/util/sysTools.py +++ b/src/util/sysTools.py @@ -174,6 +174,23 @@ def getPwd(pid): PWD_CACHE[pid] = pwd return pwd except IOError: pass # fall back to pwdx + elif os.uname()[0] in ("Darwin", "FreeBSD", "OpenBSD"): + # BSD neither useres the above proc info nor does it have pwdx. Use lsof to + # determine this instead: + # https://trac.torproject.org/projects/tor/ticket/4236 + # + # ~$ lsof -a -p 75717 -d cwd -Fn + # p75717 + # n/Users/atagar/tor/src/or + + try: + results = call("lsof -a -p %s -d cwd -Fn" % pid) + + if results and len(results) == 2 and results[1].startswith("n/"): + pwd = results[1][1:].strip() + PWD_CACHE[pid] = pwd + return pwd + except IOError, exc: pass try: # pwdx results are of the form:

1 0