October 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Continue work on trial based nettest Network Unit Testing framework
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 712f665423f24701c93889d32d040eb533065dfe Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Sep 15 14:54:51 2012 +0200 Continue work on trial based nettest Network Unit Testing framework * Implement InputUnit and InputUnitFactory --- docs/design.dia | Bin 1706 -> 1706 bytes ooni/input.py | 62 +++ ooni/nettest.py | 45 ++- ooni/runner.py | 458 ++++++++++++++++---- 4 files changed, 472 insertions(+), 93 deletions(-) diff --git a/bin/oonib b/bin/oonib old mode 100755 new mode 100644 diff --git a/bin/ooniprobe b/bin/ooniprobe old mode 100755 new mode 100644 diff --git a/docs/design.dia b/docs/design.dia old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/bin/ooni-probe b/old-to-be-ported-code/bin/ooni-probe old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/captive_portal.py b/old-to-be-ported-code/ooni/captive_portal.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/dns_cc_check.py b/old-to-be-ported-code/ooni/dns_cc_check.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/dns_poisoning.py b/old-to-be-ported-code/ooni/dns_poisoning.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/namecheck.py b/old-to-be-ported-code/ooni/namecheck.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py b/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/netalyzr_plgoo.py b/old-to-be-ported-code/ooni/plugins/netalyzr_plgoo.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/connectback.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/connectback.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/dirconntest.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/dirconntest.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/generic-host-test.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/generic-host-test.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/host-prep.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/host-prep.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/install-probe.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/install-probe.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/run-tests.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/run-tests.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/twitter-test.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/twitter-test.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/simple_dns_plgoo.py b/old-to-be-ported-code/ooni/plugins/simple_dns_plgoo.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/skel_plgoo.py b/old-to-be-ported-code/ooni/plugins/skel_plgoo.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugooni.py b/old-to-be-ported-code/ooni/plugooni.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/transparenthttp.py b/old-to-be-ported-code/ooni/transparenthttp.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/yamlooni.py b/old-to-be-ported-code/ooni/yamlooni.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/proxy-lists/parse-trusted-xff.sh b/old-to-be-ported-code/proxy-lists/parse-trusted-xff.sh old mode 100755 new mode 100644 diff --git a/ooni/input.py b/ooni/input.py new file mode 100644 index 0000000..f534393 --- /dev/null +++ b/ooni/input.py @@ -0,0 +1,62 @@ +class InputUnitFactory(object): + """ + This is a factory that takes the size of input units to be generated a set + of units that is a python iterable item and outputs InputUnit objects + containing inputUnitSize elements. + + This object is a python iterable, this means that it does not need to keep + all the elements in memory to be able to produce InputUnits. + """ + inputUnitSize = 3 + def __init__(self, inputs=[]): + self._inputs = inputs + self._idx = 0 + self._ended = False + + def __iter__(self): + return self + + def next(self): + if self._ended: + raise StopIteration + + last_element_idx = self._idx + self.inputUnitSize + input_unit_elements = self._inputs[self._idx:last_element_idx] + try: + # XXX hack to fail when we reach the end of the list + antani = self._inputs[last_element_idx] + except: + if len(input_unit_elements) > 0: + self._ended = True + return InputUnit(input_unit_elements) + else: + raise StopIteration + + self._idx += self.inputUnitSize + + return InputUnit(input_unit_elements) + + +class InputUnit(object): + """ + This is a python iterable object that contains the input elements to be + passed onto a TestCase. + """ + def __init__(self, inputs=[]): + self._inputs = inputs + + def __repr__(self): + return "<%s inputs=%s>" % (self.__class__, self._inputs) + + def __add__(self, inputs): + for input in inputs: + self._inputs.append(input) + + def __iter__(self): + return iter(self._inputs) + + def append(self, input): + self._inputs.append(input) + + + diff --git a/ooni/nettest.py b/ooni/nettest.py index fe8c05c..ab009b1 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -11,13 +11,7 @@ def _iterateTests(testSuiteOrCase): try: suite = iter(testSuiteOrCase) except TypeError: - if not testSuiteOrCase.inputs: - yield testSuiteOrCase - else: - inputs = iter(testSuiteOrCase.inputs) - print "Detected Sub shit! %s" % inputs - for input in inputs: - yield testSuiteOrCase, input + yield testSuiteOrCase else: for test in suite: for subtest in _iterateTests(test): @@ -28,7 +22,7 @@ class TestCase(unittest.TestCase): """ A test case represents the minimum """ - def run(self, result, input): + def run(self, result): """ Run the test case, storing the results in C{result}. @@ -77,21 +71,38 @@ class TestSuite(pyunit.TestSuite): pattern and a consistently overrideable C{run} method. """ - def __call__(self, result, input): - return self.run(result, input) + def __init__(self, tests=(), inputs=()): + self._tests = [] + self._inputs = [] + self.addTests(tests, inputs) + print "Adding %s %s" % (tests, inputs) + + + def __call__(self, result): + return self.run(result) + def __repr__(self): + return "<%s input=%s tests=%s>" % (self.__class__, + self._inputs, list(self)) - def run(self, result, input): + def run(self, result, input=None): """ Call C{run} on every member of the suite. """ - # we implement this because Python 2.3 unittest defines this code - # in __call__, whereas 2.4 defines the code in run. for test in self._tests: if result.shouldStop: break - print test - print "----------------" - test(result, input) - return result + return test(result, None) + + def addTests(self, tests, inputs=[]): + if isinstance(tests, basestring): + raise TypeError("tests must be and iterable of tests not a string") + for test in tests: + self.addTest(test, inputs) + + def addTest(self, test, inputs=[]): + #print "Adding: %s" % test + super(TestSuite, self).addTest(test) + self._inputs = inputs + diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py old mode 100755 new mode 100644 diff --git a/ooni/runner.py b/ooni/runner.py index c6ad90b..a8485af 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -3,6 +3,7 @@ import time import inspect from twisted.internet import defer +from twisted.python import reflect from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader from twisted.trial.runner import isPackage, isTestCase @@ -39,7 +40,7 @@ class LoggedSuite(nettest.TestSuite): object. """ - def run(self, result, input): + def run(self, result): """ Run the suite, storing all errors in C{result}. If an error is logged while no tests are running, then it will be added as an error to @@ -49,7 +50,7 @@ class LoggedSuite(nettest.TestSuite): """ observer = unittest._logObserver observer._add() - super(LoggedSuite, self).run(result, input) + super(LoggedSuite, self).run(result) observer._remove() for error in observer.getErrors(): result.addError(TestHolder(NOT_IN_TEST), error) @@ -79,100 +80,405 @@ class OONISuite(nettest.TestSuite): # so that the shutdown event completes nettest.TestCase('mktemp')._wait(d) - def run(self, result, input): + def run(self, result): try: - nettest.TestSuite.run(self, result, input) + nettest.TestSuite.run(self, result) finally: self._bail() -class OONIRunner(TrialRunner): - def run(self, test): - return TrialRunner.run(self, test) - - def _runWithoutDecoration(self, test): - """ - Private helper that runs the given test but doesn't decorate it. - """ - result = self._makeResult() - # decorate the suite with reactor cleanup and log starting - # This should move out of the runner and be presumed to be - # present - suite = OONISuite([test]) - print "HERE IS THE TEST:" - print test - print "-------------" - try: - inputs = test.inputs - except: - inputs = [None] - - startTime = time.time() - if self.mode == self.DRY_RUN: - for single in nettest._iterateTests(suite): - input = None - if type(single) == type(tuple()): - single, input = single - result.startTest(single, input) - result.addSuccess(single) - result.stopTest(single) - else: - if self.mode == self.DEBUG: - # open question - should this be self.debug() instead. - debugger = self._getDebugger() - run = lambda x: debugger.runcall(suite.run, result, x) - else: - run = lambda x: suite.run(result, x) - - oldDir = self._setUpTestdir() - try: - self._setUpLogFile() - # XXX work on this better - for input in inputs: - run(input) - finally: - self._tearDownLogFile() - self._tearDownTestdir(oldDir) - - endTime = time.time() - done = getattr(result, 'done', None) - if done is None: - warnings.warn( - "%s should implement done() but doesn't. Falling back to " - "printErrors() and friends." % reflect.qual(result.__class__), - category=DeprecationWarning, stacklevel=3) - result.printErrors() - result.writeln(result.separator) - result.writeln('Ran %d tests in %.3fs', result.testsRun, - endTime - startTime) - result.write('\n') - result.printSummary() - else: - result.done() - return result - - -class TestLoader(TestLoader): +class NetTestLoader(object): """ Reponsible for finding the modules that can work as tests and running them. If we detect that a certain test is written using the legacy OONI API we will wrap it around a next gen class to make it work here too. """ + methodPrefix = 'test' + modulePrefix = 'test_' + def __init__(self): - super(TestLoader, self).__init__() self.suiteFactory = nettest.TestSuite + self.sorter = name + self._importErrors = [] + + def sort(self, xs): + """ + Sort the given things using L{sorter}. + + @param xs: A list of test cases, class or modules. + """ + return sorted(xs, key=self.sorter) + def findTestClasses(self, module): classes = [] for name, val in inspect.getmembers(module): + try: + inputs = val.inputs + except: + inputs = None if isTestCase(val): - classes.append(val) + classes.append((val, inputs)) # This is here to allow backward compatibility with legacy OONI # tests. elif isLegacyTest(val): #val = adaptLegacyTest(val) - classes.append(val) - return self.sort(classes) - #return runner.TestLoader.findTestClasses(self, module) + classes.append((val, inputs)) + return classes + + def findByName(self, name): + """ + Return a Python object given a string describing it. + @param name: a string which may be either a filename or a + fully-qualified Python name. + @return: If C{name} is a filename, return the module. If C{name} is a + fully-qualified Python name, return the object it refers to. + """ + if os.path.exists(name): + return filenameToModule(name) + return reflect.namedAny(name) + + + def loadModule(self, module): + """ + Return a test suite with all the tests from a module. + + Included are TestCase subclasses and doctests listed in the module's + __doctests__ module. If that's not good for you, put a function named + either C{testSuite} or C{test_suite} in your module that returns a + TestSuite, and I'll use the results of that instead. + + If C{testSuite} and C{test_suite} are both present, then I'll use + C{testSuite}. + """ + ## XXX - should I add an optional parameter to disable the check for + ## a custom suite. + ## OR, should I add another method + if not isinstance(module, types.ModuleType): + raise TypeError("%r is not a module" % (module,)) + if hasattr(module, 'testSuite'): + return module.testSuite() + elif hasattr(module, 'test_suite'): + return module.test_suite() + + suite = self.suiteFactory() + for testClass, inputs in self.findTestClasses(module): + testCases = self.loadClass(testClass) + + return testCases + loadTestsFromModule = loadModule + + def loadClass(self, klass): + """ + Given a class which contains test cases, return a sorted list of + C{TestCase} instances. + """ + if not (isinstance(klass, type) or isinstance(klass, types.ClassType)): + raise TypeError("%r is not a class" % (klass,)) + if not isTestCase(klass): + raise ValueError("%r is not a test case" % (klass,)) + names = self.getTestCaseNames(klass) + print "Names %s" % names + tests = self.sort([self._makeCase(klass, self.methodPrefix+name) + for name in names]) + print "Tests %s" % tests + suite = self.suiteFactory(tests) + print "Suite: %s" % suite + return suite + loadTestsFromTestCase = loadClass + + def getTestCaseNames(self, klass): + """ + Given a class that contains C{TestCase}s, return a list of names of + methods that probably contain tests. + """ + return reflect.prefixedMethodNames(klass, self.methodPrefix) + + def loadMethod(self, method): + """ + Given a method of a C{TestCase} that represents a test, return a + C{TestCase} instance for that test. + """ + if not isinstance(method, types.MethodType): + raise TypeError("%r not a method" % (method,)) + return self._makeCase(method.im_class, _getMethodNameInClass(method)) + + def _makeCase(self, klass, methodName): + return klass(methodName) + + def loadPackage(self, package, recurse=False): + """ + Load tests from a module object representing a package, and return a + TestSuite containing those tests. + + Tests are only loaded from modules whose name begins with 'test_' + (or whatever C{modulePrefix} is set to). + + @param package: a types.ModuleType object (or reasonable facsimilie + obtained by importing) which may contain tests. + + @param recurse: A boolean. If True, inspect modules within packages + within the given package (and so on), otherwise, only inspect modules + in the package itself. + + @raise: TypeError if 'package' is not a package. + + @return: a TestSuite created with my suiteFactory, containing all the + tests. + """ + if not isPackage(package): + raise TypeError("%r is not a package" % (package,)) + pkgobj = modules.getModule(package.__name__) + if recurse: + discovery = pkgobj.walkModules() + else: + discovery = pkgobj.iterModules() + discovered = [] + for disco in discovery: + if disco.name.split(".")[-1].startswith(self.modulePrefix): + discovered.append(disco) + suite = self.suiteFactory() + for modinfo in self.sort(discovered): + try: + module = modinfo.load() + except: + thingToAdd = ErrorHolder(modinfo.name, failure.Failure()) + else: + thingToAdd = self.loadModule(module) + suite.addTest(thingToAdd) + return suite + + def loadDoctests(self, module): + """ + Return a suite of tests for all the doctests defined in C{module}. + + @param module: A module object or a module name. + """ + if isinstance(module, str): + try: + module = reflect.namedAny(module) + except: + return ErrorHolder(module, failure.Failure()) + if not inspect.ismodule(module): + warnings.warn("trial only supports doctesting modules") + return + extraArgs = {} + if sys.version_info > (2, 4): + # Work around Python issue2604: DocTestCase.tearDown clobbers globs + def saveGlobals(test): + """ + Save C{test.globs} and replace it with a copy so that if + necessary, the original will be available for the next test + run. + """ + test._savedGlobals = getattr(test, '_savedGlobals', test.globs) + test.globs = test._savedGlobals.copy() + extraArgs['setUp'] = saveGlobals + return doctest.DocTestSuite(module, **extraArgs) + + def loadAnything(self, thing, recurse=False): + """ + Given a Python object, return whatever tests that are in it. Whatever + 'in' might mean. + + @param thing: A Python object. A module, method, class or package. + @param recurse: Whether or not to look in subpackages of packages. + Defaults to False. + + @return: A C{TestCase} or C{TestSuite}. + """ + if isinstance(thing, types.ModuleType): + if isPackage(thing): + return self.loadPackage(thing, recurse) + return self.loadModule(thing) + elif isinstance(thing, types.ClassType): + return self.loadClass(thing) + elif isinstance(thing, type): + return self.loadClass(thing) + elif isinstance(thing, types.MethodType): + return self.loadMethod(thing) + raise TypeError("No loader for %r. Unrecognized type" % (thing,)) + + def loadByName(self, name, recurse=False): + """ + Given a string representing a Python object, return whatever tests + are in that object. + + If C{name} is somehow inaccessible (e.g. the module can't be imported, + there is no Python object with that name etc) then return an + L{ErrorHolder}. + + @param name: The fully-qualified name of a Python object. + """ + try: + thing = self.findByName(name) + except: + return ErrorHolder(name, failure.Failure()) + return self.loadAnything(thing, recurse) + loadTestsFromName = loadByName + + def loadByNames(self, names, recurse=False): + """ + Construct a TestSuite containing all the tests found in 'names', where + names is a list of fully qualified python names and/or filenames. The + suite returned will have no duplicate tests, even if the same object + is named twice. + """ + things = [] + errors = [] + for name in names: + try: + things.append(self.findByName(name)) + except: + errors.append(ErrorHolder(name, failure.Failure())) + suites = [self.loadAnything(thing, recurse) + for thing in self._uniqueTests(things)] + suites.extend(errors) + return self.suiteFactory(suites) + + + def _uniqueTests(self, things): + """ + Gather unique suite objects from loaded things. This will guarantee + uniqueness of inherited methods on TestCases which would otherwise hash + to same value and collapse to one test unexpectedly if using simpler + means: e.g. set(). + """ + entries = [] + for thing in things: + if isinstance(thing, types.MethodType): + entries.append((thing, thing.im_class)) + else: + entries.append((thing,)) + return [entry[0] for entry in set(entries)] + + + + +class OONIRunner(object): + """ + A specialised runner that is used by the ooniprobe frontend to run tests. + Heavily inspired by the trial TrialRunner class. + """ + + DEBUG = 'debug' + DRY_RUN = 'dry-run' + + def _getDebugger(self): + dbg = pdb.Pdb() + try: + import readline + except ImportError: + print "readline module not available" + sys.exc_clear() + for path in ('.pdbrc', 'pdbrc'): + if os.path.exists(path): + try: + rcFile = file(path, 'r') + except IOError: + sys.exc_clear() + else: + dbg.rcLines.extend(rcFile.readlines()) + return dbg + + + def _setUpTestdir(self): + self._tearDownLogFile() + currentDir = os.getcwd() + base = filepath.FilePath(self.workingDirectory) + testdir, self._testDirLock = util._unusedTestDirectory(base) + os.chdir(testdir.path) + return currentDir + + + def _tearDownTestdir(self, oldDir): + os.chdir(oldDir) + self._testDirLock.unlock() + + + _log = log + def _makeResult(self): + reporter = self.reporterFactory(self.stream, self.tbformat, + self.rterrors, self._log) + if self.uncleanWarnings: + reporter = UncleanWarningsReporterWrapper(reporter) + return reporter + + def __init__(self, reporterFactory, + mode=None, + logfile='test.log', + stream=sys.stdout, + profile=False, + tracebackFormat='default', + realTimeErrors=False, + uncleanWarnings=False, + workingDirectory=None, + forceGarbageCollection=False): + self.reporterFactory = reporterFactory + self.logfile = logfile + self.mode = mode + self.stream = stream + self.tbformat = tracebackFormat + self.rterrors = realTimeErrors + self.uncleanWarnings = uncleanWarnings + self._result = None + self.workingDirectory = workingDirectory or '_trial_temp' + self._logFileObserver = None + self._logFileObject = None + self._forceGarbageCollection = forceGarbageCollection + if profile: + self.run = util.profiled(self.run, 'profile.data') + + def _tearDownLogFile(self): + if self._logFileObserver is not None: + log.removeObserver(self._logFileObserver.emit) + self._logFileObserver = None + if self._logFileObject is not None: + self._logFileObject.close() + self._logFileObject = None + + def _setUpLogFile(self): + self._tearDownLogFile() + if self.logfile == '-': + logFile = sys.stdout + else: + logFile = file(self.logfile, 'a') + self._logFileObject = logFile + self._logFileObserver = log.FileLogObserver(logFile) + log.startLoggingWithObserver(self._logFileObserver.emit, 0) + + def run(self, test, inputs): + """ + Run the test or suite and return a result object. + """ + for input in inputs: + self._runWithInput(test, input) + + def _runWithInput(self, test, input): + """ + Private helper that runs the given test with the given input. + """ + result = self._makeResult() + # decorate the suite with reactor cleanup and log starting + # This should move out of the runner and be presumed to be + # present + suite = TrialSuite([test]) + startTime = time.time() + + ## XXX replace this with the actual way of running the test. + run = lambda: suite.run(result) + + oldDir = self._setUpTestdir() + try: + self._setUpLogFile() + run() + finally: + self._tearDownLogFile() + self._tearDownTestdir(oldDir) + + endTime = time.time() + done = getattr(result, 'done', None) + result.done() + return result diff --git a/ooni/scaffolding.py b/ooni/scaffolding.py old mode 100755 new mode 100644 diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py old mode 100755 new mode 100644

1 0

[ooni-probe/master] * Moved CustomCircuit class to /ooni/utils/circuit.py.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 21647a9c5ce54341e188619e3c67426fc9114630 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Sep 17 03:57:30 2012 +0000 * Moved CustomCircuit class to /ooni/utils/circuit.py. * Tor is started improperly, and the timeout function is stupid and just returns the deferred, rather than a timeout *on* the deferred. --- ooni/plugins/bridget.py | 289 ++++++++++++++++++++++++++--------------------- ooni/utils/circuit.py | 106 +++++++++++++++++ 2 files changed, 264 insertions(+), 131 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 49565d8..f22741b 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -13,6 +13,7 @@ # :version: 0.1.0-alpha from __future__ import with_statement +from functools import wraps from os import getcwd from os.path import isfile from os.path import join as pj @@ -22,6 +23,7 @@ from twisted.internet import defer, error, reactor from zope.interface import implements import random +import signal import sys from ooni.utils import log @@ -36,7 +38,7 @@ def portCheck(number): portCheckAllowed = "must be between 1024 and 65535." sockCheck, ctrlCheck = portCheck, portCheck -sockCheck.coerceDoc = "Port to use for Tor's SocksPort, " + portCheckAllowed +sockCheck.coerceDoc = "Port to use for Tor's SocksPort, " + portCheckAllowed ctrlCheck.coerceDoc = "Port to use for Tor's ControlPort, " + portCheckAllowed @@ -105,11 +107,11 @@ class BridgetTest(OONITest): """ implements(IPlugin, ITest) - shortName = "bridget" - description = "Use a Tor process to test connecting to bridges and relays" + shortName = "bridget" + description = "Use a Tor process to test connecting to bridges and relays" requirements = None - options = BridgetArgs - blocking = False + options = BridgetArgs + blocking = False def initialize(self): """ @@ -117,10 +119,11 @@ class BridgetTest(OONITest): running, so we need to deal with the creation of TorConfig() only once, before the experiment runs. """ - self.socks_port = 9049 - self.control_port = 9052 - self.tor_binary = '/usr/sbin/tor' - self.data_directory = None + self.socks_port = 9049 + self.control_port = 9052 + self.tor_binary = '/usr/sbin/tor' + self.data_directory = None + self.circuit_timeout = 90 if self.local_options: options = self.local_options @@ -176,11 +179,32 @@ class BridgetTest(OONITest): if not options['bridges']: e = "You must use the bridge option to test a transport." raise usage.UsageError("%s" % e) - - log.msg("Using pluggable transport ...") - ## XXX fixme there's got to be a better way to check the exec - assert type(options['transport']) is str - self.config.ClientTransportPlugin = options['transport'] + else: + ## XXX fixme there's got to be a better way to check the exec + ## + ## we could use: + ## os.setgid( NEW_GID ) + ## os.setuid( NEW_UID ) + ## to drop any and all privileges + assert type(options['transport']) is str + [self.pt_type, + self.pt_exec] = options['transport'].split(' ', 1) + log.msg("Using ClientTransportPlugin %s %s" + % (self.pt_type, self.pt_exec)) + self.pt_use = True + + ## XXX fixme we need a better way to deal with all PTs + if self.pt_type == "obfs2": + self.ctp = self.pt_type +" "+ self.pt_exec + else: + m = "Pluggable Transport type %s was " % self.pt_type + m += "unaccounted for, please contact isis (at) " + m += "torproject (dot) org, with info and it'll get " + m += "included." + log.msg("%s" % m) + self.ctp = None + else: + self.pt_use = False self.config.SocksPort = self.socks_port self.config.ControlPort = self.control_port @@ -261,7 +285,7 @@ class BridgetTest(OONITest): def reconf_controller(conf, bridge): ## if bridges and relays, use one bridge then build a circuit ## from three relays - conf.Bridge = bridge + conf.Bridge = "" ## XXX do we need a SIGHUP to restart? ## XXX see txtorcon.TorControlProtocol.add_event_listener we @@ -289,131 +313,136 @@ class BridgetTest(OONITest): def updates(prog, tag, summary): log.msg("%d%%: %s" % (prog, summary)) + if not self.circuit_timeout: + self.circuit_timeout = 90 + + class TimeoutError(Exception): + pass + + def stupid_timer(secs, e=None): + def decorator(func): + def _timer(signum, frame): + raise TimeoutError, e + def wrapper(*args, **kwargs): + signal.signal(signal.SIGALRM, _timer) + signal.alarm(secs) + try: + res = func(*args, **kwargs) + finally: + signal.alarm(0) + return res + return wraps(func)(wrapper) + return decorator + + @stupid_timer(self.circuit_timeout) + def _launch_tor_and_report_bridge_status(_config, + _reactor, + _updates, + _binary, + _callback, + _callback_arg, + _errback): + """The grossest Python function you've ever seen.""" + + log.msg("Starting Tor ...") + ## :return: a Deferred which callbacks with a TorProcessProtocol + ## connected to the fully-bootstrapped Tor; this has a + ## txtorcon.TorControlProtocol instance as .protocol. + proc_proto = launch_tor(_config, _reactor, progress_updates=_updates, + tor_binary=_binary).addCallback(_callback, + _callback_arg).addErrback(_errback) + return proc_proto + ## now build circuits + + + if len(args) == 0: log.msg("Bridget can't run without bridges or relays to test!") log.msg("Exiting ...") return sys.exit() else: - - class CustomCircuit(CircuitListenerMixin): - implements(IStreamAttacher) - - from txtorcon.interface import IRouterContainer - from txtorcon.interface import ICircuitContainer - - def __init__(self, state): - self.state = state - self.waiting_circuits = [] - - def waiting_on(self, circuit): - for (circid, d) in self.waiting_circuits: - if circuit.id == circid: - return true - return False - - def circuit_extend(self, circuit, router): - "ICircuitListener" - if circuit.purpose != 'GENERAL': - return - if self.waiting_on(circuit): - log.msg("Circuit %d (%s)" - % (circuit.id, router.id_hex)) - - def circuit_built(self, circuit): - "ICircuitListener" - if circuit.purpose != 'GENERAL': - return - log.msg("Circuit %s built ..." % circuit.id) - log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) - for (circid, d) in self.waiting_circuits: - if circid == circuit.id: - self.waiting_circuits.remove(circid, d) - d.callback(circuit) - - def circuit_failed(self, circuit, reason): - if self.waiting_on(circuit): - log.msg("A circuit we requested %s failed for reason %s" - % (circuit.id, reason)) - circid, d = None, None - for x in self.waiting_circuits: - if x[0] == circuit.id: - circid, d, stream_cc = x - if d is None: - raise Exception("Expected to find circuit.") - - self.waiting_circuits.remove((circid, d)) - log.msg("Trying to build a circuit for %s" % circid) - self.request_circuit_build(d) - - def check_circuit_route(self, circuit, router): - if router in circuit.path: - #router.update() ## XXX can i use without args? no. - TorInfo.dump(self) - - def request_circuit_build(self, deferred): - entries = self.state.entry_guards.value() - relays = self.state.routers.values() - log.msg("We have these nodes listed as entry guards:") - log.msg("%s" % entries) - log.msg("We have these nodes listed as relays:") - log.msg("%s" % relays) - path = [random.choice(entries), - random.choice(relays), - random.choice(relays)] - log.msg("Requesting a circuit: %s" - % '-->'.join(map(lambda x: x.location.countrycode, - path))) - - class AppendWaiting: - def __init__(self, attacher, deferred): - self.attacher = attacher - self.d = deferred - - def __call__(self, circuit): - """ - Return from build_circuit is a Circuit, however, - we want to wait until it is built before we can - issue an attach on it and callback to the Deferred - we issue here. - """ - log.msg("Circuit %s is in progress ..." % circuit.id) - self.attacher.waiting_circuits.append((circuit.id, - self.d)) - - fin = self.state.build_circuit(path) - fin.addCallback(AppendWaiting(self, deferred_to_callback)) - fin.addErrback(log.err) - return fin - + log.msg("Bridget: initiating test ... ") + self.london_bridge = [] + self.working_bridge = [] if len(self.bridge_list) >= 1: - for bridge in self.bridge_list: - try: - log.msg("Current Bridge: %s" % bridge) - reconf_controller(self.config, bridge) - except: - reconf_fail(bridge) + self.untested_bridge_count = len(self.bridge_list) + self.burnt_bridge_count = len(self.london_bridge) + self.current_bridge = None - log.msg("Bridget: initiating test ... ") - log.msg("Using the following as our torrc:\n%s" - % self.config.create_torrc()) - report = {'tor_config': self.config.config} - log.msg("Starting Tor ...") + ## while bridges are in the bucket + while self.config.UseBridges and self.untested_bridge_count > 0: + try: + ## if the user wanted to use pluggable transports + assert self.pt_use is True + except AssertionError as no_use_pt: + ## if the user didn't want to use pluggable transports + log.msg("Configuring Bridge line without pluggable transport") + log.msg("%s" % no_use_pt) + self.pt_good = False + else: + ## user wanted PT, and we recognized transport + if self.ctp is not None: + try: + assert self.ctp is str + except AssertionError as not_a_string: + log.msg("Error: ClientTransportPlugin string unusable: %s" + % not_a_string) + log.msg(" Exiting ...") + sys.exit() + else: + ## configure the transport + self.config.ClientTransportPlugin = self.ctp + self.pt_good = True + ## user wanted PT, but we didn't recognize it + else: + log.msg("Error: Unable to use ClientTransportPlugin %s %s" + % (self.pt_type, self.pt_exec)) + log.msg(" Exiting...") + sys.exit() + ## whether or not we're using a pluggable transport, we need + ## to set the Bridge line + finally: + log.msg("We now have %d bridges in our list..." + % self.untested_bridge_count) + self.current_bridge = self.bridge_list.pop() + self.untested_bridge_count -= 1 + + log.msg("Current Bridge: %s" % self.current_bridge) + + if self.pt_good: + self.config.Bridge = self.pt_type +" "+ self.current_bridge + else: + self.config.Bridge = self.current_bridge + + log.msg("Using the following as our torrc:\n%s" + % self.config.create_torrc()) + report = {'tor_config': self.config.create_torrc()} + self.config.save() - ## :return: a Deferred which callbacks with a TorProcessProtocol - ## connected to the fully-bootstrapped Tor; this has a - ## txtorcon.TorControlProtocol instance as .protocol. - d = launch_tor(self.config, - reactor, - progress_updates=updates, - tor_binary=self.tor_binary) - d.addCallback(bootstrap, self.config) - d.addErrback(setup_fail) - ## now build circuits - - #print "Tor process ID: %s" % d.transport.pid + try: + _launch_tor_and_report_bridge_status(self.config, + reactor, + updates, + self.tor_binary, + bootstrap, + self.config, + setup_fail) + except TimeoutError: + log.msg("Adding %s to bad bridges..." % self.current_bridge) + self.london_bridge.append(self.current_bridge) + else: + log.msg("Adding %s to good bridges..." % self.current_bridge) + self.working_bridge.append(self.current_bridge) + + d = defer.Deferred() + d.addCallback(log.msg, 'Working Bridges:\n%s\nUnreachable Bridges:\n%s\n' + % (self.working_bridge, self.london_bridge)) return d + def control(self, exp_res): + exp_res + ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) @@ -438,5 +467,3 @@ bridget = BridgetTest(None, None, None) ## ## FIX: ## o DataDirectory is not found, or permissions aren't right -## o Bridge line needs generation of transport properties -## Bridge <transport> IP:ORPort <fingerprint> diff --git a/ooni/utils/circuit.py b/ooni/utils/circuit.py new file mode 100644 index 0000000..bb1ab11 --- /dev/null +++ b/ooni/utils/circuit.py @@ -0,0 +1,106 @@ +# +# circuit.py +# ---------- +# Utilities for working with Tor circuits. +# +# This code is largely taken from the txtorcon documentation, and as +# such any and all credit should go to meejah. +# +# :author: Mike Warren, Isis Lovecruft +# :license: see included license file +# :version: 0.1.0-alpha +# +from zope.interface import implements + +from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher + + +class CustomCircuit(CircuitListenerMixin): + implements(IStreamAttacher) + + from txtorcon.interface import IRouterContainer + from txtorcon.interface import ICircuitContainer + + def __init__(self, state): + self.state = state + self.waiting_circuits = [] + + def waiting_on(self, circuit): + for (circid, d) in self.waiting_circuits: + if circuit.id == circid: + return true + return False + + def circuit_extend(self, circuit, router): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + if self.waiting_on(circuit): + log.msg("Circuit %d (%s)" + % (circuit.id, router.id_hex)) + + def circuit_built(self, circuit): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + log.msg("Circuit %s built ..." % circuit.id) + log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) + for (circid, d) in self.waiting_circuits: + if circid == circuit.id: + self.waiting_circuits.remove(circid, d) + d.callback(circuit) + + def circuit_failed(self, circuit, reason): + if self.waiting_on(circuit): + log.msg("A circuit we requested %s failed for reason %s" + % (circuit.id, reason)) + circid, d = None, None + for x in self.waiting_circuits: + if x[0] == circuit.id: + circid, d, stream_cc = x + if d is None: + raise Exception("Expected to find circuit.") + + self.waiting_circuits.remove((circid, d)) + log.msg("Trying to build a circuit for %s" % circid) + self.request_circuit_build(d) + + def check_circuit_route(self, circuit, router): + if router in circuit.path: + #router.update() ## XXX can i use without args? no. + TorInfo.dump(self) + + def request_circuit_build(self, deferred): + entries = self.state.entry_guards.value() + relays = self.state.routers.values() + log.msg("We have these nodes listed as entry guards:") + log.msg("%s" % entries) + log.msg("We have these nodes listed as relays:") + log.msg("%s" % relays) + path = [random.choice(entries), + random.choice(relays), + random.choice(relays)] + log.msg("Requesting a circuit: %s" + % '-->'.join(map(lambda x: x.location.countrycode, + path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + + def __call__(self, circuit): + """ + Return from build_circuit is a Circuit, however, + we want to wait until it is built before we can + issue an attach on it and callback to the Deferred + we issue here. + """ + log.msg("Circuit %s is in progress ..." % circuit.id) + self.attacher.waiting_circuits.append((circuit.id, + self.d)) + + fin = self.state.build_circuit(path) + fin.addCallback(AppendWaiting(self, deferred_to_callback)) + fin.addErrback(log.err) + return fin

1 0

[ooni-probe/master] Import nettest work in progress related to reporting and testsuite
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 98ebfe5e98824b4343a7a03f950f70e8f1c30faf Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 18 10:07:35 2012 +0000 Import nettest work in progress related to reporting and testsuite --- ooni/nettest.py | 64 +++++++------------- ooni/oonicli.py | 18 ++++++ ooni/reporter.py | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 210 insertions(+), 47 deletions(-) diff --git a/bin/ooniprobe b/bin/ooniprobe old mode 100644 new mode 100755 diff --git a/ooni/nettest.py b/ooni/nettest.py index ab009b1..097e947 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -18,6 +18,29 @@ def _iterateTests(testSuiteOrCase): yield subtest +class TestSuite(pyunit.TestSuite): + inputUnit = [None] + def __repr__(self): + return "<%s input=%s, tests=%s>" % (self.__class__, self.inputUnit, list(self)) + + def run(self, result, inputUnit=[None]): + """ + Call C{run} on every member of the suite. + """ + # we implement this because Python 2.3 unittest defines this code + # in __call__, whereas 2.4 defines the code in run. + idx = 0 + for input, test in itertools.product(inputUnit, self._tests): + if result.shouldStop: + break + self.inputUnit = inputUnit + test.input = input + test.idx = idx + test(result) + idx += 1 + + return result + class TestCase(unittest.TestCase): """ A test case represents the minimum @@ -65,44 +88,3 @@ class TestCase(unittest.TestCase): result.stopTest(self) -class TestSuite(pyunit.TestSuite): - """ - Extend the standard library's C{TestSuite} with support for the visitor - pattern and a consistently overrideable C{run} method. - """ - - def __init__(self, tests=(), inputs=()): - self._tests = [] - self._inputs = [] - self.addTests(tests, inputs) - print "Adding %s %s" % (tests, inputs) - - - def __call__(self, result): - return self.run(result) - - def __repr__(self): - return "<%s input=%s tests=%s>" % (self.__class__, - self._inputs, list(self)) - - def run(self, result, input=None): - """ - Call C{run} on every member of the suite. - """ - for test in self._tests: - if result.shouldStop: - break - return test(result, None) - - def addTests(self, tests, inputs=[]): - if isinstance(tests, basestring): - raise TypeError("tests must be and iterable of tests not a string") - for test in tests: - self.addTest(test, inputs) - - def addTest(self, test, inputs=[]): - #print "Adding: %s" % test - super(TestSuite, self).addTest(test) - self._inputs = inputs - - diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 199b4d4..8eabfb1 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -328,6 +328,23 @@ def _makeRunner(config): forceGarbageCollection=config['force-gc']) +if 0: + loader = runner.TestLoader() + loader.suiteFactory = TestSuite + + for inputUnit in InputUnitFactory(FooTest.inputs): + print inputUnit + + suite = loader.loadClass(FooTest) + + reporterFactory = ReporterFactory(open('reporting.log', 'a+'), testSuite=suite) + reporterFactory.writeHeader() + #testUnitReport = OONIReporter(open('reporting.log', 'a+')) + #testUnitReport.writeHeader(FooTest) + for inputUnit in InputUnitFactory(FooTest.inputs): + testUnitReport = reporterFactory.create() + suite(testUnitReport, inputUnit) + testUnitReport.done() def run(): if len(sys.argv) == 1: @@ -340,6 +357,7 @@ def run(): _initialDebugSetup(config) trialRunner = _makeRunner(config) suite = _getSuite(config) + print suite test_result = trialRunner.run(suite) if config.tracer: sys.settrace(None) diff --git a/ooni/reporter.py b/ooni/reporter.py index d20160f..07cffad 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -1,11 +1,174 @@ -from twisted.trial import reporter +import time +import sys +import yaml +import itertools -class OONIReporter(reporter.Reporter): +sys.path.insert(0, '/home/x/Documents/pen_drive_bitcoin2012/ooni-probe/ENV/lib/python2.7/site-packages') +from datetime import datetime +from twisted.python.util import OrderedDict, untilConcludes +from twisted.trial import unittest, reporter, runner - def startTest(self, test, input=None): - print "Running %s" % test - print "Input %s" % input - self._input = input +pyunit = __import__('unittest') + +class OReporter(pyunit.TestResult): + def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, + publisher=None, testSuite=None): + super(OReporter, self).__init__() + self.report = {'tests': []} + self._stream = reporter.SafeStream(stream) + self.tbformat = tbformat + self.realtime = realtime + self._startTime = None + self._warningCache = set() + + self._publisher = publisher + + def _getTime(self): + return time.time() + + def _write(self, format, *args): + s = str(format) + assert isinstance(s, type('')) + if args: + self._stream.write(s % args) + else: + self._stream.write(s) + untilConcludes(self._stream.flush) + + def _writeln(self, format, *args): + self._write(format, *args) + self._write('\n') + + def writeYamlLine(self, line): + self._write(yaml.dump([line])) + + + +class ReporterFactory(OReporter): + def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, + publisher=None, testSuite=None): + super(ReporterFactory, self).__init__(stream=stream, + tbformat=tbformat, realtime=realtime, publisher=publisher) + + self._testSuite = testSuite + self._reporters = [] + + def writeHeader(self): + pretty_date = "XXX Replace me with date.pretty_date()" + self._writeln("###########################################") + self._writeln("# OONI Probe Report for Test %s" % "XXX replace with with the test suite name") + self._writeln("# %s" % pretty_date) + self._writeln("###########################################") + + address = {'asn': 'XXX replace me with ASN', + 'ip': 'XXX replace me with IP'} + test_details = {'start_time': datetime.now(), + 'asn': address['asn'], + 'test_name': 'XXX replace me with the test name', + 'addr': address['ip']} + self.writeYamlLine(test_details) + self._writeln('') + + def create(self): + r = OONIReporter(self._stream, self.tbformat, self.realtime, + self._publisher) + self._reporters.append(OONIReporter) + return r + + +class OONIReporter(OReporter): + def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, + publisher=None): + super(OONIReporter, self).__init__(stream=stream, + tbformat=tbformat, realtime=realtime, publisher=publisher) + + self._tests = {} + self._publisher = publisher + if publisher is not None: + publisher.addObserver(self._observeWarnings) + + def getTestIndex(self, test): + try: + idx = test.idx + except: + idx = 0 + return idx + + + def startTest(self, test): super(OONIReporter, self).startTest(test) + idx = self.getTestIndex(test) + if not self._startTime: + self._startTime = self._getTime() + + test.report = {} + + self._tests[idx] = {} + self._tests[idx]['testStarted'] = self._getTime() + self._tests[idx]['input'] = test.input + self._tests[idx]['idx'] = idx + self._tests[idx]['name'] = test.name + self._tests[idx]['test'] = test + + + def stopTest(self, test): + super(OONIReporter, self).stopTest(test) + + idx = self.getTestIndex(test) + + self._tests[idx]['lastTime'] = self._getTime() - self._tests[idx]['testStarted'] + # XXX I put a dict() here so that the object is re-instantiated and I + # actually end up with the report I want. This could either be a + # python bug or a yaml bug. + self._tests[idx]['report'] = dict(test.report) + + def done(self): + """ + Summarize the result of the test run. + + The summary includes a report of all of the errors, todos, skips and + so forth that occurred during the run. It also includes the number of + tests that were run and how long it took to run them (not including + load time). + + Expects that L{_printErrors}, L{_writeln}, L{_write}, L{_printSummary} + and L{_separator} are all implemented. + """ + if self._publisher is not None: + self._publisher.removeObserver(self._observeWarnings) + if self._startTime is not None: + self.report['startTime'] = self._startTime + self.report['runTime'] = time.time() - self._startTime + self.report['testsRun'] = self.testsRun + self.report['tests'] = self._tests + self.writeReport() + + def writeReport(self): + self.writeYamlLine(self.report) + + def addSuccess(self, test): + super(OONIReporter, self).addSuccess(test) + #self.report['result'] = {'value': 'success'} + + def addError(self, *args): + super(OONIReporter, self).addError(*args) + #self.report['result'] = {'value': 'error', 'args': args} + + def addFailure(self, *args): + super(OONIReporter, self).addFailure(*args) + #self.report['result'] = {'value': 'failure', 'args': args} + + def addSkip(self, *args): + super(OONIReporter, self).addSkip(*args) + #self.report['result'] = {'value': 'skip', 'args': args} + + def addExpectedFailure(self, *args): + super(OONIReporter, self).addExpectedFailure(*args) + #self.report['result'] = {'value': 'expectedFailure', 'args': args} + + def addUnexpectedSuccess(self, *args): + super(OONIReporter, self).addUnexpectedSuccess(*args) + #self.report['result'] = {'args': args, 'value': 'unexpectedSuccess'} +

1 0

[ooni-probe/master] Made some progress, not there yet.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 7dce07bfc58566d2fdec4eec8a831d4ecd545d5f Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 18 17:51:07 2012 +0000 Made some progress, not there yet. * After this commit I am going to throw away a lot of code and start over --- ooni/input.py | 10 ++- ooni/nettest.py | 50 +++++++-- ooni/oonicli.py | 56 +++-------- ooni/reporter.py | 5 +- ooni/runner.py | 305 +++++++++-------------------------------------------- 5 files changed, 117 insertions(+), 309 deletions(-) diff --git a/ooni/input.py b/ooni/input.py index f534393..b931b82 100644 --- a/ooni/input.py +++ b/ooni/input.py @@ -43,7 +43,7 @@ class InputUnit(object): passed onto a TestCase. """ def __init__(self, inputs=[]): - self._inputs = inputs + self._inputs = iter(inputs) def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self._inputs) @@ -53,7 +53,13 @@ class InputUnit(object): self._inputs.append(input) def __iter__(self): - return iter(self._inputs) + return self + + def next(self): + try: + return self._inputs.next() + except: + raise StopIteration def append(self, input): self._inputs.append(input) diff --git a/ooni/nettest.py b/ooni/nettest.py index 0c8858b..4ab1e0f 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -19,28 +19,60 @@ def _iterateTests(testSuiteOrCase): yield subtest +class TestSuiteFactory(object): + def __init__(self, inputUnit, tests, basesuite): + self._baseSuite = basesuite + self._inputUnit = inputUnit + self._idx = 0 + self.tests = tests + + def __iter__(self): + return self + + def next(self): + try: + next_input = self._inputUnit.next() + print "Now dealing with %s %s" % (next_input, self._idx) + except: + raise StopIteration + new_test_suite = self._baseSuite(self.tests) + new_test_suite.input = next_input + new_test_suite._idx = self._idx + + self._idx += 1 + return new_test_suite + class TestSuite(pyunit.TestSuite): - inputUnit = [None] + def __init__(self, tests=()): + self._tests = [] + self.input = None + self._idx = 0 + self.addTests(tests) + def __repr__(self): - return "<%s input=%s, tests=%s>" % (self.__class__, self.inputUnit, list(self)) + return "<%s input=%s tests=%s>" % (self.__class__, + self.input, self._tests) - def run(self, result, inputUnit=[None]): + def run(self, result): """ Call C{run} on every member of the suite. """ # we implement this because Python 2.3 unittest defines this code # in __call__, whereas 2.4 defines the code in run. - idx = 0 - for input, test in itertools.product(inputUnit, self._tests): + for i, test in enumerate(self._tests): if result.shouldStop: break - self.inputUnit = inputUnit - test.input = input - test.idx = idx + test.input = self.input + test._idx = self._idx + i test(result) - idx += 1 return result class TestCase(unittest.TestCase): name = "DefaultTestName" + inputs = [None] + + def __repr__(self): + return "<%s inputs=%s>" % (self.__class__, self.inputs) + + diff --git a/ooni/oonicli.py b/ooni/oonicli.py index fa7742f..be73d30 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -17,12 +17,13 @@ import sys, os, random, gc, time, warnings from twisted.internet import defer from twisted.application import app -from twisted.python import usage, reflect, failure +from twisted.python import usage, reflect, failure, log from twisted.python.filepath import FilePath from twisted import plugin from twisted.python.util import spewer from twisted.python.compat import set from twisted.trial import itrial +from twisted.trial import runner as irunner from ooni import runner, reporter @@ -104,10 +105,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): "callback stack traces"], ["nopm", None, "don't automatically jump into debugger for " "postmorteming of exceptions"], - ["dry-run", 'n', "do everything but run the tests"], ["force-gc", None, "Have OONI run gc.collect() before and " "after each test case."], - ["profile", None, "Run tests under the Python profiler"], ["unclean-warnings", None, "Turn dirty reactor errors into warnings"], ["no-recurse", "N", "Don't recurse into packages"], @@ -118,9 +117,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): optParameters = [ ["reportfile", "o", "report.yaml", "report file name"], ["logfile", "l", "test.log", "log file name"], - ["random", "z", None, - "Run tests in random order using the specified seed"], - ['temp-directory', None, '_trial_temp', + ['temp-directory', None, '_ooni_temp', 'Path to use as working directory for tests.'], ['reporter', None, 'default', 'The reporter to use for this test run. See --help-reporters for ' @@ -129,7 +126,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): compData = usage.Completions( optActions={"tbformat": usage.CompleteList(["plain", "emacs", "cgitb"]), "logfile": usage.CompleteFiles(descr="log file name"), - "random": usage.Completer(descr="random seed")}, + }, extraActions=[usage.CompleteFiles( "*.py", descr="file | module | package | TestCase | testMethod", repeat=True)], @@ -238,20 +235,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): "argument to recursionlimit must be an integer") - def opt_random(self, option): - try: - self['random'] = long(option) - except ValueError: - raise usage.UsageError( - "Argument to --random must be a positive integer") - else: - if self['random'] < 0: - raise usage.UsageError( - "Argument to --random must be a positive integer") - elif self['random'] == 0: - self['random'] = long(time.time() * 100) - - def opt_without_module(self, option): """ Fake the lack of the specified modules, separated with commas. @@ -283,7 +266,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): failure.DO_POST_MORTEM = False - def _initialDebugSetup(config): # do this part of debug setup first for easy debugging of import failures if config['debug']: @@ -292,35 +274,22 @@ def _initialDebugSetup(config): defer.setDebugging(True) - -def _getSuites(config): - loader = _getLoader(config) +def _getSuitesAndInputs(config): + #loader = irunner.TestLoader() + loader = runner.NetTestLoader() recurse = not config['no-recurse'] print "loadByNames %s" % config['tests'] - return loader.loadByNames(config['tests'], recurse) - - -def _getLoader(config): - loader = runner.NetTestLoader() - if config['random']: - randomer = random.Random() - randomer.seed(config['random']) - loader.sorter = lambda x : randomer.random() - print 'Running tests shuffled with seed %d\n' % config['random'] - return loader - + inputs, suites = loader.loadByNamesWithInput(config['tests'], recurse) + return inputs, suites def _makeRunner(config): mode = None if config['debug']: mode = runner.OONIRunner.DEBUG - if config['dry-run']: - mode = runner.OONIRunner.DRY_RUN print "using %s" % config['reporter'] return runner.OONIRunner(config['reporter'], reportfile=config["reportfile"], mode=mode, - profile=config['profile'], logfile=config['logfile'], tracebackFormat=config['tbformat'], realTimeErrors=config['rterrors'], @@ -340,7 +309,8 @@ def run(): _initialDebugSetup(config) trialRunner = _makeRunner(config) - suites = _getSuites(config) - for suite in suites: - test_result = trialRunner.run(suite) + inputs, testSuites = _getSuitesAndInputs(config) + log.startLogging(sys.stdout) + for i, suite in enumerate(testSuites): + test_result = trialRunner.run(suite, inputs[i]) diff --git a/ooni/reporter.py b/ooni/reporter.py index 07cffad..14297cd 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -89,7 +89,7 @@ class OONIReporter(OReporter): def getTestIndex(self, test): try: - idx = test.idx + idx = test._idx except: idx = 0 return idx @@ -109,7 +109,8 @@ class OONIReporter(OReporter): self._tests[idx]['input'] = test.input self._tests[idx]['idx'] = idx self._tests[idx]['name'] = test.name - self._tests[idx]['test'] = test + #self._tests[idx]['test'] = test + print "Now starting %s" % self._tests[idx] def stopTest(self, test): diff --git a/ooni/runner.py b/ooni/runner.py index 12ab9ad..604942d 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -4,7 +4,7 @@ import types import time import inspect -from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.python import reflect, log, failure from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader @@ -27,7 +27,7 @@ def isLegacyTest(obj): except TypeError: return False -def adaptLegacyTest(obj): +def adaptLegacyTest(obj, inputs=[None]): """ We take a legacy OONITest class and convert it into a nettest.TestCase. This allows backward compatibility of old OONI tests. @@ -36,8 +36,18 @@ def adaptLegacyTest(obj): older test cases compatible with the new OONI. """ class LegacyOONITest(nettest.TestCase): - pass + inputs = [1] + original_test = obj + def test_start_legacy_test(self): + print "bla bla bla" + my_test = self.original_test() + print my_test + print "foobat" + my_test.startTest(self.input) + print "HHAHAHA" + + return LegacyOONITest class LoggedSuite(nettest.TestSuite): @@ -93,11 +103,14 @@ class OONISuite(nettest.TestSuite): self._bail() -class NetTestLoader(object): +class NetTestLoader(TestLoader): """ Reponsible for finding the modules that can work as tests and running them. If we detect that a certain test is written using the legacy OONI API we will wrap it around a next gen class to make it work here too. + + XXX This class needs to be cleaned up a *lot* of all the things we actually + don't need. """ methodPrefix = 'test' modulePrefix = 'test_' @@ -106,67 +119,19 @@ class NetTestLoader(object): self.suiteFactory = nettest.TestSuite self._importErrors = [] - def findTestClasses(self, module): classes = [] for name, val in inspect.getmembers(module): - try: - inputs = val.inputs - except: - inputs = None if isTestCase(val): - classes.append((val, inputs)) + classes.append(val) # This is here to allow backward compatibility with legacy OONI # tests. elif isLegacyTest(val): - #val = adaptLegacyTest(val) - classes.append((val, inputs)) + print "adapting! %s" % val + val = adaptLegacyTest(val) + classes.append(val) return classes - def findByName(self, name): - """ - Return a Python object given a string describing it. - - @param name: a string which may be either a filename or a - fully-qualified Python name. - - @return: If C{name} is a filename, return the module. If C{name} is a - fully-qualified Python name, return the object it refers to. - """ - if os.path.exists(name): - return filenameToModule(name) - return reflect.namedAny(name) - - - def loadModule(self, module): - """ - Return a test suite with all the tests from a module. - - Included are TestCase subclasses and doctests listed in the module's - __doctests__ module. If that's not good for you, put a function named - either C{testSuite} or C{test_suite} in your module that returns a - TestSuite, and I'll use the results of that instead. - - If C{testSuite} and C{test_suite} are both present, then I'll use - C{testSuite}. - """ - ## XXX - should I add an optional parameter to disable the check for - ## a custom suite. - ## OR, should I add another method - if not isinstance(module, types.ModuleType): - raise TypeError("%r is not a module" % (module,)) - if hasattr(module, 'testSuite'): - return module.testSuite() - elif hasattr(module, 'test_suite'): - return module.test_suite() - - suite = self.suiteFactory() - for testClass, inputs in self.findTestClasses(module): - testCases = self.loadClass(testClass) - - return testCases - loadTestsFromModule = loadModule - def loadClass(self, klass): """ Given a class which contains test cases, return a sorted list of @@ -182,186 +147,50 @@ class NetTestLoader(object): tests.append(self._makeCase(klass, self.methodPrefix+name)) suite = self.suiteFactory(tests) - suite.inputs = klass.inputs - return suite - loadTestsFromTestCase = loadClass - - def getTestCaseNames(self, klass): - """ - Given a class that contains C{TestCase}s, return a list of names of - methods that probably contain tests. - """ - return reflect.prefixedMethodNames(klass, self.methodPrefix) - - def loadMethod(self, method): - """ - Given a method of a C{TestCase} that represents a test, return a - C{TestCase} instance for that test. - """ - if not isinstance(method, types.MethodType): - raise TypeError("%r not a method" % (method,)) - return self._makeCase(method.im_class, _getMethodNameInClass(method)) - - def _makeCase(self, klass, methodName): - return klass(methodName) + print "**+*" + print tests + print "**+*" - def loadPackage(self, package, recurse=False): - """ - Load tests from a module object representing a package, and return a - TestSuite containing those tests. - - Tests are only loaded from modules whose name begins with 'test_' - (or whatever C{modulePrefix} is set to). - - @param package: a types.ModuleType object (or reasonable facsimilie - obtained by importing) which may contain tests. - - @param recurse: A boolean. If True, inspect modules within packages - within the given package (and so on), otherwise, only inspect modules - in the package itself. - - @raise: TypeError if 'package' is not a package. - - @return: a TestSuite created with my suiteFactory, containing all the - tests. - """ - if not isPackage(package): - raise TypeError("%r is not a package" % (package,)) - pkgobj = modules.getModule(package.__name__) - if recurse: - discovery = pkgobj.walkModules() - else: - discovery = pkgobj.iterModules() - discovered = [] - for disco in discovery: - if disco.name.split(".")[-1].startswith(self.modulePrefix): - discovered.append(disco) - suite = self.suiteFactory() - for modinfo in self.sort(discovered): - try: - module = modinfo.load() - except: - thingToAdd = ErrorHolder(modinfo.name, failure.Failure()) - else: - thingToAdd = self.loadModule(module) - suite.addTest(thingToAdd) return suite + loadTestsFromTestCase = loadClass - def loadDoctests(self, module): - """ - Return a suite of tests for all the doctests defined in C{module}. - - @param module: A module object or a module name. - """ - if isinstance(module, str): + def findAllInputs(self, thing): + testClasses = self.findTestClasses(thing) + # XXX will there ever be more than 1 test class with inputs? + for klass in testClasses: try: - module = reflect.namedAny(module) + inputs = klass.inputs except: - return ErrorHolder(module, failure.Failure()) - if not inspect.ismodule(module): - warnings.warn("trial only supports doctesting modules") - return - extraArgs = {} - if sys.version_info > (2, 4): - # Work around Python issue2604: DocTestCase.tearDown clobbers globs - def saveGlobals(test): - """ - Save C{test.globs} and replace it with a copy so that if - necessary, the original will be available for the next test - run. - """ - test._savedGlobals = getattr(test, '_savedGlobals', test.globs) - test.globs = test._savedGlobals.copy() - extraArgs['setUp'] = saveGlobals - return doctest.DocTestSuite(module, **extraArgs) - - def loadAnything(self, thing, recurse=False): - """ - Given a Python object, return whatever tests that are in it. Whatever - 'in' might mean. - - @param thing: A Python object. A module, method, class or package. - @param recurse: Whether or not to look in subpackages of packages. - Defaults to False. - - @return: A C{TestCase} or C{TestSuite}. - """ - print "Loading anything! %s" % thing - ret = None - if isinstance(thing, types.ModuleType): - if isPackage(thing): - ret = self.loadPackage(thing, recurse) - ret = self.loadModule(thing) - elif isinstance(thing, types.ClassType): - ret = self.loadClass(thing) - elif isinstance(thing, type): - ret = self.loadClass(thing) - elif isinstance(thing, types.MethodType): - ret = self.loadMethod(thing) - if not ret: - raise TypeError("No loader for %r. Unrecognized type" % (thing,)) - try: - ret.inputs = ret.inputs - except: - ret.inputs = [None] - return ret - - def loadByName(self, name, recurse=False): - """ - Given a string representing a Python object, return whatever tests - are in that object. - - If C{name} is somehow inaccessible (e.g. the module can't be imported, - there is no Python object with that name etc) then return an - L{ErrorHolder}. - - @param name: The fully-qualified name of a Python object. - """ - print "Load by Name!" - try: - thing = self.findByName(name) - except: - return ErrorHolder(name, failure.Failure()) - return self.loadAnything(thing, recurse) - loadTestsFromName = loadByName + pass + return inputs - def loadByNames(self, names, recurse=False): + def loadByNamesWithInput(self, names, recurse=False): """ - Construct a TestSuite containing all the tests found in 'names', where + Construct a OONITestSuite containing all the tests found in 'names', where names is a list of fully qualified python names and/or filenames. The suite returned will have no duplicate tests, even if the same object is named twice. + + This test suite will have set the attribute inputs to the inputs found + inside of the tests. """ - print "Load by Names!" + inputs = [] things = [] errors = [] for name in names: try: - things.append(self.findByName(name)) + thing = self.findByName(name) + things.append(thing) except: errors.append(ErrorHolder(name, failure.Failure())) - suites = [self.loadAnything(thing, recurse) - for thing in self._uniqueTests(things)] - suites.extend(errors) - return suites - #return self.suiteFactory(suites) - - - def _uniqueTests(self, things): - """ - Gather unique suite objects from loaded things. This will guarantee - uniqueness of inherited methods on TestCases which would otherwise hash - to same value and collapse to one test unexpectedly if using simpler - means: e.g. set(). - """ - entries = [] - for thing in things: - if isinstance(thing, types.MethodType): - entries.append((thing, thing.im_class)) - else: - entries.append((thing,)) - return [entry[0] for entry in set(entries)] + suites = [] + for thing in self._uniqueTests(things): + inputs.append(self.findAllInputs(thing)) + suite = self.loadAnything(thing, recurse) + suites.append(suite) + suites.extend(errors) + return inputs, suites class OONIRunner(object): """ @@ -457,47 +286,17 @@ class OONIRunner(object): self._logFileObserver = log.FileLogObserver(logFile) log.startLoggingWithObserver(self._logFileObserver.emit, 0) - def run(self, test): + def run(self, tests, inputs=[None]): """ Run the test or suite and return a result object. """ - print test - inputs = test.inputs reporterFactory = ReporterFactory(open(self._reportfile, 'a+'), - testSuite=test) + testSuite=tests) reporterFactory.writeHeader() - #testUnitReport = OONIReporter(open('reporting.log', 'a+')) - #testUnitReport.writeHeader(FooTest) for inputUnit in InputUnitFactory(inputs): + testSuiteFactory = nettest.TestSuiteFactory(inputUnit, tests, nettest.TestSuite) testUnitReport = reporterFactory.create() - test(testUnitReport, inputUnit) + for suite in testSuiteFactory: + suite(testUnitReport) testUnitReport.done() - def _runWithInput(self, test, input): - """ - Private helper that runs the given test with the given input. - """ - result = self._makeResult() - # decorate the suite with reactor cleanup and log starting - # This should move out of the runner and be presumed to be - # present - suite = TrialSuite([test]) - startTime = time.time() - - ## XXX replace this with the actual way of running the test. - run = lambda: suite.run(result) - - oldDir = self._setUpTestdir() - try: - self._setUpLogFile() - run() - finally: - self._tearDownLogFile() - self._tearDownTestdir(oldDir) - - endTime = time.time() - done = getattr(result, 'done', None) - result.done() - return result - -

1 0

[ooni-probe/master] Get all the pieces together and have a working trial based oonicli
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 2d31d5f14a0f3a7518d47fe748c1677cbd885828 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 18 11:52:41 2012 +0000 Get all the pieces together and have a working trial based oonicli --- ooni/nettest.py | 48 +--------------------------------- ooni/oonicli.py | 38 ++++++---------------------- ooni/runner.py | 75 ++++++++++++++++++++++++++++++++++-------------------- 3 files changed, 57 insertions(+), 104 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 097e947..0c8858b 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -1,3 +1,4 @@ +import itertools from twisted.python import log from twisted.trial import unittest, itrial @@ -42,49 +43,4 @@ class TestSuite(pyunit.TestSuite): return result class TestCase(unittest.TestCase): - """ - A test case represents the minimum - """ - def run(self, result): - """ - Run the test case, storing the results in C{result}. - - First runs C{setUp} on self, then runs the test method (defined in the - constructor), then runs C{tearDown}. As with the standard library - L{unittest.TestCase}, the return value of these methods is disregarded. - In particular, returning a L{Deferred} has no special additional - consequences. - - @param result: A L{TestResult} object. - """ - log.msg("--> %s <--" % (self.id())) - new_result = itrial.IReporter(result, None) - if new_result is None: - result = PyUnitResultAdapter(result) - else: - result = new_result - result.startTest(self) - if self.getSkip(): # don't run test methods that are marked as .skip - result.addSkip(self, self.getSkip()) - result.stopTest(self) - return - - self._passed = False - self._warnings = [] - - self._installObserver() - # All the code inside _runFixturesAndTest will be run such that warnings - # emitted by it will be collected and retrievable by flushWarnings. - unittest._collectWarnings(self._warnings.append, self._runFixturesAndTest, result) - - # Any collected warnings which the test method didn't flush get - # re-emitted so they'll be logged or show up on stdout or whatever. - for w in self.flushWarnings(): - try: - warnings.warn_explicit(**w) - except: - result.addError(self, failure.Failure()) - - result.stopTest(self) - - + name = "DefaultTestName" diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 8eabfb1..fa7742f 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -116,6 +116,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): ] optParameters = [ + ["reportfile", "o", "report.yaml", "report file name"], ["logfile", "l", "test.log", "log file name"], ["random", "z", None, "Run tests in random order using the specified seed"], @@ -292,7 +293,7 @@ def _initialDebugSetup(config): -def _getSuite(config): +def _getSuites(config): loader = _getLoader(config) recurse = not config['no-recurse'] print "loadByNames %s" % config['tests'] @@ -300,7 +301,7 @@ def _getSuite(config): def _getLoader(config): - loader = runner.TestLoader() + loader = runner.NetTestLoader() if config['random']: randomer = random.Random() randomer.seed(config['random']) @@ -309,7 +310,6 @@ def _getLoader(config): return loader - def _makeRunner(config): mode = None if config['debug']: @@ -318,6 +318,7 @@ def _makeRunner(config): mode = runner.OONIRunner.DRY_RUN print "using %s" % config['reporter'] return runner.OONIRunner(config['reporter'], + reportfile=config["reportfile"], mode=mode, profile=config['profile'], logfile=config['logfile'], @@ -328,24 +329,6 @@ def _makeRunner(config): forceGarbageCollection=config['force-gc']) -if 0: - loader = runner.TestLoader() - loader.suiteFactory = TestSuite - - for inputUnit in InputUnitFactory(FooTest.inputs): - print inputUnit - - suite = loader.loadClass(FooTest) - - reporterFactory = ReporterFactory(open('reporting.log', 'a+'), testSuite=suite) - reporterFactory.writeHeader() - #testUnitReport = OONIReporter(open('reporting.log', 'a+')) - #testUnitReport.writeHeader(FooTest) - for inputUnit in InputUnitFactory(FooTest.inputs): - testUnitReport = reporterFactory.create() - suite(testUnitReport, inputUnit) - testUnitReport.done() - def run(): if len(sys.argv) == 1: sys.argv.append("--help") @@ -354,15 +337,10 @@ def run(): config.parseOptions() except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) + _initialDebugSetup(config) trialRunner = _makeRunner(config) - suite = _getSuite(config) - print suite - test_result = trialRunner.run(suite) - if config.tracer: - sys.settrace(None) - results = config.tracer.results() - results.write_results(show_missing=1, summary=False, - coverdir=config.coverdir().path) - sys.exit(not test_result.wasSuccessful()) + suites = _getSuites(config) + for suite in suites: + test_result = trialRunner.run(suite) diff --git a/ooni/runner.py b/ooni/runner.py index a8485af..12ab9ad 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -1,13 +1,18 @@ +import os +import sys import types import time import inspect from twisted.internet import defer -from twisted.python import reflect +from twisted.python import reflect, log, failure from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader -from twisted.trial.runner import isPackage, isTestCase +from twisted.trial.runner import isPackage, isTestCase, ErrorHolder +from twisted.trial.runner import filenameToModule, _importFromFile +from ooni.reporter import ReporterFactory +from ooni.input import InputUnitFactory from ooni import nettest from ooni.plugoo import tests as oonitests @@ -34,6 +39,7 @@ def adaptLegacyTest(obj): pass + class LoggedSuite(nettest.TestSuite): """ Any errors logged in this suite will be reported to the L{TestResult} @@ -98,17 +104,8 @@ class NetTestLoader(object): def __init__(self): self.suiteFactory = nettest.TestSuite - self.sorter = name self._importErrors = [] - def sort(self, xs): - """ - Sort the given things using L{sorter}. - - @param xs: A list of test cases, class or modules. - """ - return sorted(xs, key=self.sorter) - def findTestClasses(self, module): classes = [] @@ -180,12 +177,12 @@ class NetTestLoader(object): if not isTestCase(klass): raise ValueError("%r is not a test case" % (klass,)) names = self.getTestCaseNames(klass) - print "Names %s" % names - tests = self.sort([self._makeCase(klass, self.methodPrefix+name) - for name in names]) - print "Tests %s" % tests + tests = [] + for name in names: + tests.append(self._makeCase(klass, self.methodPrefix+name)) + suite = self.suiteFactory(tests) - print "Suite: %s" % suite + suite.inputs = klass.inputs return suite loadTestsFromTestCase = loadClass @@ -289,17 +286,25 @@ class NetTestLoader(object): @return: A C{TestCase} or C{TestSuite}. """ + print "Loading anything! %s" % thing + ret = None if isinstance(thing, types.ModuleType): if isPackage(thing): - return self.loadPackage(thing, recurse) - return self.loadModule(thing) + ret = self.loadPackage(thing, recurse) + ret = self.loadModule(thing) elif isinstance(thing, types.ClassType): - return self.loadClass(thing) + ret = self.loadClass(thing) elif isinstance(thing, type): - return self.loadClass(thing) + ret = self.loadClass(thing) elif isinstance(thing, types.MethodType): - return self.loadMethod(thing) - raise TypeError("No loader for %r. Unrecognized type" % (thing,)) + ret = self.loadMethod(thing) + if not ret: + raise TypeError("No loader for %r. Unrecognized type" % (thing,)) + try: + ret.inputs = ret.inputs + except: + ret.inputs = [None] + return ret def loadByName(self, name, recurse=False): """ @@ -312,6 +317,7 @@ class NetTestLoader(object): @param name: The fully-qualified name of a Python object. """ + print "Load by Name!" try: thing = self.findByName(name) except: @@ -326,6 +332,7 @@ class NetTestLoader(object): suite returned will have no duplicate tests, even if the same object is named twice. """ + print "Load by Names!" things = [] errors = [] for name in names: @@ -336,7 +343,8 @@ class NetTestLoader(object): suites = [self.loadAnything(thing, recurse) for thing in self._uniqueTests(things)] suites.extend(errors) - return self.suiteFactory(suites) + return suites + #return self.suiteFactory(suites) def _uniqueTests(self, things): @@ -355,8 +363,6 @@ class NetTestLoader(object): return [entry[0] for entry in set(entries)] - - class OONIRunner(object): """ A specialised runner that is used by the ooniprobe frontend to run tests. @@ -407,6 +413,7 @@ class OONIRunner(object): return reporter def __init__(self, reporterFactory, + reportfile="report.yaml", mode=None, logfile='test.log', stream=sys.stdout, @@ -417,6 +424,7 @@ class OONIRunner(object): workingDirectory=None, forceGarbageCollection=False): self.reporterFactory = reporterFactory + self._reportfile = reportfile self.logfile = logfile self.mode = mode self.stream = stream @@ -449,12 +457,21 @@ class OONIRunner(object): self._logFileObserver = log.FileLogObserver(logFile) log.startLoggingWithObserver(self._logFileObserver.emit, 0) - def run(self, test, inputs): + def run(self, test): """ Run the test or suite and return a result object. """ - for input in inputs: - self._runWithInput(test, input) + print test + inputs = test.inputs + reporterFactory = ReporterFactory(open(self._reportfile, 'a+'), + testSuite=test) + reporterFactory.writeHeader() + #testUnitReport = OONIReporter(open('reporting.log', 'a+')) + #testUnitReport.writeHeader(FooTest) + for inputUnit in InputUnitFactory(inputs): + testUnitReport = reporterFactory.create() + test(testUnitReport, inputUnit) + testUnitReport.done() def _runWithInput(self, test, input): """ @@ -482,3 +499,5 @@ class OONIRunner(object): done = getattr(result, 'done', None) result.done() return result + +

1 0

[ooni-probe/master] * Switched to chainDeferreds for testing bridge by bridge, after which
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 92ff36a1bdcfa34264ca3cec11d9a53f1bdac323 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Sep 19 05:45:54 2012 +0000 * Switched to chainDeferreds for testing bridge by bridge, after which EXTEND cells are issued per relay per bridge. And it actually works. --- ooni/plugins/bridget.py | 121 +++++++++++++++++++++++++++-------------------- 1 files changed, 69 insertions(+), 52 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 1ed4957..192c2dd 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -14,23 +14,21 @@ from __future__ import with_statement from functools import wraps, partial +from random import randint from twisted.python import usage from twisted.plugin import IPlugin from twisted.internet import defer, error, reactor -from twisted.internet.endpoints import TCP4ClientEndpoint from zope.interface import implements from ooni.utils import log from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset -import tempfile import os -import random -import shutil import signal import sys + def timer(secs, e=None): def decorator(func): def _timer(signum, frame): @@ -75,16 +73,15 @@ class ValueChecker(object): raise ValueError("Port out of range") sock_check, ctrl_check = port_check, port_check - port_check_allowed = "must be between 1024 and 65535." - sock_check.coerceDoc = "Port to use for Tor's SocksPort, " + port_check_allowed - ctrl_check.coerceDoc = "Port to use for Tor's ControlPort, " + port_check_allowed + allowed = "must be between 1024 and 65535." + sock_check.coerceDoc = "Port to use for Tor's SocksPort, " +allowed + ctrl_check.coerceDoc = "Port to use for Tor's ControlPort, " +allowed def uid_check(pluggable_transport): - """Check that we are not root when trying to use pluggable transports.""" + """Check that we're not root when trying to use pluggable transports.""" uid, gid = os.getuid(), os.getgid() if uid == 0 and gid == 0: - log.msg("Error: Running bridget as root with --transport=%s not allowed." - % pluggable_transport) + log.msg("Error: Running bridget as root with transports not allowed.") log.msg("Dropping privileges to normal user...") os.setgid(1000) os.setuid(1000) @@ -242,18 +239,22 @@ class BridgetTest(OONITest): if options['bridges']: self.config.UseBridges = 1 make_asset_list(options['bridges'], self.bridges) + if options['relays']: ## first hop must be in TorState().entry_guards to build circuits self.config.EntryNodes = ','.join(relay_list) make_asset_list(options['relays'], self.relays) + if options['socks']: self.socks_port = options['socks'] if options['control']: self.control_port = options['control'] + if options['random']: log.msg("Using randomized ControlPort and SocksPort ...") - self.socks_port = random.randint(1024, 2**16) - self.control_port = random.randint(1024, 2**16) + self.socks_port = randint(1024, 2**16) + self.control_port = randint(1024, 2**16) + if options['torpath']: self.tor_binary = options['torpath'] @@ -274,9 +275,9 @@ class BridgetTest(OONITest): else: raise PTNotFoundException - config.SocksPort = self.socks_port - config.ControlPort = self.control_port - config.CookieAuthentication = 1 + self.config.SocksPort = self.socks_port + self.config.ControlPort = self.control_port + self.config.CookieAuthentication = 1 def load_assets(self): """ @@ -302,12 +303,19 @@ class BridgetTest(OONITest): The :class:`BridgetAsset` line currently being used. """ try: - from ooni.utils import circuit + from tempfile import mkstemp, mkdtemp + from shutil import rmtree + + from twisted.internet.endpoints import TCP4ClientEndpoint + + from ooni.utils import circuit from ooni.lib.txtorcon import TorProcessProtocol from ooni.lib.txtorcon import TorProtocolFactory from ooni.lib.txtorcon import TorConfig, TorState + except ImportError: raise TxtorconImportError + except TxtorconImportError: ## XXX is this something we should add to the reactor? sys.exit() @@ -330,19 +338,19 @@ class BridgetTest(OONITest): try: os.unlink(temp) except OSError: - shutil.rmtree(temp, ignore_errors=True) + rmtree(temp, ignore_errors=True) - @timer(self.circuit_timeout) + #(a)timer(self.circuit_timeout) def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): """Rewrite the Bridge line in our torrc.""" log.msg("Current Bridge: %s" % bridge) if use_pt is False: - new = state.protocol.set_conf('Bridge', bridge) + state.protocol.set_conf('Bridge', bridge) elif use_pt and pt_type is not None: - new = state.protocol.set_conf('Bridge', pt_type +' '+ bridge) + state.protocol.set_conf('Bridge', pt_type +' '+ bridge) else: raise PTNotFoundException - return new + return state.callback def reconfigure_fail(*param): log.msg("Reconfiguring TorConfig with parameters %s failed" % param) @@ -358,7 +366,7 @@ class BridgetTest(OONITest): if len(both) > 0: for bridge in both: bridge_list.remove(bridge) - return state + return state.callback def remove_relays_fail(state): log.msg("Unable to remove public relays from the bridge list.") @@ -429,6 +437,9 @@ class BridgetTest(OONITest): attacher.request_circuit_build(d) return d + def state_attach_fail(state): + log.msg("Attaching custom circuit builder failed.") + def updates(prog, tag, summary): log.msg("%d%%: %s" % (prog, summary)) @@ -443,12 +454,12 @@ class BridgetTest(OONITest): delete_list = [] if data_dir is None: - data_dir = tempfile.mkdtemp(prefix='bridget-tordata') + data_dir = mkdtemp(prefix='bridget-tordata') delete_list.append(data_dir) conf.DataDirectory = data_dir #conf.__OwningControllerProcess = os.getpid() - (fd, torrc) = tempfile.mkstemp(dir=data_dir) + (fd, torrc) = mkstemp(dir=data_dir) delete_list.append(torrc) os.write(fd, conf.create_torrc()) os.close(fd) @@ -459,10 +470,20 @@ class BridgetTest(OONITest): while self.bridges_remaining() > 0: self.current_bridge = self.bridges.pop() - #current_ip = self.current_bridge.split(':', 1)[0] - #print "CURRENT BRIDGE IP %s" % current_ip + if self.config.config.has_key('Bridge'): + log.msg("We now have %d untested bridges..." + % self.bridges_remaining()) + reconf = defer.Deferred() + reconf.addCallback(reconfigure_bridge, state, + self.current_bridge, self.use_pt, + self.pt_type) + reconf.addErrback(reconfigure_fail) + state.chainDeferred(reconf) + #reconfigure_bridge(state, self.current_bridge, + # self.use_pt, self.pt_type) + #current_ip = self.current_bridge.split(':', 1)[0] - if not self.config.config.has_key('Bridge'): + else: self.config.Bridge = self.current_bridge (torrc, self.data_directory, to_delete) = write_torrc( self.config, self.data_directory) @@ -477,11 +498,12 @@ class BridgetTest(OONITest): self.data_directory) state.addCallback(setup_done) state.addErrback(setup_fail) - state.addCallback(remove_relays, self.bridges) - state.addErrback(remove_relays_fail) - state.addCallback(state_attach, self.bridges) - ## XXX write state_attach_fail() - state.addErrback(state_attach_fail) + state.addBoth(remove_relays, self.bridges) + #state.addCallback(remove_relays, self.bridges) + #state.addErrback(remove_relays_fail) + #state.addCallback(state_attach, self.bridges) + #state.addErrback(state_attach_fail) + return state ## XXX see txtorcon.TorControlProtocol.add_event_listener we ## may not need full CustomCircuit class @@ -490,21 +512,19 @@ class BridgetTest(OONITest): ## o if bridges only, try one bridge at a time, but don't build ## circuits, just return ## o if relays only, build circuits from relays - else: - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - try: - state.addCallback(reconfigure_bridge, self.current_bridge, - self.use_pt, self.pt_type) - state.addErrback(reconfigure_fail) - except TimeoutError: - log.msg("Adding %s to unreachable bridges..." - % self.current_bridge) - self.bridges_down.append(self.current_bridge) - else: - log.msg("Adding %s to reachable bridges..." - % self.current_bridge) - self.bridges_up.append(self.current_bridge) + #else: + # try: + # state.addCallback(reconfigure_bridge, self.current_bridge, + # self.use_pt, self.pt_type) + # state.addErrback(reconfigure_fail) + # except TimeoutError: + # log.msg("Adding %s to unreachable bridges..." + # % self.current_bridge) + # self.bridges_down.append(self.current_bridge) + # else: + # log.msg("Adding %s to reachable bridges..." + # % self.current_bridge) + # self.bridges_up.append(self.current_bridge) reactor.run() ## now build circuits @@ -518,12 +538,9 @@ bridget = BridgetTest(None, None, None) ## TODO: ## o cleanup documentation ## x add DataDirectory option -## o check if bridges are public relays +## x check if bridges are public relays ## o take bridge_desc file as input, also be able to give same ## format as output -## o Add assychronous timout for deferred, so that we don't wait +## x Add assychronous timout for deferred, so that we don't wait ## forever for bridges that don't work. ## o Add mechanism for testing through another host -## -## FIX: -## o DataDirectory is not found, or permissions aren't right

1 0

[ooni-probe/master] * Refactored bridget completely.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 6f52ec0c73671e5b1c417cc1b7a0d63266cd8b9c Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Sep 19 02:04:55 2012 +0000 * Refactored bridget completely. * Broke up most of experiment() into tiny functions. * Added custom exceptions and error handlers. * Added more commandline option type enforcers, with custom handlers. * Function to drop any privileges if using pluggable transports. * The generated torrc now includes a Bridge line written to the file descriptor, if using bridges. * The Bridge line properly handles pluggable transports. TODO: Still need to fix the CustomCircuit class and the attacher. --- ooni/plugins/bridget.py | 652 ++++++++++++++++++++++++++--------------------- ooni/utils/circuit.py | 50 ++-- 2 files changed, 380 insertions(+), 322 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index f22741b..1ed4957 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -3,77 +3,163 @@ # # +-----------+ # | BRIDGET | -# | +----------------------------------------------+ -# +--------| Use a slave Tor process to test making a Tor | -# | connection to a list of bridges or relays. | -# +----------------------------------------------+ +# | +--------------------------------------------+ +# +--------| Use a Tor process to test making a Tor | +# | connection to a list of bridges or relays. | +# +--------------------------------------------+ # -# :authors: Arturo Filasto, Isis Lovecruft +# :authors: Isis Lovecruft, Arturo Filasto # :licence: see included LICENSE # :version: 0.1.0-alpha -from __future__ import with_statement -from functools import wraps -from os import getcwd -from os.path import isfile -from os.path import join as pj -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import defer, error, reactor -from zope.interface import implements +from __future__ import with_statement +from functools import wraps, partial +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor +from twisted.internet.endpoints import TCP4ClientEndpoint +from zope.interface import implements +from ooni.utils import log +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset + +import tempfile +import os import random +import shutil import signal import sys -from ooni.utils import log -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset - - -def portCheck(number): - number = int(number) - if number not in range(1024, 65535): - raise ValueError("Port out of range") - -portCheckAllowed = "must be between 1024 and 65535." -sockCheck, ctrlCheck = portCheck, portCheck -sockCheck.coerceDoc = "Port to use for Tor's SocksPort, " + portCheckAllowed -ctrlCheck.coerceDoc = "Port to use for Tor's ControlPort, " + portCheckAllowed - +def timer(secs, e=None): + def decorator(func): + def _timer(signum, frame): + raise TimeoutError, e + def wrapper(*args, **kwargs): + signal.signal(signal.SIGALRM, _timer) + signal.alarm(secs) + try: + res = func(*args, **kwargs) + finally: + signal.alarm(0) + return res + return wraps(func)(wrapper) + return decorator + + +class MissingAssetException(Exception): + """Raised when neither there are neither bridges nor relays to test.""" + def __init__(self): + log.msg("Bridget can't run without bridges or relays to test!") + return sys.exit() + +class PTNoBridgesException(Exception): + """Raised when a pluggable transport is specified, but no bridges.""" + def __init__(self): + log.msg("Pluggable transport requires the bridges option") + return sys.exit() + +class PTNotFoundException(Exception): + def __init__(self, transport_type): + m = "Pluggable Transport type %s was unaccounted " % transport_type + m += "for, please contact isis(at)torproject(dot)org and it will " + m += "get included." + log.msg("%s" % m) + return sys.exit() + +class ValueChecker(object): + def port_check(self, number): + """Check that given ports are in the allowed range.""" + number = int(number) + if number not in range(1024, 65535): + raise ValueError("Port out of range") + + sock_check, ctrl_check = port_check, port_check + port_check_allowed = "must be between 1024 and 65535." + sock_check.coerceDoc = "Port to use for Tor's SocksPort, " + port_check_allowed + ctrl_check.coerceDoc = "Port to use for Tor's ControlPort, " + port_check_allowed + + def uid_check(pluggable_transport): + """Check that we are not root when trying to use pluggable transports.""" + uid, gid = os.getuid(), os.getgid() + if uid == 0 and gid == 0: + log.msg("Error: Running bridget as root with --transport=%s not allowed." + % pluggable_transport) + log.msg("Dropping privileges to normal user...") + os.setgid(1000) + os.setuid(1000) + + def dir_check(d): + """Check that the given directory exists.""" + if not os.isdir(d): + raise ValueError("%s doesn't exist, or has wrong permissions" % d) + + def file_check(f): + if not os.isfile(f): + raise ValueError("%s does not exist, or has wrong permissions" % f) + +class RandomPortException(Exception): + """Raised when using a random port conflicts with configured ports.""" + def __init__(self): + log.msg("Unable to use random and specific ports simultaneously") + return sys.exit() + +class TimeoutError(Exception): + """Raised when a timer runs out.""" + pass + +class TxtorconImportError(ImportError): + """Raised when /ooni/lib/txtorcon cannot be imported from.""" + cwd, tx = os.getcwd(), 'lib/txtorcon/torconfig.py' + try: + log.msg("Unable to import from ooni.lib.txtorcon") + if cwd.endswith('ooni'): + check = os.path.join(cwd, tx) + else: + check = os.path.join(cwd, 'ooni/'+tx) + assert isfile(check) + except: + log.msg("Error: Some OONI libraries are missing!") + log.msg("Please go to /ooni/lib/ and do \"make all\"") class BridgetArgs(usage.Options): + """Commandline options.""" + vc = ValueChecker() + optParameters = [ ['bridges', 'b', None, 'File listing bridge IP:ORPorts to test'], ['relays', 'f', None, 'File listing relay IPs to test'], - ['socks', 's', 9049, None, portCheck], - ['control', 'c', 9052, None, portCheck], + ['socks', 's', 9049, None, vc.sock_check], + ['control', 'c', 9052, None, vc.ctrl_check], ['torpath', 'p', None, 'Path to the Tor binary to use'], ['datadir', 'd', None, 'Tor DataDirectory to use'], - ['transport', 't', None, + ['transport', 't', None, 'Tor ClientTransportPlugin'], ['resume', 'r', 0, 'Resume at this index']] - optFlags = [ - ['random', 'x', 'Use random ControlPort and SocksPort']] + optFlags = [['random', 'x', 'Use random ControlPort and SocksPort']] def postOptions(self): - if self['transport'] and not self['bridges']: - e = "Pluggable transport requires the bridges option" - raise usage.UsageError, e - if self['socks'] and self['control']: + if not self['bridges'] and not self['relays']: + raise MissingAssetException + if self['transport']: + vc.uid_check(self['transport']) + if not self['bridges']: + raise PTNoBridgesException + if self['socks'] or self['control']: if self['random']: - e = "Unable to use random and specific ports simultaneously" - raise usage.usageError, e + raise RandomPortException + if self['datadir']: + vc.dir_check(self['datadir']) + if self['torpath']: + vc.file_check(self['torpath']) class BridgetAsset(Asset): - """ - Class for parsing bridget Assets ignoring commented out lines. - """ + """Class for parsing bridget Assets ignoring commented out lines.""" def __init__(self, file=None): self = Asset.__init__(self, file) @@ -89,17 +175,15 @@ class BridgetTest(OONITest): :ivar config: An :class:`ooni.lib.txtorcon.TorConfig` instance. - :ivar relay_list: - A list of all provided relays to test. We have to do this because - txtorcon.TorState().entry_guards won't build a custom circuit if the - first hop isn't in the torrc's EntryNodes. - :ivar bridge_list: + :ivar relays: + A list of all provided relays to test. + :ivar bridges: A list of all provided bridges to test. :ivar socks_port: Integer for Tor's SocksPort. :ivar control_port: Integer for Tor's ControlPort. - :ivar plug_transport: + :ivar transport: String defining the Tor's ClientTransportPlugin, for testing a bridge's pluggable transport functionality. :ivar tor_binary: @@ -116,99 +200,83 @@ class BridgetTest(OONITest): def initialize(self): """ Extra initialization steps. We only want one child Tor process - running, so we need to deal with the creation of TorConfig() only - once, before the experiment runs. + running, so we need to deal with most of the TorConfig() only once, + before the experiment runs. """ self.socks_port = 9049 self.control_port = 9052 + self.circuit_timeout = 90 self.tor_binary = '/usr/sbin/tor' self.data_directory = None - self.circuit_timeout = 90 + self.use_pt = False + self.pt_type = None + + self.bridges, self.bridges_up, self.bridges_down = ([] for i in range(3)) + self.bridges_remaining = lambda: len(self.bridges) + self.bridges_down_count = lambda: len(self.bridges_down) + self.current_bridge = None + + self.relays, self.relays_up, self.relays_down = ([] for i in range(3)) + self.relays_remaining = lambda: len(self.relays) + self.relays_down_count = lambda: len(self.relays_down) + self.current_relay = None + + def make_asset_list(opt, lst): + log.msg("Loading information from %s ..." % opt) + with open(opt) as opt_file: + for line in opt_file.readlines(): + if line.startswith('#'): + continue + else: + lst.append(line.replace('\n','')) if self.local_options: - options = self.local_options - - if not options['bridges'] and not options['relays']: - self.suicide = True - return - try: from ooni.lib.txtorcon import TorConfig except ImportError: - log.msg ("Bridget: Unable to import from ooni.lib.txtorcon") - wd, tx = getcwd(), 'lib/txtorcon/torconfig.py' - chk = pj(wd,tx) if wd.endswith('ooni') else pj(wd,'ooni/'+tx) - try: - assert isfile(chk) - except: - log.msg("Error: Some OONI libraries are missing!") - log.msg("Please go to /ooni/lib/ and do \"make all\"") + raise TxtorconImportError - self.config = TorConfig() + options = self.local_options + config = self.config = TorConfig() if options['bridges']: self.config.UseBridges = 1 - + make_asset_list(options['bridges'], self.bridges) if options['relays']: - ## Stupid hack for testing only relays: - ## Tor doesn't use EntryNodes when UseBridges is enabled, but - ## config.state.entry_guards needs to include the first hop to - ## build a custom circuit. + ## first hop must be in TorState().entry_guards to build circuits self.config.EntryNodes = ','.join(relay_list) - + make_asset_list(options['relays'], self.relays) if options['socks']: self.socks_port = options['socks'] - if options['control']: self.control_port = options['control'] - if options['random']: log.msg("Using randomized ControlPort and SocksPort ...") self.socks_port = random.randint(1024, 2**16) self.control_port = random.randint(1024, 2**16) - if options['torpath']: self.tor_binary = options['torpath'] - if options['datadir']: - self.config.DataDirectory = options['datadir'] + if self.local_options['datadir']: + self.data_directory = local_options['datadir'] + else: + self.data_directory = None if options['transport']: - ## ClientTransportPlugin transport socks4|socks5 IP:PORT + self.use_pt = True + log.msg("Using ClientTransportPlugin %s" % options['transport']) + [self.pt_type, pt_exec] = options['transport'].split(' ', 1) + ## ClientTransportPlugin transport exec path-to-binary [options] - if not options['bridges']: - e = "You must use the bridge option to test a transport." - raise usage.UsageError("%s" % e) + ## XXX we need a better way to deal with all PTs + if self.pt_type == "obfs2": + config.ClientTransportPlugin = self.pt_type + " " + pt_exec else: - ## XXX fixme there's got to be a better way to check the exec - ## - ## we could use: - ## os.setgid( NEW_GID ) - ## os.setuid( NEW_UID ) - ## to drop any and all privileges - assert type(options['transport']) is str - [self.pt_type, - self.pt_exec] = options['transport'].split(' ', 1) - log.msg("Using ClientTransportPlugin %s %s" - % (self.pt_type, self.pt_exec)) - self.pt_use = True - - ## XXX fixme we need a better way to deal with all PTs - if self.pt_type == "obfs2": - self.ctp = self.pt_type +" "+ self.pt_exec - else: - m = "Pluggable Transport type %s was " % self.pt_type - m += "unaccounted for, please contact isis (at) " - m += "torproject (dot) org, with info and it'll get " - m += "included." - log.msg("%s" % m) - self.ctp = None - else: - self.pt_use = False + raise PTNotFoundException - self.config.SocksPort = self.socks_port - self.config.ControlPort = self.control_port - self.config.save() + config.SocksPort = self.socks_port + config.ControlPort = self.control_port + config.CookieAuthentication = 1 def load_assets(self): """ @@ -216,32 +284,14 @@ class BridgetTest(OONITest): should be given in the form IP:ORport. We don't want to load these as assets, because it's inefficient to start a Tor process for each one. """ - assets = {} - self.bridge_list = [] - self.relay_list = [] - - ## XXX fix me - ## we should probably find a more memory nice way to load addresses, - ## in case the files are really large + assets = {} if self.local_options: - - def make_asset_list(opt, lst): - log.msg("Loading information from %s ..." % opt) - with open(opt) as opt_file: - for line in opt_file.readlines(): - if line.startswith('#'): - continue - else: - lst.append(line.replace('\n','')) - if self.local_options['bridges']: - make_asset_list(self.local_options['bridges'], - self.bridge_list) - assets.update({'bridges': self.bridge_list}) + assets.update({'bridge': + BridgetAsset(self.local_options['bridges'])}) if self.local_options['relays']: - make_asset_list(self.local_options['relays'], - self.relay_list) - assets.update({'relays': self.relay_list}) + assets.update({'relay': + BridgetAsset(self.local_options['relays'])}) return assets def experiment(self, args): @@ -249,55 +299,69 @@ class BridgetTest(OONITest): XXX fill me in :param args: - The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being - used. - :meth launch_tor: - Returns a Deferred which callbacks with a - :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol - <TorProcessProtocol>` connected to the fully-bootstrapped Tor; - this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol - <TorControlProtocol>` instance as .protocol. + The :class:`BridgetAsset` line currently being used. """ try: - from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher - from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState - from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor + from ooni.utils import circuit + from ooni.lib.txtorcon import TorProcessProtocol + from ooni.lib.txtorcon import TorProtocolFactory + from ooni.lib.txtorcon import TorConfig, TorState except ImportError: - log.msg("Error: Unable to import from ooni.lib.txtorcon") - wd, tx = getcwd(), 'lib/txtorcon/torconfig.py' - chk = pj(wd,tx) if wd.endswith('ooni') else pj(wd,'ooni/'+tx) - try: - assert isfile(chk) - except: - log.msg("Error: Some OONI libraries are missing!") - log.msg(" Please go to /ooni/lib/ and do \"make all\"") - return sys.exit() + raise TxtorconImportError + except TxtorconImportError: + ## XXX is this something we should add to the reactor? + sys.exit() def bootstrap(ctrl): """ Launch a Tor process with the TorConfig instance returned from - initialize(). + initialize() and write_torrc(). """ conf = TorConfig(ctrl) conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) log.msg("Tor process connected, bootstrapping ...") - def reconf_controller(conf, bridge): - ## if bridges and relays, use one bridge then build a circuit - ## from three relays - conf.Bridge = "" - ## XXX do we need a SIGHUP to restart? + def delete_temp(delete_list): + """ + Given a list of files or directories to delete, delete all and + suppress all errors. + """ + for temp in delete_list: + try: + os.unlink(temp) + except OSError: + shutil.rmtree(temp, ignore_errors=True) + + @timer(self.circuit_timeout) + def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): + """Rewrite the Bridge line in our torrc.""" + log.msg("Current Bridge: %s" % bridge) + if use_pt is False: + new = state.protocol.set_conf('Bridge', bridge) + elif use_pt and pt_type is not None: + new = state.protocol.set_conf('Bridge', pt_type +' '+ bridge) + else: + raise PTNotFoundException + return new - ## XXX see txtorcon.TorControlProtocol.add_event_listener we - ## may not need full CustomCircuit class + def reconfigure_fail(*param): + log.msg("Reconfiguring TorConfig with parameters %s failed" % param) + reactor.stop() - ## if bridges only, try one bridge at a time, but don't build - ## circuits, just return - ## if relays only, build circuits from relays + def remove_relays(state, bridge_list): + """ + Remove bridges from our bridge list which are also listed as + public relays. + """ + ips = map(lambda addr: addr.split(':',1)[0], bridge_list) + both = set(state.relays.values()).intersection(ips) + if len(both) > 0: + for bridge in both: + bridge_list.remove(bridge) + return state - def reconf_fail(args): - log.msg("Reconfiguring Tor config with args %s failed" % args) - reactor.stop() + def remove_relays_fail(state): + log.msg("Unable to remove public relays from the bridge list.") def setup_fail(args): log.msg("Setup Failed.") @@ -310,152 +374,148 @@ class BridgetTest(OONITest): state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) report.update({'success': args}) + def start_tor(reactor, update, torrc, to_delete, control_port, + tor_binary, data_directory): + """ + Create a TCP4ClientEndpoint at our control_port, and connect + it to our reactor and a spawned Tor process. Compare with + :meth:`txtorcon.launch_tor` for differences. + """ + ## XXX do we need self.reactor? + end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port) + connection_creator = partial(end_point.connect, TorProtocolFactory()) + process_protocol = TorProcessProtocol(connection_creator, updates) + process_protocol.to_delete = to_delete + reactor.addSystemEventTrigger('before', 'shutdown', + partial(delete_temp, to_delete)) + try: + transport = reactor.spawnProcess(process_protocol, + tor_binary, + args=(tor_binary,'-f',torrc), + env={'HOME': data_directory}, + path=data_directory) + transport.closeStdin() + except RuntimeError, e: + process_protocol.connected_cb.errback(e) + finally: + return process_protocol.connected_cb + + def state_complete(state): + """Called when we've got a TorState.""" + log.msg("We've completely booted up a Tor version %s at PID %d" + % (state.protocol.version, state.tor_pid)) + + log.msg("This Tor has the following %d Circuits:" + % len(state.circuits)) + for circ in state.circuits.values(): + log.msg("%s" % circ) + + return state + + def state_attach(state, relay_list): + log.msg("Setting up custom circuit builder...") + attacher = CustomCircuit(state) + state.set_attacher(attacher, reactor) + state.add_circuit_listener(attacher) + + for circ in state.circuits.values(): + for relay in circ.path: + try: + relay_list.remove(relay) + except KeyError: + continue + ## XXX how do we attach to circuits with bridges? + d = defer.Deferred() + attacher.request_circuit_build(d) + return d + def updates(prog, tag, summary): log.msg("%d%%: %s" % (prog, summary)) - if not self.circuit_timeout: - self.circuit_timeout = 90 - - class TimeoutError(Exception): - pass - - def stupid_timer(secs, e=None): - def decorator(func): - def _timer(signum, frame): - raise TimeoutError, e - def wrapper(*args, **kwargs): - signal.signal(signal.SIGALRM, _timer) - signal.alarm(secs) - try: - res = func(*args, **kwargs) - finally: - signal.alarm(0) - return res - return wraps(func)(wrapper) - return decorator + def write_torrc(conf, data_dir=None): + """ + Create a torrc in our data_directory. If we don't yet have a + data_directory, create a temporary one. Any temporary files or + folders are added to delete_list. + + :return: delete_list, data_dir, torrc + """ + delete_list = [] + + if data_dir is None: + data_dir = tempfile.mkdtemp(prefix='bridget-tordata') + delete_list.append(data_dir) + conf.DataDirectory = data_dir + #conf.__OwningControllerProcess = os.getpid() + + (fd, torrc) = tempfile.mkstemp(dir=data_dir) + delete_list.append(torrc) + os.write(fd, conf.create_torrc()) + os.close(fd) + return torrc, data_dir, delete_list - @stupid_timer(self.circuit_timeout) - def _launch_tor_and_report_bridge_status(_config, - _reactor, - _updates, - _binary, - _callback, - _callback_arg, - _errback): - """The grossest Python function you've ever seen.""" - - log.msg("Starting Tor ...") - ## :return: a Deferred which callbacks with a TorProcessProtocol - ## connected to the fully-bootstrapped Tor; this has a - ## txtorcon.TorControlProtocol instance as .protocol. - proc_proto = launch_tor(_config, _reactor, progress_updates=_updates, - tor_binary=_binary).addCallback(_callback, - _callback_arg).addErrback(_errback) - return proc_proto - ## now build circuits - - - - if len(args) == 0: - log.msg("Bridget can't run without bridges or relays to test!") - log.msg("Exiting ...") - return sys.exit() - else: - log.msg("Bridget: initiating test ... ") - self.london_bridge = [] - self.working_bridge = [] - if len(self.bridge_list) >= 1: - self.untested_bridge_count = len(self.bridge_list) - self.burnt_bridge_count = len(self.london_bridge) - self.current_bridge = None + log.msg("Bridget: initiating test ... ") + + while self.bridges_remaining() > 0: + self.current_bridge = self.bridges.pop() + #current_ip = self.current_bridge.split(':', 1)[0] + #print "CURRENT BRIDGE IP %s" % current_ip + + if not self.config.config.has_key('Bridge'): + self.config.Bridge = self.current_bridge + (torrc, self.data_directory, to_delete) = write_torrc( + self.config, self.data_directory) + + log.msg("Starting Tor ...") + log.msg("Using the following as our torrc:\n%s" + % self.config.create_torrc()) + report = {'tor_config': self.config.create_torrc()} + + state = start_tor(reactor, updates, torrc, to_delete, + self.control_port, self.tor_binary, + self.data_directory) + state.addCallback(setup_done) + state.addErrback(setup_fail) + state.addCallback(remove_relays, self.bridges) + state.addErrback(remove_relays_fail) + state.addCallback(state_attach, self.bridges) + ## XXX write state_attach_fail() + state.addErrback(state_attach_fail) - ## while bridges are in the bucket - while self.config.UseBridges and self.untested_bridge_count > 0: + ## XXX see txtorcon.TorControlProtocol.add_event_listener we + ## may not need full CustomCircuit class + ## o if bridges and relays, use one bridge then build a circuit + ## from three relays + ## o if bridges only, try one bridge at a time, but don't build + ## circuits, just return + ## o if relays only, build circuits from relays + else: + log.msg("We now have %d untested bridges..." + % self.bridges_remaining()) try: - ## if the user wanted to use pluggable transports - assert self.pt_use is True - except AssertionError as no_use_pt: - ## if the user didn't want to use pluggable transports - log.msg("Configuring Bridge line without pluggable transport") - log.msg("%s" % no_use_pt) - self.pt_good = False + state.addCallback(reconfigure_bridge, self.current_bridge, + self.use_pt, self.pt_type) + state.addErrback(reconfigure_fail) + except TimeoutError: + log.msg("Adding %s to unreachable bridges..." + % self.current_bridge) + self.bridges_down.append(self.current_bridge) else: - ## user wanted PT, and we recognized transport - if self.ctp is not None: - try: - assert self.ctp is str - except AssertionError as not_a_string: - log.msg("Error: ClientTransportPlugin string unusable: %s" - % not_a_string) - log.msg(" Exiting ...") - sys.exit() - else: - ## configure the transport - self.config.ClientTransportPlugin = self.ctp - self.pt_good = True - ## user wanted PT, but we didn't recognize it - else: - log.msg("Error: Unable to use ClientTransportPlugin %s %s" - % (self.pt_type, self.pt_exec)) - log.msg(" Exiting...") - sys.exit() - ## whether or not we're using a pluggable transport, we need - ## to set the Bridge line - finally: - log.msg("We now have %d bridges in our list..." - % self.untested_bridge_count) - self.current_bridge = self.bridge_list.pop() - self.untested_bridge_count -= 1 - - log.msg("Current Bridge: %s" % self.current_bridge) - - if self.pt_good: - self.config.Bridge = self.pt_type +" "+ self.current_bridge - else: - self.config.Bridge = self.current_bridge - - log.msg("Using the following as our torrc:\n%s" - % self.config.create_torrc()) - report = {'tor_config': self.config.create_torrc()} - self.config.save() + log.msg("Adding %s to reachable bridges..." + % self.current_bridge) + self.bridges_up.append(self.current_bridge) - try: - _launch_tor_and_report_bridge_status(self.config, - reactor, - updates, - self.tor_binary, - bootstrap, - self.config, - setup_fail) - except TimeoutError: - log.msg("Adding %s to bad bridges..." % self.current_bridge) - self.london_bridge.append(self.current_bridge) - else: - log.msg("Adding %s to good bridges..." % self.current_bridge) - self.working_bridge.append(self.current_bridge) + reactor.run() + ## now build circuits - d = defer.Deferred() - d.addCallback(log.msg, 'Working Bridges:\n%s\nUnreachable Bridges:\n%s\n' - % (self.working_bridge, self.london_bridge)) - return d - - def control(self, exp_res): - exp_res ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) ## ISIS' NOTES ## ----------- -## self.config.save() only needs to be called if Tor is already running. -## -## to test gid, uid, and euid: -## with open('/proc/self/state') as uidfile: -## print uidfile.read(1000) -## ## TODO: -## o add option for any kwarg=arg self.config setting ## o cleanup documentation ## x add DataDirectory option ## o check if bridges are public relays diff --git a/ooni/utils/circuit.py b/ooni/utils/circuit.py index bb1ab11..0c77dd1 100644 --- a/ooni/utils/circuit.py +++ b/ooni/utils/circuit.py @@ -13,6 +13,9 @@ from zope.interface import implements from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +from ooni.utils import log + +import random class CustomCircuit(CircuitListenerMixin): @@ -28,7 +31,7 @@ class CustomCircuit(CircuitListenerMixin): def waiting_on(self, circuit): for (circid, d) in self.waiting_circuits: if circuit.id == circid: - return true + return True return False def circuit_extend(self, circuit, router): @@ -36,8 +39,7 @@ class CustomCircuit(CircuitListenerMixin): if circuit.purpose != 'GENERAL': return if self.waiting_on(circuit): - log.msg("Circuit %d (%s)" - % (circuit.id, router.id_hex)) + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) def circuit_built(self, circuit): "ICircuitListener" @@ -47,7 +49,7 @@ class CustomCircuit(CircuitListenerMixin): log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) for (circid, d) in self.waiting_circuits: if circid == circuit.id: - self.waiting_circuits.remove(circid, d) + self.waiting_circuits.remove((circid, d)) d.callback(circuit) def circuit_failed(self, circuit, reason): @@ -57,7 +59,7 @@ class CustomCircuit(CircuitListenerMixin): circid, d = None, None for x in self.waiting_circuits: if x[0] == circuit.id: - circid, d, stream_cc = x + circid, d = x if d is None: raise Exception("Expected to find circuit.") @@ -71,36 +73,32 @@ class CustomCircuit(CircuitListenerMixin): TorInfo.dump(self) def request_circuit_build(self, deferred): - entries = self.state.entry_guards.value() - relays = self.state.routers.values() - log.msg("We have these nodes listed as entry guards:") - log.msg("%s" % entries) - log.msg("We have these nodes listed as relays:") - log.msg("%s" % relays) - path = [random.choice(entries), - random.choice(relays), - random.choice(relays)] + if self.state.relays_remaining() > 0: + first, middle,last = (self.state.relays.pop() + for i in range(3)) + else: + first = random.choice(self.state.entry_guards.values()) + middle, last = (random.choice(self.state.routers.values()) + for i in range(2)) + path = [first, middle, last] + log.msg("Requesting a circuit: %s" - % '-->'.join(map(lambda x: x.location.countrycode, - path))) + % '->'.join(map(lambda node: node, path))) class AppendWaiting: def __init__(self, attacher, deferred): self.attacher = attacher self.d = deferred - - def __call__(self, circuit): + def __call__(self, circ): """ Return from build_circuit is a Circuit, however, we want to wait until it is built before we can issue an attach on it and callback to the Deferred we issue here. """ - log.msg("Circuit %s is in progress ..." % circuit.id) - self.attacher.waiting_circuits.append((circuit.id, - self.d)) - - fin = self.state.build_circuit(path) - fin.addCallback(AppendWaiting(self, deferred_to_callback)) - fin.addErrback(log.err) - return fin + log.msg("Circuit %s is in progress ..." % circ.id) + self.attacher.waiting_circuits.append((circ.id, self.d)) + + return self.state.build_circuit(path).addCallback( + AppendWaiting(self, deferred_to_callback)).addErrback( + log.err)

1 0

[ooni-probe/master] * Moving to using @defer.inlineCallbacks for rewriting the torrc Bridge line.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 4a6b164cbefb5dca519ece6cfd62486c8a14206e Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 20 03:38:29 2012 +0000 * Moving to using @defer.inlineCallbacks for rewriting the torrc Bridge line. NOT DONE YET. NOT TESTED YET. * Moving to bridges being assets and relays being a list attached to our TorState() instance, so that we can pass it to CustomCircuit. TODO: Make the CustomCircuit() instance handle iteration through relay list, and provide access to objects for unreachable and reachable relays. We should probably check how onionoo handles this. --- ooni/plugins/bridget.py | 134 +++++++++++++++++++++++++++++++---------------- ooni/utils/circuit.py | 18 +++--- 2 files changed, 97 insertions(+), 55 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 192c2dd..07d7269 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -341,55 +341,91 @@ class BridgetTest(OONITest): rmtree(temp, ignore_errors=True) #(a)timer(self.circuit_timeout) + @defer.inlineCallbacks def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): - """Rewrite the Bridge line in our torrc.""" - log.msg("Current Bridge: %s" % bridge) - if use_pt is False: - state.protocol.set_conf('Bridge', bridge) - elif use_pt and pt_type is not None: - state.protocol.set_conf('Bridge', pt_type +' '+ bridge) - else: - raise PTNotFoundException - return state.callback + """ + Rewrite the Bridge line in our torrc. If use of pluggable + transports was specified, rewrite the line as: + Bridge <transport_type> <ip>:<orport> + Otherwise, rewrite in the standard form. + """ + log.msg("Current Bridge: %s" % bridge) + try: + if use_pt is False: + reset_tor = yield state.protocol.set_conf('Bridge', bridge) + elif use_pt and pt_type is not None: + reset_tor = yield state.protocol.set_conf('Bridge', + pt_type +' '+ bridge) + else: + raise PTNotFoundException + + controller_response = reset_tor.callback + if not controller_response: + defer.returnValue((state.callback, None)) + else: + defer.returnValue((state.callback, controller_response)) + except Exception, e: + log.msg("Reconfiguring torrc Bridge line failed with %s" % bridge) def reconfigure_fail(*param): log.msg("Reconfiguring TorConfig with parameters %s failed" % param) reactor.stop() - def remove_relays(state, bridge_list): + @defer.inlineCallbacks + def remove_public_relays(state, bridges): """ Remove bridges from our bridge list which are also listed as public relays. """ - ips = map(lambda addr: addr.split(':',1)[0], bridge_list) - both = set(state.relays.values()).intersection(ips) - if len(both) > 0: - for bridge in both: - bridge_list.remove(bridge) - return state.callback + IPs = map(lambda addr: addr.split(':',1)[0], bridges) + both = set(state.routers.values()).intersection(IPs) - def remove_relays_fail(state): - log.msg("Unable to remove public relays from the bridge list.") + def __remove_line__(node, bridges=bridges): + for line in bridges: + if line.startswith(node): + log.msg("Removing %s because it is a public relay" % node) + bridges.remove(line) - def setup_fail(args): - log.msg("Setup Failed.") - report.update({'failed': args}) + if len(both) > 0: + try: + updated = yield map(lambda node: __remove_line__(node), both) + if not updated: + ## XXX do these need to be state.callback? + defer.returnValue(state) + else: + defer.returnValue(state) + except Exception, e: + log.msg("Unable to remove public relays from bridge list:\n%s" + % both) + log.err(e) + + def setup_fail(proto, bridge_list, relay_list): + log.err("Setup Failed: %s" % proto) + log.err("We were given bridge list:\n%s\nAnd our relay list:\n%s\n" + % (bridge_list, relay_list)) + report.update({'setup_fail': 'FAILED', + 'proto': proto, + 'bridge_list': bridge_list, + 'relay_list': relay_list}) reactor.stop() - def setup_done(proto): + def setup_done(proto, bridge_list, relay_list): log.msg("Setup Complete: %s" % proto) state = TorState(proto.tor_protocol) state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) - report.update({'success': args}) - - def start_tor(reactor, update, torrc, to_delete, control_port, - tor_binary, data_directory): + if bridge_list is not None: + state.post_bootstrap.addCallback(remove_public_relays, bridge_list) + if relay_list is not None: + raise NotImplemented + #report.update({'success': args}) + + def start_tor(reactor, update, torrc, to_delete, control_port, tor_binary, + data_directory, bridge_list=None, relay_list=None): """ Create a TCP4ClientEndpoint at our control_port, and connect it to our reactor and a spawned Tor process. Compare with :meth:`txtorcon.launch_tor` for differences. """ - ## XXX do we need self.reactor? end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port) connection_creator = partial(end_point.connect, TorProtocolFactory()) process_protocol = TorProcessProtocol(connection_creator, updates) @@ -406,9 +442,9 @@ class BridgetTest(OONITest): except RuntimeError, e: process_protocol.connected_cb.errback(e) finally: - return process_protocol.connected_cb + return process_protocol.connected_cb, bridge_list, relay_list - def state_complete(state): + def state_complete(state, bridge_list=None, relay_list=None): """Called when we've got a TorState.""" log.msg("We've completely booted up a Tor version %s at PID %d" % (state.protocol.version, state.tor_pid)) @@ -418,7 +454,12 @@ class BridgetTest(OONITest): for circ in state.circuits.values(): log.msg("%s" % circ) - return state + if bridge_list is not None and relay_list is None: + return state, bridge_list + elif bridge_list is None and relay_list is not None: + raise NotImplemented + else: + return state, None def state_attach(state, relay_list): log.msg("Setting up custom circuit builder...") @@ -426,12 +467,12 @@ class BridgetTest(OONITest): state.set_attacher(attacher, reactor) state.add_circuit_listener(attacher) - for circ in state.circuits.values(): - for relay in circ.path: - try: - relay_list.remove(relay) - except KeyError: - continue + #for circ in state.circuits.values(): + # for relay in circ.path: + # try: + # relay_list.remove(relay) + # except KeyError: + # continue ## XXX how do we attach to circuits with bridges? d = defer.Deferred() attacher.request_circuit_build(d) @@ -457,7 +498,6 @@ class BridgetTest(OONITest): data_dir = mkdtemp(prefix='bridget-tordata') delete_list.append(data_dir) conf.DataDirectory = data_dir - #conf.__OwningControllerProcess = os.getpid() (fd, torrc) = mkstemp(dir=data_dir) delete_list.append(torrc) @@ -468,8 +508,16 @@ class BridgetTest(OONITest): log.msg("Bridget: initiating test ... ") - while self.bridges_remaining() > 0: - self.current_bridge = self.bridges.pop() + #while self.bridges_remaining() > 0: + while args['bridge']: + + #self.current_bridge = self.bridges.pop() + self.current_bridge = args['bridge'] + try: + self.bridges.remove(self.current_bridge) + except ValueError, ve: + log.err(ve) + if self.config.config.has_key('Bridge'): log.msg("We now have %d untested bridges..." % self.bridges_remaining()) @@ -479,9 +527,6 @@ class BridgetTest(OONITest): self.pt_type) reconf.addErrback(reconfigure_fail) state.chainDeferred(reconf) - #reconfigure_bridge(state, self.current_bridge, - # self.use_pt, self.pt_type) - #current_ip = self.current_bridge.split(':', 1)[0] else: self.config.Bridge = self.current_bridge @@ -499,10 +544,7 @@ class BridgetTest(OONITest): state.addCallback(setup_done) state.addErrback(setup_fail) state.addBoth(remove_relays, self.bridges) - #state.addCallback(remove_relays, self.bridges) - #state.addErrback(remove_relays_fail) - #state.addCallback(state_attach, self.bridges) - #state.addErrback(state_attach_fail) + return state ## XXX see txtorcon.TorControlProtocol.add_event_listener we diff --git a/ooni/utils/circuit.py b/ooni/utils/circuit.py index 0c77dd1..6ee0720 100644 --- a/ooni/utils/circuit.py +++ b/ooni/utils/circuit.py @@ -3,27 +3,27 @@ # ---------- # Utilities for working with Tor circuits. # -# This code is largely taken from the txtorcon documentation, and as -# such any and all credit should go to meejah. +# This code is largely taken from attach_streams_by_country.py in the txtorcon +# documentation, and as such any and all credit should go to meejah. Minor +# adjustments have been made to use OONI's logging system, and to build custom +# circuits without actually attaching streams. # -# :author: Mike Warren, Isis Lovecruft +# :author: Meejah, Isis Lovecruft # :license: see included license file # :version: 0.1.0-alpha # -from zope.interface import implements + +import random from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +from ooni.lib.txtorcon import TorInfo from ooni.utils import log - -import random +from zope.interface import implements class CustomCircuit(CircuitListenerMixin): implements(IStreamAttacher) - from txtorcon.interface import IRouterContainer - from txtorcon.interface import ICircuitContainer - def __init__(self, state): self.state = state self.waiting_circuits = []

1 0

[ooni-probe/master] Remove now unused stuff from oonicli
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit b7c5c1fec12da443a30f0cbda1ebb44a8ba7612e Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 20 19:34:48 2012 +0000 Remove now unused stuff from oonicli --- ooni/oonicli.py | 204 +------------------------------------------------------ 1 files changed, 3 insertions(+), 201 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index f0287b4..1e7613b 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -34,68 +34,6 @@ from twisted.python.compat import set from twisted.trial import itrial from twisted.trial import runner as irunner - -def _parseLocalVariables(line): - """ - Accepts a single line in Emacs local variable declaration format and - returns a dict of all the variables {name: value}. - Raises ValueError if 'line' is in the wrong format. - - See http://www.gnu.org/software/emacs/manual/html_node/File-Variables.html - """ - paren = '-*-' - start = line.find(paren) + len(paren) - end = line.rfind(paren) - if start == -1 or end == -1: - raise ValueError("%r not a valid local variable declaration" % (line,)) - items = line[start:end].split(';') - localVars = {} - for item in items: - if len(item.strip()) == 0: - continue - split = item.split(':') - if len(split) != 2: - raise ValueError("%r contains invalid declaration %r" - % (line, item)) - localVars[split[0].strip()] = split[1].strip() - return localVars - - -def loadLocalVariables(filename): - """ - Accepts a filename and attempts to load the Emacs variable declarations - from that file, simulating what Emacs does. - - See http://www.gnu.org/software/emacs/manual/html_node/File-Variables.html - """ - f = file(filename, "r") - lines = [f.readline(), f.readline()] - f.close() - for line in lines: - try: - return _parseLocalVariables(line) - except ValueError: - pass - return {} - - -def getTestModules(filename): - testCaseVar = loadLocalVariables(filename).get('test-case-name', None) - if testCaseVar is None: - return [] - return testCaseVar.split(',') - - -def isTestFile(filename): - """ - Returns true if 'filename' looks like a file containing unit tests. - False otherwise. Doesn't care whether filename exists. - """ - basename = os.path.basename(filename) - return (basename.startswith('test_') - and os.path.splitext(basename)[1] == ('.py')) - - class Options(usage.Options, app.ReactorSelectionMixin): synopsis = """%s [options] [[file|package|module|TestCase|testmethod]...] """ % (os.path.basename(sys.argv[0]),) @@ -104,37 +42,16 @@ class Options(usage.Options, app.ReactorSelectionMixin): "network tests. These are loaded from modules, packages and" "files listed on the command line") - optFlags = [["help", "h"], - ["rterrors", "e", "realtime errors, print out tracebacks as " - "soon as they occur"], - ["debug", "b", "Run tests in the Python debugger. Will load " - "'.pdbrc' from current directory if it exists."], - ["debug-stacktraces", "B", "Report Deferred creation and " - "callback stack traces"], - ["nopm", None, "don't automatically jump into debugger for " - "postmorteming of exceptions"], - ["force-gc", None, "Have OONI run gc.collect() before and " - "after each test case."], - ["unclean-warnings", None, - "Turn dirty reactor errors into warnings"], - ["no-recurse", "N", "Don't recurse into packages"], - ['help-reporters', None, - "Help on available output plugins (reporters)"] - ] + optFlags = [["help", "h"]] optParameters = [ ["reportfile", "o", "report.yaml", "report file name"], ["logfile", "l", "test.log", "log file name"], ['temp-directory', None, '_ooni_temp', - 'Path to use as working directory for tests.'], - ['reporter', None, 'default', - 'The reporter to use for this test run. See --help-reporters for ' - 'more info.']] + 'Path to use as working directory for tests.'] + ] compData = usage.Completions( - optActions={"tbformat": usage.CompleteList(["plain", "emacs", "cgitb"]), - "logfile": usage.CompleteFiles(descr="log file name"), - }, extraActions=[usage.CompleteFiles( "*.py", descr="file | module | package | TestCase | testMethod", repeat=True)], @@ -146,55 +63,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): self['test'] = None usage.Options.__init__(self) - - def coverdir(self): - """ - Return a L{FilePath} representing the directory into which coverage - results should be written. - """ - coverdir = 'coverage' - result = FilePath(self['temp-directory']).child(coverdir) - print "Setting coverage directory to %s." % (result.path,) - return result - - - def opt_coverage(self): - """ - Generate coverage information in the I{coverage} file in the - directory specified by the I{trial-temp} option. - """ - import trace - self.tracer = trace.Trace(count=1, trace=0) - sys.settrace(self.tracer.globaltrace) - - - def opt_testmodule(self, filename): - """ - Filename to grep for test cases (-*- test-case-name) - """ - # If the filename passed to this parameter looks like a test module - # we just add that to the test suite. - # - # If not, we inspect it for an Emacs buffer local variable called - # 'test-case-name'. If that variable is declared, we try to add its - # value to the test suite as a module. - # - # This parameter allows automated processes (like Buildbot) to pass - # a list of files to OONI with the general expectation of "these files, - # whatever they are, will get tested" - if not os.path.isfile(filename): - sys.stderr.write("File %r doesn't exist\n" % (filename,)) - return - - filename = os.path.abspath(filename) - self['test'] = filename - - if isTestFile(filename): - self['tests'].add(filename) - else: - self['tests'].update(getTestModules(filename)) - - def opt_spew(self): """ Print an insanely verbose log of everything that happens. Useful @@ -202,61 +70,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): """ sys.settrace(spewer) - - def opt_help_reporters(self): - synopsis = ("OONI's output can be customized using plugins called " - "Reporters. You can\nselect any of the following " - "reporters using --reporter=<foo>\n") - print synopsis - for p in plugin.getPlugins(itrial.IReporter): - print ' ', p.longOpt, '\t', p.description - print - sys.exit(0) - - - def opt_disablegc(self): - """ - Disable the garbage collector - """ - gc.disable() - - - def opt_tbformat(self, opt): - """ - Specify the format to display tracebacks with. Valid formats are - 'plain', 'emacs', and 'cgitb' which uses the nicely verbose stdlib - cgitb.text function - """ - try: - self['tbformat'] = TBFORMAT_MAP[opt] - except KeyError: - raise usage.UsageError( - "tbformat must be 'plain', 'emacs', or 'cgitb'.") - - - def opt_recursionlimit(self, arg): - """ - see sys.setrecursionlimit() - """ - try: - sys.setrecursionlimit(int(arg)) - except (TypeError, ValueError): - raise usage.UsageError( - "argument to recursionlimit must be an integer") - - - def opt_without_module(self, option): - """ - Fake the lack of the specified modules, separated with commas. - """ - for module in option.split(","): - if module in sys.modules: - warnings.warn("Module '%s' already imported, " - "disabling anyway." % (module,), - category=RuntimeWarning) - sys.modules[module] = None - - def parseArgs(self, *args): try: self['test'] = args[0] @@ -265,18 +78,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): def postOptions(self): - # Only load reporters now, as opposed to any earlier, to avoid letting - # application-defined plugins muck up reactor selecting by importing - # t.i.reactor and causing the default to be installed. self['reporter'] = reporter.OONIReporter - if 'tbformat' not in self: - self['tbformat'] = 'default' - if self['nopm']: - if not self['debug']: - raise usage.UsageError("you must specify --debug when using " - "--nopm ") - failure.DO_POST_MORTEM = False def run(): if len(sys.argv) == 1: @@ -287,7 +90,6 @@ def run(): except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) - file_name = os.path.abspath('nettests/simpletest.py') classes = runner.findTestClassesFromFile(config['test']) casesList, options = runner.loadTestsAndOptions(classes) for idx, cases in enumerate(casesList):

1 0

[ooni-probe/master] * Still working on the @defer.inlineCallbacks structure; I think I should
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit c4978ff0e750d044b3efd9c211dc3cfa80cc2ec4 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 20 14:18:38 2012 +0000 * Still working on the @defer.inlineCallbacks structure; I think I should use DeferredLists for each of the sets of Deferreds, and then pile all the DeferredLists onto TorState(). --- ooni/plugins/bridget.py | 124 ++++++++++++++++++++++++++++++++++++---------- ooni/utils/circuit.py | 35 ++++++++------ 2 files changed, 117 insertions(+), 42 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 07d7269..d82f2e7 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -360,16 +360,25 @@ class BridgetTest(OONITest): raise PTNotFoundException controller_response = reset_tor.callback - if not controller_response: - defer.returnValue((state.callback, None)) + + #if not controller_response: + # defer.returnValue((state.callback, None)) + #else: + # defer.returnValue((state.callback, controller_response)) + if controller_response == 'OK': + defer.returnValue(state.callback) else: - defer.returnValue((state.callback, controller_response)) + log.msg("TorControlProtocol responded with error:\n%s\n" + % controller_response) + defer.returnValue(state.callback) + except Exception, e: - log.msg("Reconfiguring torrc Bridge line failed with %s" % bridge) + log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s\n" + % (bridge, e)) - def reconfigure_fail(*param): - log.msg("Reconfiguring TorConfig with parameters %s failed" % param) - reactor.stop() + def reconfigure_fail(state, bridge, bad): + log.msg("Reconfiguring TorConfig with parameters %s failed" % state) + bad.append(bridge) @defer.inlineCallbacks def remove_public_relays(state, bridges): @@ -399,6 +408,44 @@ class BridgetTest(OONITest): % both) log.err(e) + def request_circuit_build(state, deferred, path=None): + if path is None: + if relays_remaining() > 0: + first, middle,last = (state.relays.pop() + for i in range(3)) + else: + first = random.choice(state.entry_guards.values()) + middle, last = (random.choice(state.routers.values()) + for i in range(2)) + path = [first, middle, last] + else: + assert type(path) is list, "Circuit path must be a list of routers!" + assert len(path) >= 3, "Circuits must be at least three hops!" + + log.msg("Requesting a circuit: %s" + % '->'.join(map(lambda node: node, path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + def __call__(self, circ): + """ + Return from build_circuit is a Circuit, however, + we want to wait until it is built before we can + issue an attach on it and callback to the Deferred + we issue here. + """ + log.msg("Circuit %s is in progress ..." % circ.id) + self.attacher.waiting_circuits.append((circ.id, self.d)) + + return self.state.build_circuit(path).addCallback( + AppendWaiting(self, deferred)).addErrback( + log.err) + + def attacher_extend_circuit(attacher, deferred, router): + + def setup_fail(proto, bridge_list, relay_list): log.err("Setup Failed: %s" % proto) log.err("We were given bridge list:\n%s\nAnd our relay list:\n%s\n" @@ -508,27 +555,17 @@ class BridgetTest(OONITest): log.msg("Bridget: initiating test ... ") - #while self.bridges_remaining() > 0: - while args['bridge']: + while self.bridges_remaining() > 0: + + self.current_bridge = self.bridges.pop() + #self.current_bridge = args['bridge'] - #self.current_bridge = self.bridges.pop() - self.current_bridge = args['bridge'] try: self.bridges.remove(self.current_bridge) except ValueError, ve: log.err(ve) - if self.config.config.has_key('Bridge'): - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - reconf = defer.Deferred() - reconf.addCallback(reconfigure_bridge, state, - self.current_bridge, self.use_pt, - self.pt_type) - reconf.addErrback(reconfigure_fail) - state.chainDeferred(reconf) - - else: + if not self.config.config.has_key('Bridge'): self.config.Bridge = self.current_bridge (torrc, self.data_directory, to_delete) = write_torrc( self.config, self.data_directory) @@ -538,14 +575,48 @@ class BridgetTest(OONITest): % self.config.create_torrc()) report = {'tor_config': self.config.create_torrc()} - state = start_tor(reactor, updates, torrc, to_delete, - self.control_port, self.tor_binary, - self.data_directory) + state = start_tor(reactor, updates, torrc, to_delete, + self.control_port, + self.tor_binary, + self.data_directory, + bridge_list=self.bridges) state.addCallback(setup_done) state.addErrback(setup_fail) - state.addBoth(remove_relays, self.bridges) + state.addCallback(remove_public_relays, self.bridges) + + else: + log.msg("We now have %d untested bridges..." + % self.bridges_remaining()) + reconf = defer.Deferred() + reconf.addCallback(reconfigure_bridge, state, + self.current_bridge, self.use_pt, + self.pt_type) + reconf.addErrback(reconfigure_fail, state, + self.current_bridge, self.bridges_down) + state.chainDeferred(reconf) + #all = [] + #reconf = reconfigure_bridge(state, self.current_bridge, + # self.use_pt, self.pt_type) + #reconf.addCallback(reconfigure_done) + #reconf.addErrback(reconfigure_fail) + #state.DeferredList(all) + + while self.relays_remaining() >= 3: + #path = list(self.relays.pop() for i in range(3)) + #log.msg("Trying path %s" % '->'.join(map(lambda node: node, path))) + self.current_relay = self.relays.pop() + for circ in state.circuits.values(): + for node in circ.path: + if node == self.current_relay: + self.relays_up.append(self.current_relay) + try: + ext = attacher_extend_circuit(state.attacher, circ, + self.current_relay) + return state + ## still need to attach attacher to state + ## then build circuits ## XXX see txtorcon.TorControlProtocol.add_event_listener we ## may not need full CustomCircuit class @@ -569,7 +640,6 @@ class BridgetTest(OONITest): # self.bridges_up.append(self.current_bridge) reactor.run() - ## now build circuits ## So that getPlugins() can register the Test: diff --git a/ooni/utils/circuit.py b/ooni/utils/circuit.py index 6ee0720..d725879 100644 --- a/ooni/utils/circuit.py +++ b/ooni/utils/circuit.py @@ -9,7 +9,7 @@ # circuits without actually attaching streams. # # :author: Meejah, Isis Lovecruft -# :license: see included license file +# :license: see included LICENSE file # :version: 0.1.0-alpha # @@ -24,9 +24,10 @@ from zope.interface import implements class CustomCircuit(CircuitListenerMixin): implements(IStreamAttacher) - def __init__(self, state): + def __init__(self, state, relays=None): self.state = state self.waiting_circuits = [] + self.relays = relays def waiting_on(self, circuit): for (circid, d) in self.waiting_circuits: @@ -54,12 +55,12 @@ class CustomCircuit(CircuitListenerMixin): def circuit_failed(self, circuit, reason): if self.waiting_on(circuit): - log.msg("A circuit we requested %s failed for reason %s" + log.msg("Circuit %s failed for reason %s" % (circuit.id, reason)) circid, d = None, None - for x in self.waiting_circuits: - if x[0] == circuit.id: - circid, d = x + for c in self.waiting_circuits: + if c[0] == circuit.id: + circid, d = c if d is None: raise Exception("Expected to find circuit.") @@ -72,15 +73,19 @@ class CustomCircuit(CircuitListenerMixin): #router.update() ## XXX can i use without args? no. TorInfo.dump(self) - def request_circuit_build(self, deferred): - if self.state.relays_remaining() > 0: - first, middle,last = (self.state.relays.pop() - for i in range(3)) + def request_circuit_build(self, deferred, path=None): + if path is None: + if self.state.relays_remaining() > 0: + first, middle,last = (self.state.relays.pop() + for i in range(3)) + else: + first = random.choice(self.state.entry_guards.values()) + middle, last = (random.choice(self.state.routers.values()) + for i in range(2)) + path = [first, middle, last] else: - first = random.choice(self.state.entry_guards.values()) - middle, last = (random.choice(self.state.routers.values()) - for i in range(2)) - path = [first, middle, last] + assert type(path) is list, "Circuit path must be a list of routers!" + assert len(path) >= 3, "Circuits must be at least three hops!" log.msg("Requesting a circuit: %s" % '->'.join(map(lambda node: node, path))) @@ -100,5 +105,5 @@ class CustomCircuit(CircuitListenerMixin): self.attacher.waiting_circuits.append((circ.id, self.d)) return self.state.build_circuit(path).addCallback( - AppendWaiting(self, deferred_to_callback)).addErrback( + AppendWaiting(self, deferred)).addErrback( log.err)

1 0