October 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Implement working trial inspired OONIprobe refactoring
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit cbfcbdd0344d8e9a80d565ba01b64c7a40c97352 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 20 19:26:46 2012 +0000 Implement working trial inspired OONIprobe refactoring --- ooni/nettest.py | 41 +++---- ooni/oonicli.py | 66 ++++------- ooni/plugoo/tests.py | 1 + ooni/runner.py | 330 ++++++++++++------------------------------------- 4 files changed, 122 insertions(+), 316 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 4ab1e0f..0302d25 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -1,5 +1,4 @@ import itertools - from twisted.python import log from twisted.trial import unittest, itrial @@ -42,35 +41,29 @@ class TestSuiteFactory(object): self._idx += 1 return new_test_suite -class TestSuite(pyunit.TestSuite): - def __init__(self, tests=()): - self._tests = [] - self.input = None - self._idx = 0 - self.addTests(tests) - - def __repr__(self): - return "<%s input=%s tests=%s>" % (self.__class__, - self.input, self._tests) - - def run(self, result): - """ - Call C{run} on every member of the suite. - """ - # we implement this because Python 2.3 unittest defines this code - # in __call__, whereas 2.4 defines the code in run. - for i, test in enumerate(self._tests): +class InputTestSuite(pyunit.TestSuite): + def run(self, result, idx=0): + self._idx = idx + while self._tests: if result.shouldStop: break - test.input = self.input - test._idx = self._idx + i - test(result) - + test = self._tests.pop(0) + try: + test.input = self.input + test._idx = self._idx + print "IDX: %s" % self._idx + test(result) + except: + test(result) + self._idx += 1 return result class TestCase(unittest.TestCase): name = "DefaultTestName" - inputs = [None] + inputs = [] + + def getOptions(self): + return {'inputs': self.inputs} def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index be73d30..f0287b4 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -15,6 +15,15 @@ import sys, os, random, gc, time, warnings +import unittest +import inspect + +from ooni.input import InputUnitFactory +from ooni.reporter import ReporterFactory +from ooni.nettest import InputTestSuite +from ooni.plugoo import tests +from ooni import nettest, runner, reporter + from twisted.internet import defer from twisted.application import app from twisted.python import usage, reflect, failure, log @@ -24,7 +33,6 @@ from twisted.python.util import spewer from twisted.python.compat import set from twisted.trial import itrial from twisted.trial import runner as irunner -from ooni import runner, reporter def _parseLocalVariables(line): @@ -132,11 +140,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): repeat=True)], ) - fallbackReporter = reporter.OONIReporter tracer = None def __init__(self): - self['tests'] = set() + self['test'] = None usage.Options.__init__(self) @@ -178,7 +185,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): if not os.path.isfile(filename): sys.stderr.write("File %r doesn't exist\n" % (filename,)) return + filename = os.path.abspath(filename) + self['test'] = filename + if isTestFile(filename): self['tests'].add(filename) else: @@ -248,7 +258,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): - self['tests'].update(args) + try: + self['test'] = args[0] + except: + raise usage.UsageError("No test filename specified!") def postOptions(self): @@ -265,39 +278,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): "--nopm ") failure.DO_POST_MORTEM = False - -def _initialDebugSetup(config): - # do this part of debug setup first for easy debugging of import failures - if config['debug']: - failure.startDebugMode() - if config['debug'] or config['debug-stacktraces']: - defer.setDebugging(True) - - -def _getSuitesAndInputs(config): - #loader = irunner.TestLoader() - loader = runner.NetTestLoader() - recurse = not config['no-recurse'] - print "loadByNames %s" % config['tests'] - inputs, suites = loader.loadByNamesWithInput(config['tests'], recurse) - return inputs, suites - -def _makeRunner(config): - mode = None - if config['debug']: - mode = runner.OONIRunner.DEBUG - print "using %s" % config['reporter'] - return runner.OONIRunner(config['reporter'], - reportfile=config["reportfile"], - mode=mode, - logfile=config['logfile'], - tracebackFormat=config['tbformat'], - realTimeErrors=config['rterrors'], - uncleanWarnings=config['unclean-warnings'], - workingDirectory=config['temp-directory'], - forceGarbageCollection=config['force-gc']) - - def run(): if len(sys.argv) == 1: sys.argv.append("--help") @@ -307,10 +287,10 @@ def run(): except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) - _initialDebugSetup(config) - trialRunner = _makeRunner(config) - inputs, testSuites = _getSuitesAndInputs(config) - log.startLogging(sys.stdout) - for i, suite in enumerate(testSuites): - test_result = trialRunner.run(suite, inputs[i]) + file_name = os.path.abspath('nettests/simpletest.py') + classes = runner.findTestClassesFromFile(config['test']) + casesList, options = runner.loadTestsAndOptions(classes) + for idx, cases in enumerate(casesList): + orunner = runner.ORunner(cases, options[idx]) + orunner.run() diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 2b1e87c..e4ee187 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -133,6 +133,7 @@ class OONITest(object): @param args: the asset(s) lines that we are working on. """ self.start_time = date.now() + print "FOWID" log.msg("Starting test %s" % self.__class__) return self._do_experiment(args) diff --git a/ooni/runner.py b/ooni/runner.py index 604942d..ddde83b 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -13,9 +13,16 @@ from twisted.trial.runner import filenameToModule, _importFromFile from ooni.reporter import ReporterFactory from ooni.input import InputUnitFactory +from ooni.nettest import InputTestSuite from ooni import nettest from ooni.plugoo import tests as oonitests +def isTestCase(thing): + try: + return issubclass(thing, unittest.TestCase) + except TypeError: + return False + def isLegacyTest(obj): """ Returns True if the test in question is written using the OONITest legacy @@ -23,7 +30,10 @@ def isLegacyTest(obj): We do this for backward compatibility of the OONIProbe API. """ try: - return issubclass(obj, oonitests.OONITest) + if issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest: + return True + else: + return False except TypeError: return False @@ -36,267 +46,89 @@ def adaptLegacyTest(obj, inputs=[None]): older test cases compatible with the new OONI. """ class LegacyOONITest(nettest.TestCase): - inputs = [1] + inputs = [None] original_test = obj + @defer.inlineCallbacks def test_start_legacy_test(self): print "bla bla bla" - my_test = self.original_test() - print my_test - print "foobat" - my_test.startTest(self.input) - print "HHAHAHA" + print self.original_test + my_test = self.original_test(None, None, None) + yield my_test.startTest(self.input) return LegacyOONITest -class LoggedSuite(nettest.TestSuite): - """ - Any errors logged in this suite will be reported to the L{TestResult} - object. - """ - - def run(self, result): - """ - Run the suite, storing all errors in C{result}. If an error is logged - while no tests are running, then it will be added as an error to - C{result}. - - @param result: A L{TestResult} object. - """ - observer = unittest._logObserver - observer._add() - super(LoggedSuite, self).run(result) - observer._remove() - for error in observer.getErrors(): - result.addError(TestHolder(NOT_IN_TEST), error) - observer.flushErrors() +def findTestClassesFromFile(filename): + classes = [] + print "FILENAME %s" % filename + module = filenameToModule(filename) + for name, val in inspect.getmembers(module): + if isTestCase(val): + classes.append(val) + elif isLegacyTest(val): + classes.append(adaptLegacyTest(val)) + return classes -class OONISuite(nettest.TestSuite): - """ - Suite to wrap around every single test in a C{trial} run. Used internally - by OONI to set up things necessary for OONI tests to work, regardless of - what context they are run in. - """ +def makeTestCases(klass, tests, methodPrefix): + cases = [] + for test in tests: + cases.append(klass(methodPrefix+test)) + return cases - def __init__(self, tests=()): - suite = LoggedSuite(tests) - super(OONISuite, self).__init__([suite]) - - def _bail(self): - from twisted.internet import reactor - d = defer.Deferred() - reactor.addSystemEventTrigger('after', 'shutdown', - lambda: d.callback(None)) - reactor.fireSystemEvent('shutdown') # radix's suggestion - # As long as TestCase does crap stuff with the reactor we need to - # manually shutdown the reactor here, and that requires util.wait - # :( - # so that the shutdown event completes - nettest.TestCase('mktemp')._wait(d) - - def run(self, result): - try: - nettest.TestSuite.run(self, result) - finally: - self._bail() - - -class NetTestLoader(TestLoader): - """ - Reponsible for finding the modules that can work as tests and running them. - If we detect that a certain test is written using the legacy OONI API we - will wrap it around a next gen class to make it work here too. - - XXX This class needs to be cleaned up a *lot* of all the things we actually - don't need. - """ +def loadTestsAndOptions(classes): methodPrefix = 'test' - modulePrefix = 'test_' - - def __init__(self): - self.suiteFactory = nettest.TestSuite - self._importErrors = [] - - def findTestClasses(self, module): - classes = [] - for name, val in inspect.getmembers(module): - if isTestCase(val): - classes.append(val) - # This is here to allow backward compatibility with legacy OONI - # tests. - elif isLegacyTest(val): - print "adapting! %s" % val - val = adaptLegacyTest(val) - classes.append(val) - return classes - - def loadClass(self, klass): - """ - Given a class which contains test cases, return a sorted list of - C{TestCase} instances. - """ - if not (isinstance(klass, type) or isinstance(klass, types.ClassType)): - raise TypeError("%r is not a class" % (klass,)) - if not isTestCase(klass): - raise ValueError("%r is not a test case" % (klass,)) - names = self.getTestCaseNames(klass) - tests = [] - for name in names: - tests.append(self._makeCase(klass, self.methodPrefix+name)) - - suite = self.suiteFactory(tests) - print "**+*" - print tests - print "**+*" - - return suite - loadTestsFromTestCase = loadClass - - def findAllInputs(self, thing): - testClasses = self.findTestClasses(thing) - # XXX will there ever be more than 1 test class with inputs? - for klass in testClasses: - try: - inputs = klass.inputs - except: - pass - return inputs - - def loadByNamesWithInput(self, names, recurse=False): - """ - Construct a OONITestSuite containing all the tests found in 'names', where - names is a list of fully qualified python names and/or filenames. The - suite returned will have no duplicate tests, even if the same object - is named twice. - - This test suite will have set the attribute inputs to the inputs found - inside of the tests. - """ - inputs = [] - things = [] - errors = [] - for name in names: - try: - thing = self.findByName(name) - things.append(thing) - except: - errors.append(ErrorHolder(name, failure.Failure())) - suites = [] - for thing in self._uniqueTests(things): - inputs.append(self.findAllInputs(thing)) - suite = self.loadAnything(thing, recurse) - suites.append(suite) - - suites.extend(errors) - return inputs, suites - -class OONIRunner(object): - """ - A specialised runner that is used by the ooniprobe frontend to run tests. - Heavily inspired by the trial TrialRunner class. - """ - - DEBUG = 'debug' - DRY_RUN = 'dry-run' - - def _getDebugger(self): - dbg = pdb.Pdb() + suiteFactory = InputTestSuite + options = [] + testCases = [] + for klass in classes: try: - import readline - except ImportError: - print "readline module not available" - sys.exc_clear() - for path in ('.pdbrc', 'pdbrc'): - if os.path.exists(path): - try: - rcFile = file(path, 'r') - except IOError: - sys.exc_clear() - else: - dbg.rcLines.extend(rcFile.readlines()) - return dbg - - - def _setUpTestdir(self): - self._tearDownLogFile() - currentDir = os.getcwd() - base = filepath.FilePath(self.workingDirectory) - testdir, self._testDirLock = util._unusedTestDirectory(base) - os.chdir(testdir.path) - return currentDir - - - def _tearDownTestdir(self, oldDir): - os.chdir(oldDir) - self._testDirLock.unlock() - - - _log = log - def _makeResult(self): - reporter = self.reporterFactory(self.stream, self.tbformat, - self.rterrors, self._log) - if self.uncleanWarnings: - reporter = UncleanWarningsReporterWrapper(reporter) - return reporter - - def __init__(self, reporterFactory, - reportfile="report.yaml", - mode=None, - logfile='test.log', - stream=sys.stdout, - profile=False, - tracebackFormat='default', - realTimeErrors=False, - uncleanWarnings=False, - workingDirectory=None, - forceGarbageCollection=False): - self.reporterFactory = reporterFactory - self._reportfile = reportfile - self.logfile = logfile - self.mode = mode - self.stream = stream - self.tbformat = tracebackFormat - self.rterrors = realTimeErrors - self.uncleanWarnings = uncleanWarnings - self._result = None - self.workingDirectory = workingDirectory or '_trial_temp' - self._logFileObserver = None - self._logFileObject = None - self._forceGarbageCollection = forceGarbageCollection - if profile: - self.run = util.profiled(self.run, 'profile.data') - - def _tearDownLogFile(self): - if self._logFileObserver is not None: - log.removeObserver(self._logFileObserver.emit) - self._logFileObserver = None - if self._logFileObject is not None: - self._logFileObject.close() - self._logFileObject = None - - def _setUpLogFile(self): - self._tearDownLogFile() - if self.logfile == '-': - logFile = sys.stdout + k = klass() + options.append(k.getOptions()) + except AttributeError: + options.append([]) + + tests = reflect.prefixedMethodNames(klass, methodPrefix) + if tests: + cases = makeTestCases(klass, tests, methodPrefix) + testCases.append(cases) else: - logFile = file(self.logfile, 'a') - self._logFileObject = logFile - self._logFileObserver = log.FileLogObserver(logFile) - log.startLoggingWithObserver(self._logFileObserver.emit, 0) + options.pop() + + return testCases, options + +class ORunner(object): + def __init__(self, cases, options=None): + self.baseSuite = InputTestSuite + self.cases = cases + self.options = options + self.inputs = options['inputs'] + self.reporterFactory = ReporterFactory(open('foo.log', 'a+'), + testSuite=self.baseSuite(self.cases)) + + def runWithInputUnit(self, inputUnit): + idx = 0 + result = self.reporterFactory.create() + for input in inputUnit: + suite = self.baseSuite(self.cases) + suite.input = input + suite(result, idx) + + # XXX refactor all of this index bullshit to avoid having to pass + # this index around. Probably what I want to do is go and make + # changes to report to support the concept of having multiple runs + # of the same test. + # We currently need to do this addition in order to get the number + # of times the test cases that have run inside of the test suite. + idx += (suite._idx - idx) + + result.done() + + def run(self): + self.reporterFactory.writeHeader() + + for inputUnit in InputUnitFactory(self.inputs): + self.runWithInputUnit(inputUnit) - def run(self, tests, inputs=[None]): - """ - Run the test or suite and return a result object. - """ - reporterFactory = ReporterFactory(open(self._reportfile, 'a+'), - testSuite=tests) - reporterFactory.writeHeader() - for inputUnit in InputUnitFactory(inputs): - testSuiteFactory = nettest.TestSuiteFactory(inputUnit, tests, nettest.TestSuite) - testUnitReport = reporterFactory.create() - for suite in testSuiteFactory: - suite(testUnitReport) - testUnitReport.done()

1 0

[ooni-probe/master] Remove log file
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 7150e97e4e6237777ff1d343ade7953dea0af413 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 21 11:41:51 2012 +0000 Remove log file --- ooni/ooniprobe.log | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/ooni/ooniprobe.log b/ooni/ooniprobe.log deleted file mode 100644 index 700e820..0000000 --- a/ooni/ooniprobe.log +++ /dev/null @@ -1 +0,0 @@ -2012-05-31 02:27:28,010 ooniprobe INFO Started ooni-probe

1 0

[ooni-probe/master] Implement backward compatibility of new ooniprobe with legacy tests
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 7643541a424655c689e252783a50037846368773 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 21 10:40:41 2012 +0000 Implement backward compatibility of new ooniprobe with legacy tests * Reporting system works with old tests * Properly document some of the functions --- ooni/input.py | 17 +++++------- ooni/nettest.py | 44 ++++----------------------------- ooni/oonicli.py | 8 ++++-- ooni/plugins/httpt.py | 9 ++++--- ooni/plugoo/tests.py | 7 ++--- ooni/reporter.py | 30 ++++++++++++++++++++--- ooni/runner.py | 64 +++++++++++++++++++++++++++++++++++++++---------- 7 files changed, 102 insertions(+), 77 deletions(-) diff --git a/ooni/input.py b/ooni/input.py index b931b82..69507b4 100644 --- a/ooni/input.py +++ b/ooni/input.py @@ -9,7 +9,7 @@ class InputUnitFactory(object): """ inputUnitSize = 3 def __init__(self, inputs=[]): - self._inputs = inputs + self._inputs = iter(inputs) self._idx = 0 self._ended = False @@ -21,16 +21,13 @@ class InputUnitFactory(object): raise StopIteration last_element_idx = self._idx + self.inputUnitSize - input_unit_elements = self._inputs[self._idx:last_element_idx] - try: - # XXX hack to fail when we reach the end of the list - antani = self._inputs[last_element_idx] - except: - if len(input_unit_elements) > 0: + input_unit_elements = [] + for i in xrange(last_element_idx): + try: + input_unit_elements.append(self._inputs.next()) + except: self._ended = True - return InputUnit(input_unit_elements) - else: - raise StopIteration + break self._idx += self.inputUnitSize diff --git a/ooni/nettest.py b/ooni/nettest.py index 0302d25..f3476cf 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -4,44 +4,11 @@ from twisted.trial import unittest, itrial pyunit = __import__('unittest') -def _iterateTests(testSuiteOrCase): +class InputTestSuite(pyunit.TestSuite): """ - Iterate through all of the test cases in C{testSuiteOrCase}. + This in an extension of a unittest test suite. It adds support for inputs + and the tracking of current index via idx. """ - try: - suite = iter(testSuiteOrCase) - except TypeError: - yield testSuiteOrCase - else: - for test in suite: - for subtest in _iterateTests(test): - yield subtest - - -class TestSuiteFactory(object): - def __init__(self, inputUnit, tests, basesuite): - self._baseSuite = basesuite - self._inputUnit = inputUnit - self._idx = 0 - self.tests = tests - - def __iter__(self): - return self - - def next(self): - try: - next_input = self._inputUnit.next() - print "Now dealing with %s %s" % (next_input, self._idx) - except: - raise StopIteration - new_test_suite = self._baseSuite(self.tests) - new_test_suite.input = next_input - new_test_suite._idx = self._idx - - self._idx += 1 - return new_test_suite - -class InputTestSuite(pyunit.TestSuite): def run(self, result, idx=0): self._idx = idx while self._tests: @@ -51,7 +18,6 @@ class InputTestSuite(pyunit.TestSuite): try: test.input = self.input test._idx = self._idx - print "IDX: %s" % self._idx test(result) except: test(result) @@ -59,8 +25,8 @@ class InputTestSuite(pyunit.TestSuite): return result class TestCase(unittest.TestCase): - name = "DefaultTestName" - inputs = [] + name = "DefaultOONITestCase" + inputs = [None] def getOptions(self): return {'inputs': self.inputs} diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 1e7613b..f14bf6e 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -73,10 +73,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): try: self['test'] = args[0] + self['subArgs'] = args[1:] except: raise usage.UsageError("No test filename specified!") - def postOptions(self): self['reporter'] = reporter.OONIReporter @@ -90,9 +90,11 @@ def run(): except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) - classes = runner.findTestClassesFromFile(config['test']) + classes = runner.findTestClassesFromConfig(config) + casesList, options = runner.loadTestsAndOptions(classes) for idx, cases in enumerate(casesList): - orunner = runner.ORunner(cases, options[idx]) + orunner = runner.ORunner(cases, options[idx], config) orunner.run() + diff --git a/ooni/plugins/httpt.py b/ooni/plugins/httpt.py index 46f3b17..358f1ea 100644 --- a/ooni/plugins/httpt.py +++ b/ooni/plugins/httpt.py @@ -65,13 +65,14 @@ class httptTest(http.HTTPTest): def processRedirect(self, location): self.result['redirect'] = None - if self.local_options['rules']: + try: + rules_file = self.local_options['rules'] import yaml - rules = yaml.load(open(self.local_options['rules'])) + rules = yaml.load(open(rules_file)) log.msg("Testing rules %s" % rules) redirect = self.testRules(rules, location) self.result['redirect'] = redirect - else: + except TypeError: log.msg("No rules file. Got a redirect, but nothing to do.") @@ -90,4 +91,4 @@ class httptTest(http.HTTPTest): # We need to instantiate it otherwise getPlugins does not detect it # XXX Find a way to load plugins without instantiating them. -httpt = httptTest(None, None, None) +#httpt = httptTest(None, None, None) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index e4ee187..37e0664 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -25,19 +25,19 @@ class OONITest(object): name = "oonitest" # By default we set this to False, meaning that we don't block blocking = False - reactor = None + reactor = reactor tool = False ended = False def __init__(self, local_options, global_options, report, ooninet=None, - reactor=None): + my_reactor=reactor): # These are the options that are read through the tests suboptions self.local_options = local_options # These are the options global to all of OONI self.global_options = global_options self.report = report #self.ooninet = ooninet - self.reactor = reactor + self.reactor = my_reactor self.result = {} self.initialize() @@ -133,7 +133,6 @@ class OONITest(object): @param args: the asset(s) lines that we are working on. """ self.start_time = date.now() - print "FOWID" log.msg("Starting test %s" % self.__class__) return self._do_experiment(args) diff --git a/ooni/reporter.py b/ooni/reporter.py index 14297cd..4b3ed6f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -11,6 +11,10 @@ from twisted.trial import unittest, reporter, runner pyunit = __import__('unittest') class OReporter(pyunit.TestResult): + """ + This is an extension of the unittest TestResult. It adds support for + reporting to yaml format. + """ def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, publisher=None, testSuite=None): super(OReporter, self).__init__() @@ -45,6 +49,10 @@ class OReporter(pyunit.TestResult): class ReporterFactory(OReporter): + """ + This is a reporter factory. It emits new instances of Reports. It is also + responsible for writing the OONI Report headers. + """ def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, publisher=None, testSuite=None): super(ReporterFactory, self).__init__(stream=stream, @@ -77,6 +85,14 @@ class ReporterFactory(OReporter): class OONIReporter(OReporter): + """ + This is a special reporter that has knowledge about the fact that there can + exist more test runs of the same kind per run. + These multiple test runs are kept track of through idx. + + An instance of such reporter should be created per InputUnit. Every input + unit will invoke size_of_input_unit * test_cases times startTest(). + """ def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, publisher=None): super(OONIReporter, self).__init__(stream=stream, @@ -119,10 +135,16 @@ class OONIReporter(OReporter): idx = self.getTestIndex(test) self._tests[idx]['lastTime'] = self._getTime() - self._tests[idx]['testStarted'] - # XXX I put a dict() here so that the object is re-instantiated and I - # actually end up with the report I want. This could either be a - # python bug or a yaml bug. - self._tests[idx]['report'] = dict(test.report) + # This is here for allowing reporting of legacy tests. + # XXX In the future this should be removed. + try: + self._tests[idx]['report'] = list(test.legacy_report) + except: + # XXX I put a dict() here so that the object is re-instantiated and I + # actually end up with the report I want. This could either be a + # python bug or a yaml bug. + self._tests[idx]['report'] = dict(test.report) + def done(self): """ diff --git a/ooni/runner.py b/ooni/runner.py index ddde83b..0bcafb0 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -15,6 +15,7 @@ from ooni.reporter import ReporterFactory from ooni.input import InputUnitFactory from ooni.nettest import InputTestSuite from ooni import nettest +from ooni.utils import log from ooni.plugoo import tests as oonitests def isTestCase(thing): @@ -37,7 +38,7 @@ def isLegacyTest(obj): except TypeError: return False -def adaptLegacyTest(obj, inputs=[None]): +def adaptLegacyTest(obj, config): """ We take a legacy OONITest class and convert it into a nettest.TestCase. This allows backward compatibility of old OONI tests. @@ -45,30 +46,62 @@ def adaptLegacyTest(obj, inputs=[None]): XXX perhaps we could implement another extra layer that makes the even older test cases compatible with the new OONI. """ + class legacy_reporter(object): + def __init__(self, report_target): + self.report_target = report_target + + def __call__(self, what): + self.report_target.append(what) + class LegacyOONITest(nettest.TestCase): - inputs = [None] - original_test = obj + try: + name = obj.shortName + except: + name = "LegacyOONITest" + + originalTest = obj + + subOptions = obj.options() + subOptions.parseOptions(config['subArgs']) + + test_class = obj(None, None, None, None) + test_class.local_options = subOptions + assets = test_class.load_assets() + + # XXX here we are only taking assets that are set to one item only. + for key, inputs in assets.items(): + pass + + inputs = inputs + local_options = subOptions @defer.inlineCallbacks def test_start_legacy_test(self): - print "bla bla bla" - print self.original_test - my_test = self.original_test(None, None, None) - yield my_test.startTest(self.input) + + self.legacy_report = [] + + my_test = self.originalTest(None, None, None) + my_test.report = legacy_reporter(self.legacy_report) + args = {} + args[self.key] = self.input + result = yield my_test.startTest(args) + print "Finished!" + print result return LegacyOONITest -def findTestClassesFromFile(filename): +def findTestClassesFromConfig(config): + filename = config['test'] + classes = [] - print "FILENAME %s" % filename module = filenameToModule(filename) for name, val in inspect.getmembers(module): if isTestCase(val): classes.append(val) elif isLegacyTest(val): - classes.append(adaptLegacyTest(val)) + classes.append(adaptLegacyTest(val, config)) return classes def makeTestCases(klass, tests, methodPrefix): @@ -99,13 +132,17 @@ def loadTestsAndOptions(classes): return testCases, options class ORunner(object): - def __init__(self, cases, options=None): + def __init__(self, cases, options=None, config=None): self.baseSuite = InputTestSuite self.cases = cases self.options = options self.inputs = options['inputs'] - self.reporterFactory = ReporterFactory(open('foo.log', 'a+'), - testSuite=self.baseSuite(self.cases)) + try: + reportFile = open(config['reportfile'], 'a+') + except: + reportFile = open('report.yaml', 'a+') + self.reporterFactory = ReporterFactory(reportFile, + testSuite=self.baseSuite(self.cases)) def runWithInputUnit(self, inputUnit): idx = 0 @@ -126,6 +163,7 @@ class ORunner(object): result.done() def run(self): + log.start() self.reporterFactory.writeHeader() for inputUnit in InputUnitFactory(self.inputs):

1 0

[ooni-probe/master] Make the ascii art thing only be a special command line option
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 05b8d5b809db6ee34ee76c9e0c1a67d163b586ef Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 21 12:14:44 2012 +0000 Make the ascii art thing only be a special command line option --- ooni/oonicli.py | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index f14bf6e..e5c990f 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -42,7 +42,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): "network tests. These are loaded from modules, packages and" "files listed on the command line") - optFlags = [["help", "h"]] + optFlags = [["help", "h"], + ] optParameters = [ ["reportfile", "o", "report.yaml", "report file name"], @@ -63,6 +64,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): self['test'] = None usage.Options.__init__(self) + def opt_asciilulz(self): + from ooni.utils import logo + print logo.getlogo() + def opt_spew(self): """ Print an insanely verbose log of everything that happens. Useful @@ -73,6 +78,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): try: self['test'] = args[0] + self['subArgs'] = args[1:] except: raise usage.UsageError("No test filename specified!") @@ -97,4 +103,3 @@ def run(): orunner = runner.ORunner(cases, options[idx], config) orunner.run() -

1 0

[ooni-probe/master] Write some more docstrings and fix some code.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 26f7afc073cb86276a04d1faeec65dde59dd858f Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 21 12:05:17 2012 +0000 Write some more docstrings and fix some code. --- ooni/protocols/http.py | 4 ++-- ooni/reporter.py | 6 ++++-- ooni/runner.py | 23 +++++++++++++++++++++-- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py index 09bb9b9..569c382 100644 --- a/ooni/protocols/http.py +++ b/ooni/protocols/http.py @@ -56,7 +56,7 @@ class HTTPTest(OONITest): def _processResponseBody(self, data): self.response['body'] = data - #self.result['response'] = self.response + self.result['response'] = self.response self.processResponseBody(data) def processResponseBody(self, data): @@ -127,7 +127,7 @@ class HTTPTest(OONITest): if self.randomize_ua: self.randomize_useragent() - #self.result['request'] = self.request + self.result['request'] = self.request self.result['url'] = url return self.agent.request(self.request['method'], self.request['url'], Headers(self.request['headers']), diff --git a/ooni/reporter.py b/ooni/reporter.py index 4b3ed6f..b145e87 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -138,12 +138,14 @@ class OONIReporter(OReporter): # This is here for allowing reporting of legacy tests. # XXX In the future this should be removed. try: - self._tests[idx]['report'] = list(test.legacy_report) + report = list(test.legacy_report) except: # XXX I put a dict() here so that the object is re-instantiated and I # actually end up with the report I want. This could either be a # python bug or a yaml bug. - self._tests[idx]['report'] = dict(test.report) + report = dict(test.report) + + self._tests[idx]['report'] = report def done(self): diff --git a/ooni/runner.py b/ooni/runner.py index 0bcafb0..ff6b47e 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -85,13 +85,18 @@ def adaptLegacyTest(obj, config): args = {} args[self.key] = self.input result = yield my_test.startTest(args) - print "Finished!" - print result + self.report['result'] = result return LegacyOONITest def findTestClassesFromConfig(config): + """ + Takes as input the command line config parameters and returns the test + case classes. + If it detects that a certain test class is using the old OONIProbe format, + then it will adapt it to the new testing system. + """ filename = config['test'] classes = [] @@ -105,12 +110,20 @@ def findTestClassesFromConfig(config): return classes def makeTestCases(klass, tests, methodPrefix): + """ + Takes a class some tests and returns the test cases. methodPrefix is how + the test case functions should be prefixed with. + """ cases = [] for test in tests: cases.append(klass(methodPrefix+test)) return cases def loadTestsAndOptions(classes): + """ + Takes a list of classes and returnes their testcases and options. + Legacy tests will be adapted. + """ methodPrefix = 'test' suiteFactory = InputTestSuite options = [] @@ -132,6 +145,12 @@ def loadTestsAndOptions(classes): return testCases, options class ORunner(object): + """ + This is a specialized runner used by the ooniprobe command line tool. + I am responsible for reading the inputs from the test files and splitting + them in input units. I also create all the report instances required to run + the tests. + """ def __init__(self, cases, options=None, config=None): self.baseSuite = InputTestSuite self.cases = cases

1 0

[ooni-probe/master] Document TestCase
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 839564d3620c24e0057c1dfd9d39ef15f8b202ac Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 21 13:07:15 2012 +0000 Document TestCase --- ooni/nettest.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 53 insertions(+), 1 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index f3476cf..2a02a43 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -24,11 +24,63 @@ class InputTestSuite(pyunit.TestSuite): self._idx += 1 return result +def lineByLine(fp): + for x in fp.readlines(): + yield x.strip() + fp.close() + class TestCase(unittest.TestCase): - name = "DefaultOONITestCase" + """ + This is the monad of the OONI nettest universe. When you write a nettest + you will subclass this object. + + _inputs_ can be set to a static set of inputs. All the tests (the methods + starting with the "test_" prefix) will be run once per input. At every run + the _input_ attribute of the TestCase instance will be set to the value of + the current iteration over inputs. Any python iterable object can be set + to inputs. + + _inputFile_ attribute should be set to an array containing the command line + argument that should be used as the input file. Such array looks like this: + + ["commandlinearg", "c", "The description"] + + The second value of such arrray is the shorthand for the command line arg. + The user will then be able to specify inputs to the test via: + + ooniprobe mytest.py --commandlinearg path/to/file.txt + + or + + ooniprobe mytest.py -c path/to/file.txt + + + _inputProcessor_ should be set to a function that takes as argument an + open file descriptor and it will yield the input to be passed to the test + instance. + + _name_ should be set to the name of the test. + + _author_ should contain the name and contact details for the test author. + The format for such string is as follows: + + The Name <email(a)example.com> + + _version_ is the version string of the test. + """ + name = "IDidNotChangeTheName" + author = "John Doe <foo(a)example.com>" + version = "0" + inputs = [None] + inputFile = None + inputProcessor = lineByLine def getOptions(self): + if self.inputFile: + fp = open(self.inputFile) + self.inputs = inputProcessor(fp) + return {'inputs': self.inputs} def __repr__(self):

1 0

[ooni-probe/master] * Added documentation.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit fa731c54c4c28764a345523ca3ab632e81fad884 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Sep 23 12:58:21 2012 +0000 * Added documentation. * Moved general setup, circuit, and bootstrap, to utils/onion.py. --- ooni/plugins/bridget.py | 323 +++++++++++++++++++---------------------------- ooni/utils/circuit.py | 109 ---------------- ooni/utils/onion.py | 260 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 389 insertions(+), 303 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index d82f2e7..c4667e1 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -13,7 +13,6 @@ # :version: 0.1.0-alpha from __future__ import with_statement -from functools import wraps, partial from random import randint from twisted.python import usage from twisted.plugin import IPlugin @@ -25,11 +24,12 @@ from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset import os -import signal import sys def timer(secs, e=None): + import signal + import functools.wraps def decorator(func): def _timer(signum, frame): raise TimeoutError, e @@ -41,7 +41,7 @@ def timer(secs, e=None): finally: signal.alarm(0) return res - return wraps(func)(wrapper) + return functools.wraps(func)(wrapper) return decorator @@ -297,27 +297,60 @@ class BridgetTest(OONITest): def experiment(self, args): """ - XXX fill me in + We cannot use the Asset model, because that model calls + self.experiment() with the current Assets, which would be one relay + and one bridge, then it gives the defer.Deferred returned from + self.experiment() to self.control(), which means that, for each + (bridge, relay) pair, experiment gets called again, which instantiates + an additional Tor process that attempts to bind to the same + ports. Thus, additionally instantiated Tor processes return with + RuntimeErrors, which break the final defer.chainDeferred.callback(), + sending it into the errback chain. + + if bridges: + 1. configure first bridge line + 2a. configure data_dir, if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor } if any of these + 4. remove bridges which are public relays } fail, add current + 5. SIGHUP for each bridge } bridge to unreach- + } able bridges. + if relays: + 1a. configure the data_dir, if it doesn't exist + 1b. write torrc to a tempfile in data_dir + 2. start tor + 3. remove any of our relays which are already part of current + circuits + 4a. attach CustomCircuit() to self.state + 4b. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays + 5. + if bridges and relays: + 1. configure first bridge line + 2a. configure data_dir if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor + 4. remove bridges which are public relays + 5. remove any of our relays which are already part of current + circuits + 6a. attach CustomCircuit() to self.state + 6b. for each bridge, build three circuits, with three + relays each + 6c. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays :param args: The :class:`BridgetAsset` line currently being used. """ try: - from tempfile import mkstemp, mkdtemp - from shutil import rmtree - - from twisted.internet.endpoints import TCP4ClientEndpoint - - from ooni.utils import circuit - from ooni.lib.txtorcon import TorProcessProtocol - from ooni.lib.txtorcon import TorProtocolFactory + from ooni.utils.onion import start_tor, CustomCircuit from ooni.lib.txtorcon import TorConfig, TorState - except ImportError: raise TxtorconImportError - - except TxtorconImportError: - ## XXX is this something we should add to the reactor? + except TxtorconImportError, tie: + log.err(tie) sys.exit() def bootstrap(ctrl): @@ -329,18 +362,6 @@ class BridgetTest(OONITest): conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) log.msg("Tor process connected, bootstrapping ...") - def delete_temp(delete_list): - """ - Given a list of files or directories to delete, delete all and - suppress all errors. - """ - for temp in delete_list: - try: - os.unlink(temp) - except OSError: - rmtree(temp, ignore_errors=True) - - #(a)timer(self.circuit_timeout) @defer.inlineCallbacks def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): """ @@ -352,10 +373,11 @@ class BridgetTest(OONITest): log.msg("Current Bridge: %s" % bridge) try: if use_pt is False: - reset_tor = yield state.protocol.set_conf('Bridge', bridge) - elif use_pt and pt_type is not None: reset_tor = yield state.protocol.set_conf('Bridge', - pt_type +' '+ bridge) + bridge) + elif use_pt and pt_type is not None: + reset_tor = yield state.protocol.set_conf( + 'Bridge', pt_type +' '+ bridge) else: raise PTNotFoundException @@ -368,16 +390,17 @@ class BridgetTest(OONITest): if controller_response == 'OK': defer.returnValue(state.callback) else: - log.msg("TorControlProtocol responded with error:\n%s\n" - % controller_response) + log.msg("TorControlProtocol responded with error:\n%s", + controller_response) defer.returnValue(state.callback) except Exception, e: - log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s\n" - % (bridge, e)) + log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s", + bridge, e) def reconfigure_fail(state, bridge, bad): - log.msg("Reconfiguring TorConfig with parameters %s failed" % state) + log.msg("Reconfiguring TorConfig with parameters %s failed", + state) bad.append(bridge) @defer.inlineCallbacks @@ -392,104 +415,34 @@ class BridgetTest(OONITest): def __remove_line__(node, bridges=bridges): for line in bridges: if line.startswith(node): - log.msg("Removing %s because it is a public relay" % node) - bridges.remove(line) + try: + log.msg("Removing %s because it is a public relay", + node) + bridges.remove(line) + except ValueError, ve: + log.err(ve) if len(both) > 0: try: - updated = yield map(lambda node: __remove_line__(node), both) + updated = yield map(lambda node: + __remove_line__(node), both) if not updated: ## XXX do these need to be state.callback? defer.returnValue(state) else: defer.returnValue(state) except Exception, e: - log.msg("Unable to remove public relays from bridge list:\n%s" - % both) + log.msg("Removing public relays from bridge list failed:\n%s", + both) log.err(e) - - def request_circuit_build(state, deferred, path=None): - if path is None: - if relays_remaining() > 0: - first, middle,last = (state.relays.pop() - for i in range(3)) - else: - first = random.choice(state.entry_guards.values()) - middle, last = (random.choice(state.routers.values()) - for i in range(2)) - path = [first, middle, last] - else: - assert type(path) is list, "Circuit path must be a list of routers!" - assert len(path) >= 3, "Circuits must be at least three hops!" - - log.msg("Requesting a circuit: %s" - % '->'.join(map(lambda node: node, path))) - - class AppendWaiting: - def __init__(self, attacher, deferred): - self.attacher = attacher - self.d = deferred - def __call__(self, circ): - """ - Return from build_circuit is a Circuit, however, - we want to wait until it is built before we can - issue an attach on it and callback to the Deferred - we issue here. - """ - log.msg("Circuit %s is in progress ..." % circ.id) - self.attacher.waiting_circuits.append((circ.id, self.d)) - - return self.state.build_circuit(path).addCallback( - AppendWaiting(self, deferred)).addErrback( - log.err) + except ValueError, ve: + log.err(ve) def attacher_extend_circuit(attacher, deferred, router): - - - def setup_fail(proto, bridge_list, relay_list): - log.err("Setup Failed: %s" % proto) - log.err("We were given bridge list:\n%s\nAnd our relay list:\n%s\n" - % (bridge_list, relay_list)) - report.update({'setup_fail': 'FAILED', - 'proto': proto, - 'bridge_list': bridge_list, - 'relay_list': relay_list}) - reactor.stop() - - def setup_done(proto, bridge_list, relay_list): - log.msg("Setup Complete: %s" % proto) - state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) - if bridge_list is not None: - state.post_bootstrap.addCallback(remove_public_relays, bridge_list) - if relay_list is not None: - raise NotImplemented - #report.update({'success': args}) - - def start_tor(reactor, update, torrc, to_delete, control_port, tor_binary, - data_directory, bridge_list=None, relay_list=None): - """ - Create a TCP4ClientEndpoint at our control_port, and connect - it to our reactor and a spawned Tor process. Compare with - :meth:`txtorcon.launch_tor` for differences. - """ - end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port) - connection_creator = partial(end_point.connect, TorProtocolFactory()) - process_protocol = TorProcessProtocol(connection_creator, updates) - process_protocol.to_delete = to_delete - reactor.addSystemEventTrigger('before', 'shutdown', - partial(delete_temp, to_delete)) - try: - transport = reactor.spawnProcess(process_protocol, - tor_binary, - args=(tor_binary,'-f',torrc), - env={'HOME': data_directory}, - path=data_directory) - transport.closeStdin() - except RuntimeError, e: - process_protocol.connected_cb.errback(e) - finally: - return process_protocol.connected_cb, bridge_list, relay_list + ## XXX todo write me + ## state.attacher.extend_circuit + raise NotImplemented + #attacher.extend_circuit def state_complete(state, bridge_list=None, relay_list=None): """Called when we've got a TorState.""" @@ -508,12 +461,14 @@ class BridgetTest(OONITest): else: return state, None - def state_attach(state, relay_list): + def state_attach(state, path): log.msg("Setting up custom circuit builder...") attacher = CustomCircuit(state) state.set_attacher(attacher, reactor) state.add_circuit_listener(attacher) + return state + ## OLD #for circ in state.circuits.values(): # for relay in circ.path: # try: @@ -526,80 +481,47 @@ class BridgetTest(OONITest): return d def state_attach_fail(state): - log.msg("Attaching custom circuit builder failed.") + log.err("Attaching custom circuit builder failed: %s", state) - def updates(prog, tag, summary): - log.msg("%d%%: %s" % (prog, summary)) - - def write_torrc(conf, data_dir=None): - """ - Create a torrc in our data_directory. If we don't yet have a - data_directory, create a temporary one. Any temporary files or - folders are added to delete_list. - - :return: delete_list, data_dir, torrc - """ - delete_list = [] - - if data_dir is None: - data_dir = mkdtemp(prefix='bridget-tordata') - delete_list.append(data_dir) - conf.DataDirectory = data_dir - - (fd, torrc) = mkstemp(dir=data_dir) - delete_list.append(torrc) - os.write(fd, conf.create_torrc()) - os.close(fd) - return torrc, data_dir, delete_list - log.msg("Bridget: initiating test ... ") - while self.bridges_remaining() > 0: + if self.bridges_remaining() > 0: + for self.current_bridge in self.bridges: + #self.current_bridge = bridge + + if not self.config.config.has_key('Bridge'): - self.current_bridge = self.bridges.pop() - #self.current_bridge = args['bridge'] + self.config.Bridge = self.current_bridge - try: - self.bridges.remove(self.current_bridge) - except ValueError, ve: - log.err(ve) - - if not self.config.config.has_key('Bridge'): - self.config.Bridge = self.current_bridge - (torrc, self.data_directory, to_delete) = write_torrc( - self.config, self.data_directory) - - log.msg("Starting Tor ...") - log.msg("Using the following as our torrc:\n%s" - % self.config.create_torrc()) - report = {'tor_config': self.config.create_torrc()} - - state = start_tor(reactor, updates, torrc, to_delete, - self.control_port, - self.tor_binary, - self.data_directory, - bridge_list=self.bridges) - state.addCallback(setup_done) - state.addErrback(setup_fail) - state.addCallback(remove_public_relays, self.bridges) + state = start_tor(reactor, + self.config, + self.control_port, + self.tor_binary, + self.data_directory) + state.addCallback(remove_public_relays, self.bridges) + + else: + log.msg("We now have %d untested bridges..." + % self.bridges_remaining()) + reconf = defer.Deferred() + reconf.addCallback(reconfigure_bridge, state, + self.current_bridge, self.use_pt, + self.pt_type) + reconf.addErrback(reconfigure_fail, state, + self.current_bridge, self.bridges_down) + state.chainDeferred(reconf) + state.callback() + #all = [] + #reconf = reconfigure_bridge(state, self.current_bridge, + # self.use_pt, self.pt_type) + #reconf.addCallback(reconfigure_done) + #reconf.addErrback(reconfigure_fail) + #state.DeferredList(all) + + if self.relays_remaining() != 0: + state.chainDeferred() - else: - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - reconf = defer.Deferred() - reconf.addCallback(reconfigure_bridge, state, - self.current_bridge, self.use_pt, - self.pt_type) - reconf.addErrback(reconfigure_fail, state, - self.current_bridge, self.bridges_down) - state.chainDeferred(reconf) - #all = [] - #reconf = reconfigure_bridge(state, self.current_bridge, - # self.use_pt, self.pt_type) - #reconf.addCallback(reconfigure_done) - #reconf.addErrback(reconfigure_fail) - #state.DeferredList(all) while self.relays_remaining() >= 3: #path = list(self.relays.pop() for i in range(3)) @@ -609,10 +531,19 @@ class BridgetTest(OONITest): for node in circ.path: if node == self.current_relay: self.relays_up.append(self.current_relay) - try: - ext = attacher_extend_circuit(state.attacher, circ, - self.current_relay) - + + if len(circ.path) < 3: + try: + parameters = (state.attacher, circ, + self.current_relay) + ext = attacher_extend_circuit(parameters) + ext.addCallback(attacher_extend_circuit_done, + parameters) + except Exception, e: + log.msg("Extend circuit failed: %s %s" + % (e, parameters)) + else: + continue return state ## still need to attach attacher to state @@ -647,6 +578,10 @@ bridget = BridgetTest(None, None, None) ## ISIS' NOTES ## ----------- +## nickm points out that the format operator '%', when used in log.LEVEL(), +## forces string generation even when LEVEL is not used, increasing overhead; +## we should do 'log.err("string with stuff %s", stuff)' instead. +## ## TODO: ## o cleanup documentation ## x add DataDirectory option diff --git a/ooni/utils/circuit.py b/ooni/utils/circuit.py deleted file mode 100644 index d725879..0000000 --- a/ooni/utils/circuit.py +++ /dev/null @@ -1,109 +0,0 @@ -# -# circuit.py -# ---------- -# Utilities for working with Tor circuits. -# -# This code is largely taken from attach_streams_by_country.py in the txtorcon -# documentation, and as such any and all credit should go to meejah. Minor -# adjustments have been made to use OONI's logging system, and to build custom -# circuits without actually attaching streams. -# -# :author: Meejah, Isis Lovecruft -# :license: see included LICENSE file -# :version: 0.1.0-alpha -# - -import random - -from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher -from ooni.lib.txtorcon import TorInfo -from ooni.utils import log -from zope.interface import implements - - -class CustomCircuit(CircuitListenerMixin): - implements(IStreamAttacher) - - def __init__(self, state, relays=None): - self.state = state - self.waiting_circuits = [] - self.relays = relays - - def waiting_on(self, circuit): - for (circid, d) in self.waiting_circuits: - if circuit.id == circid: - return True - return False - - def circuit_extend(self, circuit, router): - "ICircuitListener" - if circuit.purpose != 'GENERAL': - return - if self.waiting_on(circuit): - log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) - - def circuit_built(self, circuit): - "ICircuitListener" - if circuit.purpose != 'GENERAL': - return - log.msg("Circuit %s built ..." % circuit.id) - log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) - for (circid, d) in self.waiting_circuits: - if circid == circuit.id: - self.waiting_circuits.remove((circid, d)) - d.callback(circuit) - - def circuit_failed(self, circuit, reason): - if self.waiting_on(circuit): - log.msg("Circuit %s failed for reason %s" - % (circuit.id, reason)) - circid, d = None, None - for c in self.waiting_circuits: - if c[0] == circuit.id: - circid, d = c - if d is None: - raise Exception("Expected to find circuit.") - - self.waiting_circuits.remove((circid, d)) - log.msg("Trying to build a circuit for %s" % circid) - self.request_circuit_build(d) - - def check_circuit_route(self, circuit, router): - if router in circuit.path: - #router.update() ## XXX can i use without args? no. - TorInfo.dump(self) - - def request_circuit_build(self, deferred, path=None): - if path is None: - if self.state.relays_remaining() > 0: - first, middle,last = (self.state.relays.pop() - for i in range(3)) - else: - first = random.choice(self.state.entry_guards.values()) - middle, last = (random.choice(self.state.routers.values()) - for i in range(2)) - path = [first, middle, last] - else: - assert type(path) is list, "Circuit path must be a list of routers!" - assert len(path) >= 3, "Circuits must be at least three hops!" - - log.msg("Requesting a circuit: %s" - % '->'.join(map(lambda node: node, path))) - - class AppendWaiting: - def __init__(self, attacher, deferred): - self.attacher = attacher - self.d = deferred - def __call__(self, circ): - """ - Return from build_circuit is a Circuit, however, - we want to wait until it is built before we can - issue an attach on it and callback to the Deferred - we issue here. - """ - log.msg("Circuit %s is in progress ..." % circ.id) - self.attacher.waiting_circuits.append((circ.id, self.d)) - - return self.state.build_circuit(path).addCallback( - AppendWaiting(self, deferred)).addErrback( - log.err) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py new file mode 100644 index 0000000..76ec909 --- /dev/null +++ b/ooni/utils/onion.py @@ -0,0 +1,260 @@ +# +# onion.py +# ---------- +# Utilities for working with Tor. +# +# This code is largely taken from txtorcon and its documentation, and as such +# any and all credit should go to Meejah. Minor adjustments have been made to +# use OONI's logging system, and to build custom circuits without actually +# attaching streams. +# +# :author: Meejah, Isis Lovecruft +# :license: see included LICENSE file +# :copyright: copyright (c) 2012 The Tor Project, Inc. +# :version: 0.1.0-alpha +# + +import random + +from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +from ooni.utils import log +from zope.interface import implements + + +def __setup_done__(proto): + log.msg("Setup Complete: %s" % proto) + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete).addErrback(__setup_fail__) + +def __setup_fail__(proto): + log.err("Setup Failed: %s" % proto) + report.update({'setup_fail': proto}) + reactor.stop() + +def __updates__(_progress, _tag, _summary): + log.msg("%d%%: %s", _progress, _summary) + +def write_torrc(conf, data_dir=None): + """ + Create a torrc in our data_dir. If we don't yet have a data_dir, create a + temporary one. Any temporary files or folders are added to delete_list. + + :return: torrc, data_dir, delete_list + """ + try: + from os import write, close + from tempfile import mkstemp, mkdtemp + except ImportError, ie: + log.err(ie) + + delete_list = [] + + if data_dir is None: + data_dir = mkdtemp(prefix='bridget-tordata') + delete_list.append(data_dir) + conf.DataDirectory = data_dir + + (fd, torrc) = mkstemp(dir=data_dir) + delete_list.append(torrc) + write(fd, conf.create_torrc()) + close(fd) + return torrc, data_dir, delete_list + +def delete_files_or_dirs(delete_list): + """ + Given a list of files or directories to delete, delete all and suppress + all errors. + + :param delete_list: + A list of files or directories to delete. + """ + try: + from os import unlink + from shutil import rmtree + except ImportError, ie: + log.err(ie) + + for temp in delete_list: + try: + unlink(temp) + except OSError: + rmtree(temp, ignore_errors=True) + +def start_tor(reactor, config, control_port, tor_binary, data_dir, + report=None, progress=__updates__, process_cb=__setup_done__, + process_eb=__setup_fail__): + """ + Use a txtorcon.TorConfig() instance, config, to write a torrc to a + tempfile in our DataDirectory, data_dir. If data_dir is None, a temp + directory will be created. Finally, create a TCP4ClientEndpoint at our + control_port, and connect it to our reactor and a spawned Tor + process. Compare with :meth:`txtorcon.launch_tor` for differences. + + :param reactor: + An instance of class:`twisted.internet.reactor`. + :param config: + An instance of class:`txtorcon.TorConfig` with all torrc options + already configured. ivar:`config.ControlPort`, + ivar:`config.SocksPort`, ivar:`config.CookieAuthentication`, should + already be set, as well as ivar:`config.UseBridges` and + ivar:`config.Bridge` if bridges are to be used. + ivar:`txtorcon.DataDirectory` does not need to be set. + :param control_port: + The port number to use for Tor's ControlPort. + :param tor_binary: + The full path to the Tor binary to use. + :param data_dir: + The directory to use as Tor's DataDirectory. + :param report: + The class:`ooni.plugoo.reports.Report` instance to .update(). + :param progress: + A non-blocking function to handle bootstrapping updates, which takes + three parameters: _progress, _tag, and _summary. + :param process_cb: + The function to callback to after + class:`ooni.lib.txtorcon.TorProcessProtocol` returns with the fully + bootstrapped Tor process. + :param process_eb: + The function to errback to if + class:`ooni.lib.txtorcon.TorProcessProtocol` fails. + :return: + A class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a + class:`txtorcon.TorControlProtocol` as .protocol. + """ + try: + from functools import partial + from twisted.internet.endpoints import TCP4ClientEndpoint + from ooni.lib.txtorcon import TorProtocolFactory + from ooni.lib.txtorcon import TorProcessProtocol + except ImportError, ie: + log.err(ie) + + ## TODO: add option to specify an already existing torrc, which + ## will require prior parsing to enforce necessary lines + (torrc, data_dir, to_delete) = write_torrc(config, data_dir) + + log.msg("Starting Tor ...") + log.msg("Using the following as our torrc:\n%s", config.create_torrc()) + if report is None: + report = {'torrc': config.create_torrc()} + else: + report.update({'torrc': config.create_torrc()}) + + end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port) + connection_creator = partial(end_point.connect, TorProtocolFactory()) + process_protocol = TorProcessProtocol(connection_creator, progress) + process_protocol.to_delete = to_delete + + reactor.addSystemEventTrigger('before', 'shutdown', + partial(delete_files_or_dirs, to_delete)) + try: + transport = reactor.spawnProcess(process_protocol, + tor_binary, + args=(tor_binary,'-f',torrc), + env={'HOME': data_dir}, + path=data_dir) + transport.closeStdin() + except RuntimeError, e: + log.err("Starting Tor failed: %s", e) + process_protocol.connected_cb.errback(e) + except NotImplementedError, e: + url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" + log.err("Running bridget on Windows requires pywin32: %s", url) + process_protocol.connected_cb.errback(e) + + proto = process_protocol.connected_cb ## new defer.Deferred() + proto.addCallback(process_cb) + proto.addErrback(process_eb) + return proto + + +class CustomCircuit(CircuitListenerMixin): + implements(IStreamAttacher) + + def __init__(self, state, relays=None): + self.state = state + self.waiting_circuits = [] + self.relays = relays + + def waiting_on(self, circuit): + for (circid, d) in self.waiting_circuits: + if circuit.id == circid: + return True + return False + + def circuit_extend(self, circuit, router): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + if self.waiting_on(circuit): + log.msg("Circuit %d (%s)", circuit.id, router.id_hex) + + def circuit_built(self, circuit): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + log.msg("Circuit %s built ...", circuit.id) + log.msg("Full path of %s: %s", circuit.id, circuit.path) + for (circid, d) in self.waiting_circuits: + if circid == circuit.id: + self.waiting_circuits.remove((circid, d)) + d.callback(circuit) + + def circuit_failed(self, circuit, reason): + if self.waiting_on(circuit): + log.msg("Circuit %s failed for reason %s", circuit.id, reason) + circid, d = None, None + for c in self.waiting_circuits: + if c[0] == circuit.id: + circid, d = c + if d is None: + raise Exception("Expected to find circuit.") + + self.waiting_circuits.remove((circid, d)) + log.msg("Trying to build a circuit for %s", circid) + self.request_circuit_build(d) + + def check_circuit_route(self, router): + """ + Check if a relay is a hop in one of our already built circuits. + + """ + for circ in self.state.circuits.values(): + if router in circuit.path: + #router.update() ## XXX can i use without args? no. + TorInfo.dump(self) + + def request_circuit_build(self, deferred, path=None): + if path is None: + if self.state.relays_remaining() > 0: + first, middle,last = (self.state.relays.pop() + for i in range(3)) + else: + first = random.choice(self.state.entry_guards.values()) + middle, last = (random.choice(self.state.routers.values()) + for i in range(2)) + path = [first, middle, last] + else: + assert type(path) is list, "Circuit path must be a list of relays!" + assert len(path) >= 3, "Circuits must be at least three hops!" + + log.msg("Requesting a circuit: %s" + % '->'.join(map(lambda node: node, path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + def __call__(self, circ): + """ + Return from build_circuit is a Circuit, however, + we want to wait until it is built before we can + issue an attach on it and callback to the Deferred + we issue here. + """ + log.msg("Circuit %s is in progress ...", circ.id) + self.attacher.waiting_circuits.append((circ.id, self.d)) + + return self.state.build_circuit(path).addCallback( + AppendWaiting(self, deferred)).addErrback( + log.err)

1 0

[ooni-probe/master] * Added singleton_semaphore utility function for safely adding processes to a
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 086c5d340d8abaea4d4e447690e26b0c619fcfea Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 25 13:56:01 2012 +0000 * Added singleton_semaphore utility function for safely adding processes to a reactor. * Fixes #6969 (bridget tries to start multiple tor processes). * Fixes #6970 (bridget skips testing the first bridge). --- ooni/plugins/bridget.py | 100 +++++++++++++++++++++------------------------- ooni/utils/onion.py | 67 ++++++++++++++++++++++++------- 2 files changed, 97 insertions(+), 70 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 83c9ed6..31537ec 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -370,7 +370,9 @@ class BridgetTest(OONITest): The :class:`BridgetAsset` line currently being used. """ try: - from ooni.utils.onion import start_tor, CustomCircuit + from ooni.utils.onion import start_tor, singleton_semaphore + from ooni.utils.onion import setup_done, setup_fail + from ooni.utils.onion import CustomCircuit from ooni.lib.txtorcon import TorConfig, TorState except ImportError: raise TxtorconImportError @@ -507,54 +509,49 @@ class BridgetTest(OONITest): log.msg("Bridget: initiating test ... ") - #defer.setDebugging(True) + d = defer.Deferred - if self.bridges_remaining() > 0: - if not 'Bridge' in self.config.config: - self.config.Bridge = self.bridges.pop() - - ## Necessary for avoiding starting several processes: - self.config.save() - assert self.config.config.has_key('Bridge'), "NO BRIDGE" - - state = start_tor(self.reactor, self.config, self.control_port, - self.tor_binary, self.data_directory) - #state.addCallback(remove_public_relays, self.bridges) - #state.callback - #rm_public_relays = defer.Deferred() - #rm_public_relays.addCallback(remove_public_relays, - # self.bridges) - #state.chainDeferred(rm_public_relays) - #state = defer.DeferredList([state]) - from ooni.utils.onion import __setup_done__, __setup_fail__ - state.addCallback(__setup_done__) - state.addErrback(__setup_fail__) - + if self.bridges_remaining() > 0 and not 'Bridge' in self.config.config: + self.config.Bridge = self.bridges.pop() + + ## Necessary for avoiding starting several processes: + self.config.save() + assert self.config.config.has_key('Bridge'), "NO BRIDGE" + + tor = start_tor(self.reactor, + self.config, + self.control_port, + self.tor_binary, + self.data_directory).addCallback( + setup_done).addErrback( + setup_fail) + self.tor_process_semaphore = True + + run_once = d().addCallback(singleton_semaphore, tor) + run_once.addErrback(setup_fail) - ## XXX Should do something like: if state.iscomplete - #if 'Bridge' in self.config.config: - if state and 'Bridge' in self.config.config: + only_bridges = d().addCallback(remove_public_relays, self.bridges) + + state = defer.gatherResults([run_once, only_bridges], consumeErrors=True) + log.debug("%s" % state.callbacks) + + if self.bridges_remaining() > 0: all = [] for bridge in self.bridges: self.current_bridge = bridge log.msg("We now have %d untested bridges..." % self.bridges_remaining()) - #reconf = defer.Deferred() - #reconf.addCallback(reconfigure_bridge, state, - # self.current_bridge, self.use_pt, - # self.pt_type) - #reconf.addErrback(reconfigure_fail, state, - # self.current_bridge, self.bridges_down) - #state.chainDeferred(reconf) - #state.callback - - reconf = reconfigure_bridge(state, - self.current_bridge, - self.use_pt, - self.pt_type) - all.append(reconf) - state.chainDeferred(defer.DeferredList(all)) - #state.addCallback(defer.DeferredList(all)) + reconf = d().addCallback(reconfigure_bridge, state, + self.current_bridge, + self.use_pt, + self.pt_type) + reconf.addCallback(reconfigure_done, self.current_bridge, + self.bridges_up) + reconf.addErrback(reconfigure_fail, self.current_bridge, + self.bridges_down) + all.append(reconf) + state.chainDeferred(defer.DeferredList(all)) + log.debug("%s" % state.callbacks) if self.relays_remaining() > 0: while self.relays_remaining() >= 3: @@ -568,23 +565,18 @@ class BridgetTest(OONITest): self.relays_up.append(self.current_relay) if len(circ.path) < 3: try: - parameters = (state.attacher, circ, - self.current_relay) - ext = attacher_extend_circuit(parameters) + ext = attacher_extend_circuit(state.attacher, circ, + self.current_relay) ext.addCallback(attacher_extend_circuit_done, - parameters) + state.attacher, circ, + self.current_relay) except Exception, e: - log.msg("Extend circuit failed: %s %s" - % (e, parameters)) + log.msg("Extend circuit failed: %s" % e) else: continue - #return state - ## still need to attach attacher to state - ## then build circuits - - - reactor.run() + #reactor.run() + return state def control(self, experiment_result, args): experiment_result.callback diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 08f27d4..5f15e90 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -23,18 +23,18 @@ from twisted.internet import defer from zope.interface import implements -def __setup_done__(proto): - log.msg("Setup Complete: %s" % proto) +def setup_done(proto): + log.msg("Setup Complete") state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallback(__state_complete__) - state.post_bootstrap.addErrback(__setup_fail__) + state.post_bootstrap.addCallback(state_complete) + state.post_bootstrap.addErrback(setup_fail) -def __setup_fail__(proto): +def setup_fail(proto): log.err("Setup Failed: %s" % proto) report.update({'setup_fail': proto}) reactor.stop() -def __state_complete__(state, bridge_list=None, relay_list=None): +def state_complete(state): """Called when we've got a TorState.""" log.msg("We've completely booted up a Tor version %s at PID %d" % (state.protocol.version, state.tor_pid)) @@ -44,14 +44,10 @@ def __state_complete__(state, bridge_list=None, relay_list=None): for circ in state.circuits.values(): log.msg("%s" % circ) - if bridge_list is not None and relay_list is None: - return state, bridge_list - elif bridge_list is None and relay_list is not None: - raise NotImplemented - else: - return state, None + return state -def __updates__(_progress, _tag, _summary): +def updates(_progress, _tag, _summary): + """Log updates on the Tor bootstrapping process.""" log.msg("%d%%: %s" % (_progress, _summary)) def write_torrc(conf, data_dir=None): @@ -59,6 +55,11 @@ def write_torrc(conf, data_dir=None): Create a torrc in our data_dir. If we don't yet have a data_dir, create a temporary one. Any temporary files or folders are added to delete_list. + :param conf: + A :class:`ooni.lib.txtorcon.TorConfig` object, with all configuration + values saved. + :param data_dir: + The Tor DataDirectory to use. :return: torrc, data_dir, delete_list """ try: @@ -78,6 +79,7 @@ def write_torrc(conf, data_dir=None): delete_list.append(torrc) write(fd, conf.create_torrc()) close(fd) + return torrc, data_dir, delete_list def delete_files_or_dirs(delete_list): @@ -101,9 +103,41 @@ def delete_files_or_dirs(delete_list): rmtree(temp, ignore_errors=True) @defer.inlineCallbacks +def singleton_semaphore(deferred_process_init, callbacks=[], errbacks=[]): + """ + Initialize a process only once, and do not return until + that initialization is complete. + + :param deferred_process_init: + A deferred which returns a connected process via + :meth:`twisted.internet.reactor.spawnProcess`. + :param callbacks: + A list of callback functions to add to the initialized processes' + deferred. + :param errbacks: + A list of errback functions to add to the initialized processes' + deferred. + :return: + The final state of the :param deferred_process_init: after the + callback chain has completed. This should be a fully initialized + process connected to a :class:`twisted.internet.reactor`. + """ + assert type(callbacks) is list + assert type(errbacks) is list + + for cb in callbacks: + deferred_process_init.addCallback(cb) + for eb in errbacks: + deferred_process_init.addErrback(eb) + + only_once = defer.DeferredSemaphore(1) + singleton = yield only_once.run(deferred_process_init) + defer.returnValue(singleton) + +(a)defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, - report=None, progress=__updates__, process_cb=__setup_done__, - process_eb=__setup_fail__): + report=None, progress=updates, process_cb=setup_done, + process_eb=setup_fail): """ Use a txtorcon.TorConfig() instance, config, to write a torrc to a tempfile in our DataDirectory, data_dir. If data_dir is None, a temp @@ -139,7 +173,8 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, The function to errback to if class:`ooni.lib.txtorcon.TorProcessProtocol` fails. :return: - A class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a + The result of the callback of a + class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a class:`txtorcon.TorControlProtocol` as .protocol. """ try:

1 0

[ooni-probe/master] * Still squashing bugs.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit cafc88339b058798005766cf97fabd659f0e20f5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Sep 23 19:00:33 2012 +0000 * Still squashing bugs. * Fixed the log.debug() function. * Put more general things in the onion.py utility (renamed circuit.py to onion.py). * Updated the installation instructions and documentation on dependencies. * Added my name to the license. --- INSTALL | 6 +++- LICENSE | 2 +- ooni/plugins/bridget.py | 85 ++++++++++++++++++++++------------------------ ooni/utils/log.py | 4 ++- ooni/utils/onion.py | 47 +++++++++++++++++-------- 5 files changed, 82 insertions(+), 62 deletions(-) diff --git a/INSTALL b/INSTALL index bac1a39..d972d26 100644 --- a/INSTALL +++ b/INSTALL @@ -1,5 +1,5 @@ # Dependencies -* Twisted: http://twistedmatrix.com/trac/ +* Twisted (>12.0.0): http://twistedmatrix.com/trac/ * PyYAML: http://pyyaml.org/ * Scapy: http://www.secdev.org/projects/scapy/ * pypcap: http://code.google.com/p/pypcap/ @@ -7,6 +7,10 @@ Optional: * BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +* Dnspython[1]: http://www.dnspython.org/ +* Paramiko[2]: http://www.lag.net/paramiko/ + +[1][2] These dependencies will soon be removed completely. # Installation diff --git a/LICENSE b/LICENSE index f1a9745..a0c3c18 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012, Jacob Appelbaum, Arturo Filastò +Copyright (c) 2012, Jacob Appelbaum, Arturo Filastò, Isis Lovecruft All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index c4667e1..cd7ba4e 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -189,7 +189,7 @@ class BridgetTest(OONITest): implements(IPlugin, ITest) shortName = "bridget" - description = "Use a Tor process to test connecting to bridges and relays" + description = "Use a Tor process to test connecting to bridges or relays" requirements = None options = BridgetArgs blocking = False @@ -218,7 +218,7 @@ class BridgetTest(OONITest): self.relays_down_count = lambda: len(self.relays_down) self.current_relay = None - def make_asset_list(opt, lst): + def __make_asset_list__(opt, lst): log.msg("Loading information from %s ..." % opt) with open(opt) as opt_file: for line in opt_file.readlines(): @@ -238,12 +238,12 @@ class BridgetTest(OONITest): if options['bridges']: self.config.UseBridges = 1 - make_asset_list(options['bridges'], self.bridges) + __make_asset_list__(options['bridges'], self.bridges) if options['relays']: - ## first hop must be in TorState().entry_guards to build circuits + ## first hop must be in TorState().guards to build circuits self.config.EntryNodes = ','.join(relay_list) - make_asset_list(options['relays'], self.relays) + __make_asset_list__(options['relays'], self.relays) if options['socks']: self.socks_port = options['socks'] @@ -279,6 +279,7 @@ class BridgetTest(OONITest): self.config.ControlPort = self.control_port self.config.CookieAuthentication = 1 + ''' def load_assets(self): """ Load bridges and/or relays from files given in user options. Bridges @@ -294,6 +295,7 @@ class BridgetTest(OONITest): assets.update({'relay': BridgetAsset(self.local_options['relays'])}) return assets + ''' def experiment(self, args): """ @@ -353,6 +355,7 @@ class BridgetTest(OONITest): log.err(tie) sys.exit() + ## XXX qu'est-que fuck? ou est utiliser ce fonction? def bootstrap(ctrl): """ Launch a Tor process with the TorConfig instance returned from @@ -388,19 +391,19 @@ class BridgetTest(OONITest): #else: # defer.returnValue((state.callback, controller_response)) if controller_response == 'OK': - defer.returnValue(state.callback) + defer.returnValue((state.callback, controller_response)) else: - log.msg("TorControlProtocol responded with error:\n%s", - controller_response) - defer.returnValue(state.callback) + log.msg("TorControlProtocol responded with error:\n%s" + % controller_response) + defer.returnValue((state.callback, None)) except Exception, e: - log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s", - bridge, e) + log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s" + % (bridge, e)) def reconfigure_fail(state, bridge, bad): - log.msg("Reconfiguring TorConfig with parameters %s failed", - state) + log.msg("Reconfiguring TorConfig with parameters %s failed" + % state) bad.append(bridge) @defer.inlineCallbacks @@ -416,8 +419,8 @@ class BridgetTest(OONITest): for line in bridges: if line.startswith(node): try: - log.msg("Removing %s because it is a public relay", - node) + log.msg("Removing %s because it is a public relay" + % node) bridges.remove(line) except ValueError, ve: log.err(ve) @@ -432,8 +435,8 @@ class BridgetTest(OONITest): else: defer.returnValue(state) except Exception, e: - log.msg("Removing public relays from bridge list failed:\n%s", - both) + log.msg("Removing public relays from bridge list failed:\n%s" + % both) log.err(e) except ValueError, ve: log.err(ve) @@ -444,22 +447,22 @@ class BridgetTest(OONITest): raise NotImplemented #attacher.extend_circuit - def state_complete(state, bridge_list=None, relay_list=None): - """Called when we've got a TorState.""" - log.msg("We've completely booted up a Tor version %s at PID %d" - % (state.protocol.version, state.tor_pid)) - - log.msg("This Tor has the following %d Circuits:" - % len(state.circuits)) - for circ in state.circuits.values(): - log.msg("%s" % circ) - - if bridge_list is not None and relay_list is None: - return state, bridge_list - elif bridge_list is None and relay_list is not None: - raise NotImplemented - else: - return state, None + #def state_complete(state, bridge_list=None, relay_list=None): + # """Called when we've got a TorState.""" + # log.msg("We've completely booted up a Tor version %s at PID %d" + # % (state.protocol.version, state.tor_pid)) + # + # log.msg("This Tor has the following %d Circuits:" + # % len(state.circuits)) + # for circ in state.circuits.values(): + # log.msg("%s" % circ) + # + # if bridge_list is not None and relay_list is None: + # return state, bridge_list + # elif bridge_list is None and relay_list is not None: + # raise NotImplemented + # else: + # return state, None def state_attach(state, path): log.msg("Setting up custom circuit builder...") @@ -481,7 +484,7 @@ class BridgetTest(OONITest): return d def state_attach_fail(state): - log.err("Attaching custom circuit builder failed: %s", state) + log.err("Attaching custom circuit builder failed: %s" % state) log.msg("Bridget: initiating test ... ") @@ -491,16 +494,13 @@ class BridgetTest(OONITest): #self.current_bridge = bridge if not self.config.config.has_key('Bridge'): - self.config.Bridge = self.current_bridge - - state = start_tor(reactor, + state = start_tor(self.reactor, self.config, self.control_port, self.tor_binary, self.data_directory) state.addCallback(remove_public_relays, self.bridges) - else: log.msg("We now have %d untested bridges..." % self.bridges_remaining()) @@ -511,7 +511,7 @@ class BridgetTest(OONITest): reconf.addErrback(reconfigure_fail, state, self.current_bridge, self.bridges_down) state.chainDeferred(reconf) - state.callback() + state.callback(controller_response) #all = [] #reconf = reconfigure_bridge(state, self.current_bridge, # self.use_pt, self.pt_type) @@ -570,17 +570,14 @@ class BridgetTest(OONITest): # % self.current_bridge) # self.bridges_up.append(self.current_bridge) - reactor.run() - + #reactor.run() ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) + ## ISIS' NOTES ## ----------- -## nickm points out that the format operator '%', when used in log.LEVEL(), -## forces string generation even when LEVEL is not used, increasing overhead; -## we should do 'log.err("string with stuff %s", stuff)' instead. ## ## TODO: ## o cleanup documentation diff --git a/ooni/utils/log.py b/ooni/utils/log.py index dd5cf13..54c59ea 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -88,7 +88,9 @@ def start(logstdout, logfile=None, verbosity=None): log.msg("Starting OONI...") def debug(message, level="debug", **kw): - log.msg(message, logLevel=level, **kw) + print "[%s] %s" % (level, message) + ## If we want debug messages in the logfile: + #log.msg(message, logLevel=level, **kw) def msg(message, level="info", **kw): log.msg(message, logLevel=level, **kw) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 76ec909..46373b4 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -24,13 +24,31 @@ from zope.interface import implements def __setup_done__(proto): log.msg("Setup Complete: %s" % proto) state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallback(state_complete).addErrback(__setup_fail__) + state.post_bootstrap.addCallback(__state_complete__) + state.post_bootstrap.addErrback(__setup_fail__) def __setup_fail__(proto): log.err("Setup Failed: %s" % proto) report.update({'setup_fail': proto}) reactor.stop() +def __state_complete__(state, bridge_list=None, relay_list=None): + """Called when we've got a TorState.""" + log.msg("We've completely booted up a Tor version %s at PID %d" + % (state.protocol.version, state.tor_pid)) + + log.msg("This Tor has the following %d Circuits:" + % len(state.circuits)) + for circ in state.circuits.values(): + log.msg("%s" % circ) + + if bridge_list is not None and relay_list is None: + return state, bridge_list + elif bridge_list is None and relay_list is not None: + raise NotImplemented + else: + return state, None + def __updates__(_progress, _tag, _summary): log.msg("%d%%: %s", _progress, _summary) @@ -134,7 +152,7 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, (torrc, data_dir, to_delete) = write_torrc(config, data_dir) log.msg("Starting Tor ...") - log.msg("Using the following as our torrc:\n%s", config.create_torrc()) + log.msg("Using the following as our torrc:\n%s" % config.create_torrc()) if report is None: report = {'torrc': config.create_torrc()} else: @@ -144,6 +162,8 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, connection_creator = partial(end_point.connect, TorProtocolFactory()) process_protocol = TorProcessProtocol(connection_creator, progress) process_protocol.to_delete = to_delete + process_protocol.addCallback(process_cb) + process_protocol.addErrback(process_eb) reactor.addSystemEventTrigger('before', 'shutdown', partial(delete_files_or_dirs, to_delete)) @@ -154,18 +174,15 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, env={'HOME': data_dir}, path=data_dir) transport.closeStdin() - except RuntimeError, e: - log.err("Starting Tor failed: %s", e) + except RuntimeError as e: + log.err("Starting Tor failed: %s" % e) process_protocol.connected_cb.errback(e) except NotImplementedError, e: url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" - log.err("Running bridget on Windows requires pywin32: %s", url) + log.err("Running bridget on Windows requires pywin32: %s" % url) process_protocol.connected_cb.errback(e) - proto = process_protocol.connected_cb ## new defer.Deferred() - proto.addCallback(process_cb) - proto.addErrback(process_eb) - return proto + return process_protocol.connected_cb ## new defer.Deferred() class CustomCircuit(CircuitListenerMixin): @@ -187,14 +204,14 @@ class CustomCircuit(CircuitListenerMixin): if circuit.purpose != 'GENERAL': return if self.waiting_on(circuit): - log.msg("Circuit %d (%s)", circuit.id, router.id_hex) + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) def circuit_built(self, circuit): "ICircuitListener" if circuit.purpose != 'GENERAL': return - log.msg("Circuit %s built ...", circuit.id) - log.msg("Full path of %s: %s", circuit.id, circuit.path) + log.msg("Circuit %s built ..." % circuit.id) + log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) for (circid, d) in self.waiting_circuits: if circid == circuit.id: self.waiting_circuits.remove((circid, d)) @@ -202,7 +219,7 @@ class CustomCircuit(CircuitListenerMixin): def circuit_failed(self, circuit, reason): if self.waiting_on(circuit): - log.msg("Circuit %s failed for reason %s", circuit.id, reason) + log.msg("Circuit %s failed for reason %s" % (circuit.id, reason)) circid, d = None, None for c in self.waiting_circuits: if c[0] == circuit.id: @@ -211,7 +228,7 @@ class CustomCircuit(CircuitListenerMixin): raise Exception("Expected to find circuit.") self.waiting_circuits.remove((circid, d)) - log.msg("Trying to build a circuit for %s", circid) + log.msg("Trying to build a circuit for %s" % circid) self.request_circuit_build(d) def check_circuit_route(self, router): @@ -252,7 +269,7 @@ class CustomCircuit(CircuitListenerMixin): issue an attach on it and callback to the Deferred we issue here. """ - log.msg("Circuit %s is in progress ...", circ.id) + log.msg("Circuit %s is in progress ..." % circ.id) self.attacher.waiting_circuits.append((circ.id, self.d)) return self.state.build_circuit(path).addCallback(

1 0

[ooni-probe/master] * Fixes #6968, #6971
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit fef162fe803fad2ff7eab45d321baf1086f6c980 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 25 00:07:54 2012 +0000 * Fixes #6968, #6971 * Found bug: first bridge doesn't get tested #6970 * I'm trying to use trac more to keep other people up-to-date, in case that wasn't obviously. I hate Trac like nobody's business. --- ooni/plugins/bridget.py | 201 +++++++++++++++++++++++++--------------------- ooni/utils/onion.py | 38 +++++++--- 2 files changed, 137 insertions(+), 102 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index cd7ba4e..83c9ed6 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -66,33 +66,43 @@ class PTNotFoundException(Exception): return sys.exit() class ValueChecker(object): - def port_check(self, number): + def port_check(self, port): """Check that given ports are in the allowed range.""" - number = int(number) - if number not in range(1024, 65535): + port = int(port) + if port not in range(1024, 65535): raise ValueError("Port out of range") + log.err() sock_check, ctrl_check = port_check, port_check allowed = "must be between 1024 and 65535." sock_check.coerceDoc = "Port to use for Tor's SocksPort, " +allowed ctrl_check.coerceDoc = "Port to use for Tor's ControlPort, " +allowed - def uid_check(pluggable_transport): - """Check that we're not root when trying to use pluggable transports.""" + def uid_check(self, pluggable_transport): + """ + Check that we're not root when trying to use pluggable transports. If + we are, setuid to normal user (1000) if we're running on a posix-based + system, and if we're Windows just tell the user that we can't be run + as root with the specified options and then exit. + """ uid, gid = os.getuid(), os.getgid() if uid == 0 and gid == 0: log.msg("Error: Running bridget as root with transports not allowed.") + if os.name == 'posix': log.msg("Dropping privileges to normal user...") os.setgid(1000) os.setuid(1000) + else: + sys.exit(0) - def dir_check(d): + def dir_check(self, d): """Check that the given directory exists.""" - if not os.isdir(d): + if not os.path.isdir(d): raise ValueError("%s doesn't exist, or has wrong permissions" % d) - def file_check(f): - if not os.isfile(f): + def file_check(self, f): + """Check that the given file exists.""" + if not os.path.isfile(f): raise ValueError("%s does not exist, or has wrong permissions" % f) class RandomPortException(Exception): @@ -121,8 +131,9 @@ class TxtorconImportError(ImportError): class BridgetArgs(usage.Options): """Commandline options.""" + global vc vc = ValueChecker() - + optParameters = [ ['bridges', 'b', None, 'File listing bridge IP:ORPorts to test'], @@ -227,6 +238,17 @@ class BridgetTest(OONITest): else: lst.append(line.replace('\n','')) + def __parse_data_dir__(data_dir): + if data_dir.startswith('~'): + data_dir = os.path.expanduser(data_dir) + elif data_dir.startswith('/'): + data_dir = os.path.join(os.getcwd(), data_dir) + elif data_dir.startswith('./'): + data_dir = os.path.abspath(data_dir) + else: + data_dir = os.path.join(os.getcwd(), data_dir) + return data_dir + if self.local_options: try: from ooni.lib.txtorcon import TorConfig @@ -258,17 +280,18 @@ class BridgetTest(OONITest): if options['torpath']: self.tor_binary = options['torpath'] - if self.local_options['datadir']: - self.data_directory = local_options['datadir'] + if options['datadir']: + self.data_directory = __parse_data_dir__(options['datadir']) else: self.data_directory = None if options['transport']: self.use_pt = True - log.msg("Using ClientTransportPlugin %s" % options['transport']) + log.msg("Using ClientTransportPlugin %s" + % options['transport']) [self.pt_type, pt_exec] = options['transport'].split(' ', 1) - ## ClientTransportPlugin transport exec path-to-binary [options] + ## ClientTransportPlugin transport exec pathtobinary [options] ## XXX we need a better way to deal with all PTs if self.pt_type == "obfs2": config.ClientTransportPlugin = self.pt_type + " " + pt_exec @@ -355,6 +378,7 @@ class BridgetTest(OONITest): log.err(tie) sys.exit() + ''' ## XXX qu'est-que fuck? ou est utiliser ce fonction? def bootstrap(ctrl): """ @@ -364,6 +388,7 @@ class BridgetTest(OONITest): conf = TorConfig(ctrl) conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) log.msg("Tor process connected, bootstrapping ...") + ''' @defer.inlineCallbacks def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): @@ -385,13 +410,19 @@ class BridgetTest(OONITest): raise PTNotFoundException controller_response = reset_tor.callback + controller_response.addCallback(reconfigure_done, + bridge, + reachable) + controller_response.addErrback(reconfigure_fail, + bridge, + reachable) #if not controller_response: # defer.returnValue((state.callback, None)) #else: # defer.returnValue((state.callback, controller_response)) if controller_response == 'OK': - defer.returnValue((state.callback, controller_response)) + defer.returnValue((state, controller_response)) else: log.msg("TorControlProtocol responded with error:\n%s" % controller_response) @@ -400,11 +431,16 @@ class BridgetTest(OONITest): except Exception, e: log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s" % (bridge, e)) + defer.returnValue((state.callback, e)) - def reconfigure_fail(state, bridge, bad): + def reconfigure_done(response, bridge, reachable): + log.msg("Reconfiguring with 'Bridge %s' successful" % bridge) + reachable.append(bridge) + + def reconfigure_fail(response, bridge, unreachable): log.msg("Reconfiguring TorConfig with parameters %s failed" % state) - bad.append(bridge) + unreachable.append(bridge) @defer.inlineCallbacks def remove_public_relays(state, bridges): @@ -423,7 +459,7 @@ class BridgetTest(OONITest): % node) bridges.remove(line) except ValueError, ve: - log.err(ve) + log.debug(ve) if len(both) > 0: try: @@ -447,23 +483,6 @@ class BridgetTest(OONITest): raise NotImplemented #attacher.extend_circuit - #def state_complete(state, bridge_list=None, relay_list=None): - # """Called when we've got a TorState.""" - # log.msg("We've completely booted up a Tor version %s at PID %d" - # % (state.protocol.version, state.tor_pid)) - # - # log.msg("This Tor has the following %d Circuits:" - # % len(state.circuits)) - # for circ in state.circuits.values(): - # log.msg("%s" % circ) - # - # if bridge_list is not None and relay_list is None: - # return state, bridge_list - # elif bridge_list is None and relay_list is not None: - # raise NotImplemented - # else: - # return state, None - def state_attach(state, path): log.msg("Setting up custom circuit builder...") attacher = CustomCircuit(state) @@ -488,50 +507,65 @@ class BridgetTest(OONITest): log.msg("Bridget: initiating test ... ") + #defer.setDebugging(True) if self.bridges_remaining() > 0: - for self.current_bridge in self.bridges: - #self.current_bridge = bridge + if not 'Bridge' in self.config.config: + self.config.Bridge = self.bridges.pop() + + ## Necessary for avoiding starting several processes: + self.config.save() + assert self.config.config.has_key('Bridge'), "NO BRIDGE" + + state = start_tor(self.reactor, self.config, self.control_port, + self.tor_binary, self.data_directory) + #state.addCallback(remove_public_relays, self.bridges) + #state.callback + #rm_public_relays = defer.Deferred() + #rm_public_relays.addCallback(remove_public_relays, + # self.bridges) + #state.chainDeferred(rm_public_relays) + #state = defer.DeferredList([state]) + from ooni.utils.onion import __setup_done__, __setup_fail__ + state.addCallback(__setup_done__) + state.addErrback(__setup_fail__) - if not self.config.config.has_key('Bridge'): - self.config.Bridge = self.current_bridge - state = start_tor(self.reactor, - self.config, - self.control_port, - self.tor_binary, - self.data_directory) - state.addCallback(remove_public_relays, self.bridges) - else: - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - reconf = defer.Deferred() - reconf.addCallback(reconfigure_bridge, state, - self.current_bridge, self.use_pt, - self.pt_type) - reconf.addErrback(reconfigure_fail, state, - self.current_bridge, self.bridges_down) - state.chainDeferred(reconf) - state.callback(controller_response) - #all = [] - #reconf = reconfigure_bridge(state, self.current_bridge, - # self.use_pt, self.pt_type) - #reconf.addCallback(reconfigure_done) - #reconf.addErrback(reconfigure_fail) - #state.DeferredList(all) - - if self.relays_remaining() != 0: - state.chainDeferred() - + ## XXX Should do something like: if state.iscomplete + #if 'Bridge' in self.config.config: + if state and 'Bridge' in self.config.config: + all = [] + for bridge in self.bridges: + self.current_bridge = bridge + log.msg("We now have %d untested bridges..." + % self.bridges_remaining()) + #reconf = defer.Deferred() + #reconf.addCallback(reconfigure_bridge, state, + # self.current_bridge, self.use_pt, + # self.pt_type) + #reconf.addErrback(reconfigure_fail, state, + # self.current_bridge, self.bridges_down) + #state.chainDeferred(reconf) + #state.callback + + reconf = reconfigure_bridge(state, + self.current_bridge, + self.use_pt, + self.pt_type) + all.append(reconf) + state.chainDeferred(defer.DeferredList(all)) + #state.addCallback(defer.DeferredList(all)) + + if self.relays_remaining() > 0: while self.relays_remaining() >= 3: #path = list(self.relays.pop() for i in range(3)) - #log.msg("Trying path %s" % '->'.join(map(lambda node: node, path))) + #log.msg("Trying path %s" % '->'.join(map(lambda node: + # node, path))) self.current_relay = self.relays.pop() for circ in state.circuits.values(): for node in circ.path: if node == self.current_relay: self.relays_up.append(self.current_relay) - if len(circ.path) < 3: try: parameters = (state.attacher, circ, @@ -545,32 +579,15 @@ class BridgetTest(OONITest): else: continue - return state + #return state ## still need to attach attacher to state ## then build circuits - ## XXX see txtorcon.TorControlProtocol.add_event_listener we - ## may not need full CustomCircuit class - ## o if bridges and relays, use one bridge then build a circuit - ## from three relays - ## o if bridges only, try one bridge at a time, but don't build - ## circuits, just return - ## o if relays only, build circuits from relays - #else: - # try: - # state.addCallback(reconfigure_bridge, self.current_bridge, - # self.use_pt, self.pt_type) - # state.addErrback(reconfigure_fail) - # except TimeoutError: - # log.msg("Adding %s to unreachable bridges..." - # % self.current_bridge) - # self.bridges_down.append(self.current_bridge) - # else: - # log.msg("Adding %s to reachable bridges..." - # % self.current_bridge) - # self.bridges_up.append(self.current_bridge) - - #reactor.run() + + reactor.run() + + def control(self, experiment_result, args): + experiment_result.callback ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) @@ -585,6 +602,6 @@ bridget = BridgetTest(None, None, None) ## x check if bridges are public relays ## o take bridge_desc file as input, also be able to give same ## format as output -## x Add assychronous timout for deferred, so that we don't wait +## x Add asynchronous timeout for deferred, so that we don't wait ## forever for bridges that don't work. ## o Add mechanism for testing through another host diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 46373b4..08f27d4 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -17,7 +17,9 @@ import random from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +from ooni.lib.txtorcon import TorState from ooni.utils import log +from twisted.internet import defer from zope.interface import implements @@ -50,7 +52,7 @@ def __state_complete__(state, bridge_list=None, relay_list=None): return state, None def __updates__(_progress, _tag, _summary): - log.msg("%d%%: %s", _progress, _summary) + log.msg("%d%%: %s" % (_progress, _summary)) def write_torrc(conf, data_dir=None): """ @@ -98,6 +100,7 @@ def delete_files_or_dirs(delete_list): except OSError: rmtree(temp, ignore_errors=True) +(a)defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, report=None, progress=__updates__, process_cb=__setup_done__, process_eb=__setup_fail__): @@ -162,18 +165,17 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, connection_creator = partial(end_point.connect, TorProtocolFactory()) process_protocol = TorProcessProtocol(connection_creator, progress) process_protocol.to_delete = to_delete - process_protocol.addCallback(process_cb) - process_protocol.addErrback(process_eb) reactor.addSystemEventTrigger('before', 'shutdown', partial(delete_files_or_dirs, to_delete)) try: - transport = reactor.spawnProcess(process_protocol, - tor_binary, - args=(tor_binary,'-f',torrc), - env={'HOME': data_dir}, - path=data_dir) - transport.closeStdin() + transport = yield reactor.spawnProcess(process_protocol, + tor_binary, + args=(tor_binary,'-f',torrc), + env={'HOME': data_dir}, + path=data_dir) + if transport: + transport.closeStdin() except RuntimeError as e: log.err("Starting Tor failed: %s" % e) process_protocol.connected_cb.errback(e) @@ -182,7 +184,23 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, log.err("Running bridget on Windows requires pywin32: %s" % url) process_protocol.connected_cb.errback(e) - return process_protocol.connected_cb ## new defer.Deferred() + #proc_proto = process_protocol.connected_cb + #proc_proto.addCallback(process_cb) + #proc_proto.addErrback(process_eb) + # + #d = yield process_protocol.connected_cb.addCallback( + # process_cb).addErrback( + # process_eb) + #d = yield process_protocol.connected_cb + #d.addCallback(process_cb) + #d.addErrback(process_eb) + # + #defer.returnValue(d) + + d = yield process_protocol.connected_cb + defer.returnValue(d) + + #return process_protocol.connected_cb.addCallback(process_cb).addErrback(process_eb) class CustomCircuit(CircuitListenerMixin):

1 0