- tor-commits - lists.torproject.org

[ooni-probe/master] Get all the pieces together and have a working trial based oonicli
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 2d31d5f14a0f3a7518d47fe748c1677cbd885828 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 18 11:52:41 2012 +0000 Get all the pieces together and have a working trial based oonicli --- ooni/nettest.py | 48 +--------------------------------- ooni/oonicli.py | 38 ++++++---------------------- ooni/runner.py | 75 ++++++++++++++++++++++++++++++++++-------------------- 3 files changed, 57 insertions(+), 104 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 097e947..0c8858b 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -1,3 +1,4 @@ +import itertools from twisted.python import log from twisted.trial import unittest, itrial @@ -42,49 +43,4 @@ class TestSuite(pyunit.TestSuite): return result class TestCase(unittest.TestCase): - """ - A test case represents the minimum - """ - def run(self, result): - """ - Run the test case, storing the results in C{result}. - - First runs C{setUp} on self, then runs the test method (defined in the - constructor), then runs C{tearDown}. As with the standard library - L{unittest.TestCase}, the return value of these methods is disregarded. - In particular, returning a L{Deferred} has no special additional - consequences. - - @param result: A L{TestResult} object. - """ - log.msg("--> %s <--" % (self.id())) - new_result = itrial.IReporter(result, None) - if new_result is None: - result = PyUnitResultAdapter(result) - else: - result = new_result - result.startTest(self) - if self.getSkip(): # don't run test methods that are marked as .skip - result.addSkip(self, self.getSkip()) - result.stopTest(self) - return - - self._passed = False - self._warnings = [] - - self._installObserver() - # All the code inside _runFixturesAndTest will be run such that warnings - # emitted by it will be collected and retrievable by flushWarnings. - unittest._collectWarnings(self._warnings.append, self._runFixturesAndTest, result) - - # Any collected warnings which the test method didn't flush get - # re-emitted so they'll be logged or show up on stdout or whatever. - for w in self.flushWarnings(): - try: - warnings.warn_explicit(**w) - except: - result.addError(self, failure.Failure()) - - result.stopTest(self) - - + name = "DefaultTestName" diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 8eabfb1..fa7742f 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -116,6 +116,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): ] optParameters = [ + ["reportfile", "o", "report.yaml", "report file name"], ["logfile", "l", "test.log", "log file name"], ["random", "z", None, "Run tests in random order using the specified seed"], @@ -292,7 +293,7 @@ def _initialDebugSetup(config): -def _getSuite(config): +def _getSuites(config): loader = _getLoader(config) recurse = not config['no-recurse'] print "loadByNames %s" % config['tests'] @@ -300,7 +301,7 @@ def _getSuite(config): def _getLoader(config): - loader = runner.TestLoader() + loader = runner.NetTestLoader() if config['random']: randomer = random.Random() randomer.seed(config['random']) @@ -309,7 +310,6 @@ def _getLoader(config): return loader - def _makeRunner(config): mode = None if config['debug']: @@ -318,6 +318,7 @@ def _makeRunner(config): mode = runner.OONIRunner.DRY_RUN print "using %s" % config['reporter'] return runner.OONIRunner(config['reporter'], + reportfile=config["reportfile"], mode=mode, profile=config['profile'], logfile=config['logfile'], @@ -328,24 +329,6 @@ def _makeRunner(config): forceGarbageCollection=config['force-gc']) -if 0: - loader = runner.TestLoader() - loader.suiteFactory = TestSuite - - for inputUnit in InputUnitFactory(FooTest.inputs): - print inputUnit - - suite = loader.loadClass(FooTest) - - reporterFactory = ReporterFactory(open('reporting.log', 'a+'), testSuite=suite) - reporterFactory.writeHeader() - #testUnitReport = OONIReporter(open('reporting.log', 'a+')) - #testUnitReport.writeHeader(FooTest) - for inputUnit in InputUnitFactory(FooTest.inputs): - testUnitReport = reporterFactory.create() - suite(testUnitReport, inputUnit) - testUnitReport.done() - def run(): if len(sys.argv) == 1: sys.argv.append("--help") @@ -354,15 +337,10 @@ def run(): config.parseOptions() except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) + _initialDebugSetup(config) trialRunner = _makeRunner(config) - suite = _getSuite(config) - print suite - test_result = trialRunner.run(suite) - if config.tracer: - sys.settrace(None) - results = config.tracer.results() - results.write_results(show_missing=1, summary=False, - coverdir=config.coverdir().path) - sys.exit(not test_result.wasSuccessful()) + suites = _getSuites(config) + for suite in suites: + test_result = trialRunner.run(suite) diff --git a/ooni/runner.py b/ooni/runner.py index a8485af..12ab9ad 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -1,13 +1,18 @@ +import os +import sys import types import time import inspect from twisted.internet import defer -from twisted.python import reflect +from twisted.python import reflect, log, failure from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader -from twisted.trial.runner import isPackage, isTestCase +from twisted.trial.runner import isPackage, isTestCase, ErrorHolder +from twisted.trial.runner import filenameToModule, _importFromFile +from ooni.reporter import ReporterFactory +from ooni.input import InputUnitFactory from ooni import nettest from ooni.plugoo import tests as oonitests @@ -34,6 +39,7 @@ def adaptLegacyTest(obj): pass + class LoggedSuite(nettest.TestSuite): """ Any errors logged in this suite will be reported to the L{TestResult} @@ -98,17 +104,8 @@ class NetTestLoader(object): def __init__(self): self.suiteFactory = nettest.TestSuite - self.sorter = name self._importErrors = [] - def sort(self, xs): - """ - Sort the given things using L{sorter}. - - @param xs: A list of test cases, class or modules. - """ - return sorted(xs, key=self.sorter) - def findTestClasses(self, module): classes = [] @@ -180,12 +177,12 @@ class NetTestLoader(object): if not isTestCase(klass): raise ValueError("%r is not a test case" % (klass,)) names = self.getTestCaseNames(klass) - print "Names %s" % names - tests = self.sort([self._makeCase(klass, self.methodPrefix+name) - for name in names]) - print "Tests %s" % tests + tests = [] + for name in names: + tests.append(self._makeCase(klass, self.methodPrefix+name)) + suite = self.suiteFactory(tests) - print "Suite: %s" % suite + suite.inputs = klass.inputs return suite loadTestsFromTestCase = loadClass @@ -289,17 +286,25 @@ class NetTestLoader(object): @return: A C{TestCase} or C{TestSuite}. """ + print "Loading anything! %s" % thing + ret = None if isinstance(thing, types.ModuleType): if isPackage(thing): - return self.loadPackage(thing, recurse) - return self.loadModule(thing) + ret = self.loadPackage(thing, recurse) + ret = self.loadModule(thing) elif isinstance(thing, types.ClassType): - return self.loadClass(thing) + ret = self.loadClass(thing) elif isinstance(thing, type): - return self.loadClass(thing) + ret = self.loadClass(thing) elif isinstance(thing, types.MethodType): - return self.loadMethod(thing) - raise TypeError("No loader for %r. Unrecognized type" % (thing,)) + ret = self.loadMethod(thing) + if not ret: + raise TypeError("No loader for %r. Unrecognized type" % (thing,)) + try: + ret.inputs = ret.inputs + except: + ret.inputs = [None] + return ret def loadByName(self, name, recurse=False): """ @@ -312,6 +317,7 @@ class NetTestLoader(object): @param name: The fully-qualified name of a Python object. """ + print "Load by Name!" try: thing = self.findByName(name) except: @@ -326,6 +332,7 @@ class NetTestLoader(object): suite returned will have no duplicate tests, even if the same object is named twice. """ + print "Load by Names!" things = [] errors = [] for name in names: @@ -336,7 +343,8 @@ class NetTestLoader(object): suites = [self.loadAnything(thing, recurse) for thing in self._uniqueTests(things)] suites.extend(errors) - return self.suiteFactory(suites) + return suites + #return self.suiteFactory(suites) def _uniqueTests(self, things): @@ -355,8 +363,6 @@ class NetTestLoader(object): return [entry[0] for entry in set(entries)] - - class OONIRunner(object): """ A specialised runner that is used by the ooniprobe frontend to run tests. @@ -407,6 +413,7 @@ class OONIRunner(object): return reporter def __init__(self, reporterFactory, + reportfile="report.yaml", mode=None, logfile='test.log', stream=sys.stdout, @@ -417,6 +424,7 @@ class OONIRunner(object): workingDirectory=None, forceGarbageCollection=False): self.reporterFactory = reporterFactory + self._reportfile = reportfile self.logfile = logfile self.mode = mode self.stream = stream @@ -449,12 +457,21 @@ class OONIRunner(object): self._logFileObserver = log.FileLogObserver(logFile) log.startLoggingWithObserver(self._logFileObserver.emit, 0) - def run(self, test, inputs): + def run(self, test): """ Run the test or suite and return a result object. """ - for input in inputs: - self._runWithInput(test, input) + print test + inputs = test.inputs + reporterFactory = ReporterFactory(open(self._reportfile, 'a+'), + testSuite=test) + reporterFactory.writeHeader() + #testUnitReport = OONIReporter(open('reporting.log', 'a+')) + #testUnitReport.writeHeader(FooTest) + for inputUnit in InputUnitFactory(inputs): + testUnitReport = reporterFactory.create() + test(testUnitReport, inputUnit) + testUnitReport.done() def _runWithInput(self, test, input): """ @@ -482,3 +499,5 @@ class OONIRunner(object): done = getattr(result, 'done', None) result.done() return result + +

1 0

[ooni-probe/master] * Switched to chainDeferreds for testing bridge by bridge, after which
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 92ff36a1bdcfa34264ca3cec11d9a53f1bdac323 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Sep 19 05:45:54 2012 +0000 * Switched to chainDeferreds for testing bridge by bridge, after which EXTEND cells are issued per relay per bridge. And it actually works. --- ooni/plugins/bridget.py | 121 +++++++++++++++++++++++++++-------------------- 1 files changed, 69 insertions(+), 52 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 1ed4957..192c2dd 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -14,23 +14,21 @@ from __future__ import with_statement from functools import wraps, partial +from random import randint from twisted.python import usage from twisted.plugin import IPlugin from twisted.internet import defer, error, reactor -from twisted.internet.endpoints import TCP4ClientEndpoint from zope.interface import implements from ooni.utils import log from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset -import tempfile import os -import random -import shutil import signal import sys + def timer(secs, e=None): def decorator(func): def _timer(signum, frame): @@ -75,16 +73,15 @@ class ValueChecker(object): raise ValueError("Port out of range") sock_check, ctrl_check = port_check, port_check - port_check_allowed = "must be between 1024 and 65535." - sock_check.coerceDoc = "Port to use for Tor's SocksPort, " + port_check_allowed - ctrl_check.coerceDoc = "Port to use for Tor's ControlPort, " + port_check_allowed + allowed = "must be between 1024 and 65535." + sock_check.coerceDoc = "Port to use for Tor's SocksPort, " +allowed + ctrl_check.coerceDoc = "Port to use for Tor's ControlPort, " +allowed def uid_check(pluggable_transport): - """Check that we are not root when trying to use pluggable transports.""" + """Check that we're not root when trying to use pluggable transports.""" uid, gid = os.getuid(), os.getgid() if uid == 0 and gid == 0: - log.msg("Error: Running bridget as root with --transport=%s not allowed." - % pluggable_transport) + log.msg("Error: Running bridget as root with transports not allowed.") log.msg("Dropping privileges to normal user...") os.setgid(1000) os.setuid(1000) @@ -242,18 +239,22 @@ class BridgetTest(OONITest): if options['bridges']: self.config.UseBridges = 1 make_asset_list(options['bridges'], self.bridges) + if options['relays']: ## first hop must be in TorState().entry_guards to build circuits self.config.EntryNodes = ','.join(relay_list) make_asset_list(options['relays'], self.relays) + if options['socks']: self.socks_port = options['socks'] if options['control']: self.control_port = options['control'] + if options['random']: log.msg("Using randomized ControlPort and SocksPort ...") - self.socks_port = random.randint(1024, 2**16) - self.control_port = random.randint(1024, 2**16) + self.socks_port = randint(1024, 2**16) + self.control_port = randint(1024, 2**16) + if options['torpath']: self.tor_binary = options['torpath'] @@ -274,9 +275,9 @@ class BridgetTest(OONITest): else: raise PTNotFoundException - config.SocksPort = self.socks_port - config.ControlPort = self.control_port - config.CookieAuthentication = 1 + self.config.SocksPort = self.socks_port + self.config.ControlPort = self.control_port + self.config.CookieAuthentication = 1 def load_assets(self): """ @@ -302,12 +303,19 @@ class BridgetTest(OONITest): The :class:`BridgetAsset` line currently being used. """ try: - from ooni.utils import circuit + from tempfile import mkstemp, mkdtemp + from shutil import rmtree + + from twisted.internet.endpoints import TCP4ClientEndpoint + + from ooni.utils import circuit from ooni.lib.txtorcon import TorProcessProtocol from ooni.lib.txtorcon import TorProtocolFactory from ooni.lib.txtorcon import TorConfig, TorState + except ImportError: raise TxtorconImportError + except TxtorconImportError: ## XXX is this something we should add to the reactor? sys.exit() @@ -330,19 +338,19 @@ class BridgetTest(OONITest): try: os.unlink(temp) except OSError: - shutil.rmtree(temp, ignore_errors=True) + rmtree(temp, ignore_errors=True) - @timer(self.circuit_timeout) + #(a)timer(self.circuit_timeout) def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): """Rewrite the Bridge line in our torrc.""" log.msg("Current Bridge: %s" % bridge) if use_pt is False: - new = state.protocol.set_conf('Bridge', bridge) + state.protocol.set_conf('Bridge', bridge) elif use_pt and pt_type is not None: - new = state.protocol.set_conf('Bridge', pt_type +' '+ bridge) + state.protocol.set_conf('Bridge', pt_type +' '+ bridge) else: raise PTNotFoundException - return new + return state.callback def reconfigure_fail(*param): log.msg("Reconfiguring TorConfig with parameters %s failed" % param) @@ -358,7 +366,7 @@ class BridgetTest(OONITest): if len(both) > 0: for bridge in both: bridge_list.remove(bridge) - return state + return state.callback def remove_relays_fail(state): log.msg("Unable to remove public relays from the bridge list.") @@ -429,6 +437,9 @@ class BridgetTest(OONITest): attacher.request_circuit_build(d) return d + def state_attach_fail(state): + log.msg("Attaching custom circuit builder failed.") + def updates(prog, tag, summary): log.msg("%d%%: %s" % (prog, summary)) @@ -443,12 +454,12 @@ class BridgetTest(OONITest): delete_list = [] if data_dir is None: - data_dir = tempfile.mkdtemp(prefix='bridget-tordata') + data_dir = mkdtemp(prefix='bridget-tordata') delete_list.append(data_dir) conf.DataDirectory = data_dir #conf.__OwningControllerProcess = os.getpid() - (fd, torrc) = tempfile.mkstemp(dir=data_dir) + (fd, torrc) = mkstemp(dir=data_dir) delete_list.append(torrc) os.write(fd, conf.create_torrc()) os.close(fd) @@ -459,10 +470,20 @@ class BridgetTest(OONITest): while self.bridges_remaining() > 0: self.current_bridge = self.bridges.pop() - #current_ip = self.current_bridge.split(':', 1)[0] - #print "CURRENT BRIDGE IP %s" % current_ip + if self.config.config.has_key('Bridge'): + log.msg("We now have %d untested bridges..." + % self.bridges_remaining()) + reconf = defer.Deferred() + reconf.addCallback(reconfigure_bridge, state, + self.current_bridge, self.use_pt, + self.pt_type) + reconf.addErrback(reconfigure_fail) + state.chainDeferred(reconf) + #reconfigure_bridge(state, self.current_bridge, + # self.use_pt, self.pt_type) + #current_ip = self.current_bridge.split(':', 1)[0] - if not self.config.config.has_key('Bridge'): + else: self.config.Bridge = self.current_bridge (torrc, self.data_directory, to_delete) = write_torrc( self.config, self.data_directory) @@ -477,11 +498,12 @@ class BridgetTest(OONITest): self.data_directory) state.addCallback(setup_done) state.addErrback(setup_fail) - state.addCallback(remove_relays, self.bridges) - state.addErrback(remove_relays_fail) - state.addCallback(state_attach, self.bridges) - ## XXX write state_attach_fail() - state.addErrback(state_attach_fail) + state.addBoth(remove_relays, self.bridges) + #state.addCallback(remove_relays, self.bridges) + #state.addErrback(remove_relays_fail) + #state.addCallback(state_attach, self.bridges) + #state.addErrback(state_attach_fail) + return state ## XXX see txtorcon.TorControlProtocol.add_event_listener we ## may not need full CustomCircuit class @@ -490,21 +512,19 @@ class BridgetTest(OONITest): ## o if bridges only, try one bridge at a time, but don't build ## circuits, just return ## o if relays only, build circuits from relays - else: - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - try: - state.addCallback(reconfigure_bridge, self.current_bridge, - self.use_pt, self.pt_type) - state.addErrback(reconfigure_fail) - except TimeoutError: - log.msg("Adding %s to unreachable bridges..." - % self.current_bridge) - self.bridges_down.append(self.current_bridge) - else: - log.msg("Adding %s to reachable bridges..." - % self.current_bridge) - self.bridges_up.append(self.current_bridge) + #else: + # try: + # state.addCallback(reconfigure_bridge, self.current_bridge, + # self.use_pt, self.pt_type) + # state.addErrback(reconfigure_fail) + # except TimeoutError: + # log.msg("Adding %s to unreachable bridges..." + # % self.current_bridge) + # self.bridges_down.append(self.current_bridge) + # else: + # log.msg("Adding %s to reachable bridges..." + # % self.current_bridge) + # self.bridges_up.append(self.current_bridge) reactor.run() ## now build circuits @@ -518,12 +538,9 @@ bridget = BridgetTest(None, None, None) ## TODO: ## o cleanup documentation ## x add DataDirectory option -## o check if bridges are public relays +## x check if bridges are public relays ## o take bridge_desc file as input, also be able to give same ## format as output -## o Add assychronous timout for deferred, so that we don't wait +## x Add assychronous timout for deferred, so that we don't wait ## forever for bridges that don't work. ## o Add mechanism for testing through another host -## -## FIX: -## o DataDirectory is not found, or permissions aren't right

1 0

[ooni-probe/master] * Refactored bridget completely.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 6f52ec0c73671e5b1c417cc1b7a0d63266cd8b9c Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Sep 19 02:04:55 2012 +0000 * Refactored bridget completely. * Broke up most of experiment() into tiny functions. * Added custom exceptions and error handlers. * Added more commandline option type enforcers, with custom handlers. * Function to drop any privileges if using pluggable transports. * The generated torrc now includes a Bridge line written to the file descriptor, if using bridges. * The Bridge line properly handles pluggable transports. TODO: Still need to fix the CustomCircuit class and the attacher. --- ooni/plugins/bridget.py | 652 ++++++++++++++++++++++++++--------------------- ooni/utils/circuit.py | 50 ++-- 2 files changed, 380 insertions(+), 322 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index f22741b..1ed4957 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -3,77 +3,163 @@ # # +-----------+ # | BRIDGET | -# | +----------------------------------------------+ -# +--------| Use a slave Tor process to test making a Tor | -# | connection to a list of bridges or relays. | -# +----------------------------------------------+ +# | +--------------------------------------------+ +# +--------| Use a Tor process to test making a Tor | +# | connection to a list of bridges or relays. | +# +--------------------------------------------+ # -# :authors: Arturo Filasto, Isis Lovecruft +# :authors: Isis Lovecruft, Arturo Filasto # :licence: see included LICENSE # :version: 0.1.0-alpha -from __future__ import with_statement -from functools import wraps -from os import getcwd -from os.path import isfile -from os.path import join as pj -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import defer, error, reactor -from zope.interface import implements +from __future__ import with_statement +from functools import wraps, partial +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor +from twisted.internet.endpoints import TCP4ClientEndpoint +from zope.interface import implements +from ooni.utils import log +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset + +import tempfile +import os import random +import shutil import signal import sys -from ooni.utils import log -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset - - -def portCheck(number): - number = int(number) - if number not in range(1024, 65535): - raise ValueError("Port out of range") - -portCheckAllowed = "must be between 1024 and 65535." -sockCheck, ctrlCheck = portCheck, portCheck -sockCheck.coerceDoc = "Port to use for Tor's SocksPort, " + portCheckAllowed -ctrlCheck.coerceDoc = "Port to use for Tor's ControlPort, " + portCheckAllowed - +def timer(secs, e=None): + def decorator(func): + def _timer(signum, frame): + raise TimeoutError, e + def wrapper(*args, **kwargs): + signal.signal(signal.SIGALRM, _timer) + signal.alarm(secs) + try: + res = func(*args, **kwargs) + finally: + signal.alarm(0) + return res + return wraps(func)(wrapper) + return decorator + + +class MissingAssetException(Exception): + """Raised when neither there are neither bridges nor relays to test.""" + def __init__(self): + log.msg("Bridget can't run without bridges or relays to test!") + return sys.exit() + +class PTNoBridgesException(Exception): + """Raised when a pluggable transport is specified, but no bridges.""" + def __init__(self): + log.msg("Pluggable transport requires the bridges option") + return sys.exit() + +class PTNotFoundException(Exception): + def __init__(self, transport_type): + m = "Pluggable Transport type %s was unaccounted " % transport_type + m += "for, please contact isis(at)torproject(dot)org and it will " + m += "get included." + log.msg("%s" % m) + return sys.exit() + +class ValueChecker(object): + def port_check(self, number): + """Check that given ports are in the allowed range.""" + number = int(number) + if number not in range(1024, 65535): + raise ValueError("Port out of range") + + sock_check, ctrl_check = port_check, port_check + port_check_allowed = "must be between 1024 and 65535." + sock_check.coerceDoc = "Port to use for Tor's SocksPort, " + port_check_allowed + ctrl_check.coerceDoc = "Port to use for Tor's ControlPort, " + port_check_allowed + + def uid_check(pluggable_transport): + """Check that we are not root when trying to use pluggable transports.""" + uid, gid = os.getuid(), os.getgid() + if uid == 0 and gid == 0: + log.msg("Error: Running bridget as root with --transport=%s not allowed." + % pluggable_transport) + log.msg("Dropping privileges to normal user...") + os.setgid(1000) + os.setuid(1000) + + def dir_check(d): + """Check that the given directory exists.""" + if not os.isdir(d): + raise ValueError("%s doesn't exist, or has wrong permissions" % d) + + def file_check(f): + if not os.isfile(f): + raise ValueError("%s does not exist, or has wrong permissions" % f) + +class RandomPortException(Exception): + """Raised when using a random port conflicts with configured ports.""" + def __init__(self): + log.msg("Unable to use random and specific ports simultaneously") + return sys.exit() + +class TimeoutError(Exception): + """Raised when a timer runs out.""" + pass + +class TxtorconImportError(ImportError): + """Raised when /ooni/lib/txtorcon cannot be imported from.""" + cwd, tx = os.getcwd(), 'lib/txtorcon/torconfig.py' + try: + log.msg("Unable to import from ooni.lib.txtorcon") + if cwd.endswith('ooni'): + check = os.path.join(cwd, tx) + else: + check = os.path.join(cwd, 'ooni/'+tx) + assert isfile(check) + except: + log.msg("Error: Some OONI libraries are missing!") + log.msg("Please go to /ooni/lib/ and do \"make all\"") class BridgetArgs(usage.Options): + """Commandline options.""" + vc = ValueChecker() + optParameters = [ ['bridges', 'b', None, 'File listing bridge IP:ORPorts to test'], ['relays', 'f', None, 'File listing relay IPs to test'], - ['socks', 's', 9049, None, portCheck], - ['control', 'c', 9052, None, portCheck], + ['socks', 's', 9049, None, vc.sock_check], + ['control', 'c', 9052, None, vc.ctrl_check], ['torpath', 'p', None, 'Path to the Tor binary to use'], ['datadir', 'd', None, 'Tor DataDirectory to use'], - ['transport', 't', None, + ['transport', 't', None, 'Tor ClientTransportPlugin'], ['resume', 'r', 0, 'Resume at this index']] - optFlags = [ - ['random', 'x', 'Use random ControlPort and SocksPort']] + optFlags = [['random', 'x', 'Use random ControlPort and SocksPort']] def postOptions(self): - if self['transport'] and not self['bridges']: - e = "Pluggable transport requires the bridges option" - raise usage.UsageError, e - if self['socks'] and self['control']: + if not self['bridges'] and not self['relays']: + raise MissingAssetException + if self['transport']: + vc.uid_check(self['transport']) + if not self['bridges']: + raise PTNoBridgesException + if self['socks'] or self['control']: if self['random']: - e = "Unable to use random and specific ports simultaneously" - raise usage.usageError, e + raise RandomPortException + if self['datadir']: + vc.dir_check(self['datadir']) + if self['torpath']: + vc.file_check(self['torpath']) class BridgetAsset(Asset): - """ - Class for parsing bridget Assets ignoring commented out lines. - """ + """Class for parsing bridget Assets ignoring commented out lines.""" def __init__(self, file=None): self = Asset.__init__(self, file) @@ -89,17 +175,15 @@ class BridgetTest(OONITest): :ivar config: An :class:`ooni.lib.txtorcon.TorConfig` instance. - :ivar relay_list: - A list of all provided relays to test. We have to do this because - txtorcon.TorState().entry_guards won't build a custom circuit if the - first hop isn't in the torrc's EntryNodes. - :ivar bridge_list: + :ivar relays: + A list of all provided relays to test. + :ivar bridges: A list of all provided bridges to test. :ivar socks_port: Integer for Tor's SocksPort. :ivar control_port: Integer for Tor's ControlPort. - :ivar plug_transport: + :ivar transport: String defining the Tor's ClientTransportPlugin, for testing a bridge's pluggable transport functionality. :ivar tor_binary: @@ -116,99 +200,83 @@ class BridgetTest(OONITest): def initialize(self): """ Extra initialization steps. We only want one child Tor process - running, so we need to deal with the creation of TorConfig() only - once, before the experiment runs. + running, so we need to deal with most of the TorConfig() only once, + before the experiment runs. """ self.socks_port = 9049 self.control_port = 9052 + self.circuit_timeout = 90 self.tor_binary = '/usr/sbin/tor' self.data_directory = None - self.circuit_timeout = 90 + self.use_pt = False + self.pt_type = None + + self.bridges, self.bridges_up, self.bridges_down = ([] for i in range(3)) + self.bridges_remaining = lambda: len(self.bridges) + self.bridges_down_count = lambda: len(self.bridges_down) + self.current_bridge = None + + self.relays, self.relays_up, self.relays_down = ([] for i in range(3)) + self.relays_remaining = lambda: len(self.relays) + self.relays_down_count = lambda: len(self.relays_down) + self.current_relay = None + + def make_asset_list(opt, lst): + log.msg("Loading information from %s ..." % opt) + with open(opt) as opt_file: + for line in opt_file.readlines(): + if line.startswith('#'): + continue + else: + lst.append(line.replace('\n','')) if self.local_options: - options = self.local_options - - if not options['bridges'] and not options['relays']: - self.suicide = True - return - try: from ooni.lib.txtorcon import TorConfig except ImportError: - log.msg ("Bridget: Unable to import from ooni.lib.txtorcon") - wd, tx = getcwd(), 'lib/txtorcon/torconfig.py' - chk = pj(wd,tx) if wd.endswith('ooni') else pj(wd,'ooni/'+tx) - try: - assert isfile(chk) - except: - log.msg("Error: Some OONI libraries are missing!") - log.msg("Please go to /ooni/lib/ and do \"make all\"") + raise TxtorconImportError - self.config = TorConfig() + options = self.local_options + config = self.config = TorConfig() if options['bridges']: self.config.UseBridges = 1 - + make_asset_list(options['bridges'], self.bridges) if options['relays']: - ## Stupid hack for testing only relays: - ## Tor doesn't use EntryNodes when UseBridges is enabled, but - ## config.state.entry_guards needs to include the first hop to - ## build a custom circuit. + ## first hop must be in TorState().entry_guards to build circuits self.config.EntryNodes = ','.join(relay_list) - + make_asset_list(options['relays'], self.relays) if options['socks']: self.socks_port = options['socks'] - if options['control']: self.control_port = options['control'] - if options['random']: log.msg("Using randomized ControlPort and SocksPort ...") self.socks_port = random.randint(1024, 2**16) self.control_port = random.randint(1024, 2**16) - if options['torpath']: self.tor_binary = options['torpath'] - if options['datadir']: - self.config.DataDirectory = options['datadir'] + if self.local_options['datadir']: + self.data_directory = local_options['datadir'] + else: + self.data_directory = None if options['transport']: - ## ClientTransportPlugin transport socks4|socks5 IP:PORT + self.use_pt = True + log.msg("Using ClientTransportPlugin %s" % options['transport']) + [self.pt_type, pt_exec] = options['transport'].split(' ', 1) + ## ClientTransportPlugin transport exec path-to-binary [options] - if not options['bridges']: - e = "You must use the bridge option to test a transport." - raise usage.UsageError("%s" % e) + ## XXX we need a better way to deal with all PTs + if self.pt_type == "obfs2": + config.ClientTransportPlugin = self.pt_type + " " + pt_exec else: - ## XXX fixme there's got to be a better way to check the exec - ## - ## we could use: - ## os.setgid( NEW_GID ) - ## os.setuid( NEW_UID ) - ## to drop any and all privileges - assert type(options['transport']) is str - [self.pt_type, - self.pt_exec] = options['transport'].split(' ', 1) - log.msg("Using ClientTransportPlugin %s %s" - % (self.pt_type, self.pt_exec)) - self.pt_use = True - - ## XXX fixme we need a better way to deal with all PTs - if self.pt_type == "obfs2": - self.ctp = self.pt_type +" "+ self.pt_exec - else: - m = "Pluggable Transport type %s was " % self.pt_type - m += "unaccounted for, please contact isis (at) " - m += "torproject (dot) org, with info and it'll get " - m += "included." - log.msg("%s" % m) - self.ctp = None - else: - self.pt_use = False + raise PTNotFoundException - self.config.SocksPort = self.socks_port - self.config.ControlPort = self.control_port - self.config.save() + config.SocksPort = self.socks_port + config.ControlPort = self.control_port + config.CookieAuthentication = 1 def load_assets(self): """ @@ -216,32 +284,14 @@ class BridgetTest(OONITest): should be given in the form IP:ORport. We don't want to load these as assets, because it's inefficient to start a Tor process for each one. """ - assets = {} - self.bridge_list = [] - self.relay_list = [] - - ## XXX fix me - ## we should probably find a more memory nice way to load addresses, - ## in case the files are really large + assets = {} if self.local_options: - - def make_asset_list(opt, lst): - log.msg("Loading information from %s ..." % opt) - with open(opt) as opt_file: - for line in opt_file.readlines(): - if line.startswith('#'): - continue - else: - lst.append(line.replace('\n','')) - if self.local_options['bridges']: - make_asset_list(self.local_options['bridges'], - self.bridge_list) - assets.update({'bridges': self.bridge_list}) + assets.update({'bridge': + BridgetAsset(self.local_options['bridges'])}) if self.local_options['relays']: - make_asset_list(self.local_options['relays'], - self.relay_list) - assets.update({'relays': self.relay_list}) + assets.update({'relay': + BridgetAsset(self.local_options['relays'])}) return assets def experiment(self, args): @@ -249,55 +299,69 @@ class BridgetTest(OONITest): XXX fill me in :param args: - The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being - used. - :meth launch_tor: - Returns a Deferred which callbacks with a - :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol - <TorProcessProtocol>` connected to the fully-bootstrapped Tor; - this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol - <TorControlProtocol>` instance as .protocol. + The :class:`BridgetAsset` line currently being used. """ try: - from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher - from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState - from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor + from ooni.utils import circuit + from ooni.lib.txtorcon import TorProcessProtocol + from ooni.lib.txtorcon import TorProtocolFactory + from ooni.lib.txtorcon import TorConfig, TorState except ImportError: - log.msg("Error: Unable to import from ooni.lib.txtorcon") - wd, tx = getcwd(), 'lib/txtorcon/torconfig.py' - chk = pj(wd,tx) if wd.endswith('ooni') else pj(wd,'ooni/'+tx) - try: - assert isfile(chk) - except: - log.msg("Error: Some OONI libraries are missing!") - log.msg(" Please go to /ooni/lib/ and do \"make all\"") - return sys.exit() + raise TxtorconImportError + except TxtorconImportError: + ## XXX is this something we should add to the reactor? + sys.exit() def bootstrap(ctrl): """ Launch a Tor process with the TorConfig instance returned from - initialize(). + initialize() and write_torrc(). """ conf = TorConfig(ctrl) conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) log.msg("Tor process connected, bootstrapping ...") - def reconf_controller(conf, bridge): - ## if bridges and relays, use one bridge then build a circuit - ## from three relays - conf.Bridge = "" - ## XXX do we need a SIGHUP to restart? + def delete_temp(delete_list): + """ + Given a list of files or directories to delete, delete all and + suppress all errors. + """ + for temp in delete_list: + try: + os.unlink(temp) + except OSError: + shutil.rmtree(temp, ignore_errors=True) + + @timer(self.circuit_timeout) + def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): + """Rewrite the Bridge line in our torrc.""" + log.msg("Current Bridge: %s" % bridge) + if use_pt is False: + new = state.protocol.set_conf('Bridge', bridge) + elif use_pt and pt_type is not None: + new = state.protocol.set_conf('Bridge', pt_type +' '+ bridge) + else: + raise PTNotFoundException + return new - ## XXX see txtorcon.TorControlProtocol.add_event_listener we - ## may not need full CustomCircuit class + def reconfigure_fail(*param): + log.msg("Reconfiguring TorConfig with parameters %s failed" % param) + reactor.stop() - ## if bridges only, try one bridge at a time, but don't build - ## circuits, just return - ## if relays only, build circuits from relays + def remove_relays(state, bridge_list): + """ + Remove bridges from our bridge list which are also listed as + public relays. + """ + ips = map(lambda addr: addr.split(':',1)[0], bridge_list) + both = set(state.relays.values()).intersection(ips) + if len(both) > 0: + for bridge in both: + bridge_list.remove(bridge) + return state - def reconf_fail(args): - log.msg("Reconfiguring Tor config with args %s failed" % args) - reactor.stop() + def remove_relays_fail(state): + log.msg("Unable to remove public relays from the bridge list.") def setup_fail(args): log.msg("Setup Failed.") @@ -310,152 +374,148 @@ class BridgetTest(OONITest): state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) report.update({'success': args}) + def start_tor(reactor, update, torrc, to_delete, control_port, + tor_binary, data_directory): + """ + Create a TCP4ClientEndpoint at our control_port, and connect + it to our reactor and a spawned Tor process. Compare with + :meth:`txtorcon.launch_tor` for differences. + """ + ## XXX do we need self.reactor? + end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port) + connection_creator = partial(end_point.connect, TorProtocolFactory()) + process_protocol = TorProcessProtocol(connection_creator, updates) + process_protocol.to_delete = to_delete + reactor.addSystemEventTrigger('before', 'shutdown', + partial(delete_temp, to_delete)) + try: + transport = reactor.spawnProcess(process_protocol, + tor_binary, + args=(tor_binary,'-f',torrc), + env={'HOME': data_directory}, + path=data_directory) + transport.closeStdin() + except RuntimeError, e: + process_protocol.connected_cb.errback(e) + finally: + return process_protocol.connected_cb + + def state_complete(state): + """Called when we've got a TorState.""" + log.msg("We've completely booted up a Tor version %s at PID %d" + % (state.protocol.version, state.tor_pid)) + + log.msg("This Tor has the following %d Circuits:" + % len(state.circuits)) + for circ in state.circuits.values(): + log.msg("%s" % circ) + + return state + + def state_attach(state, relay_list): + log.msg("Setting up custom circuit builder...") + attacher = CustomCircuit(state) + state.set_attacher(attacher, reactor) + state.add_circuit_listener(attacher) + + for circ in state.circuits.values(): + for relay in circ.path: + try: + relay_list.remove(relay) + except KeyError: + continue + ## XXX how do we attach to circuits with bridges? + d = defer.Deferred() + attacher.request_circuit_build(d) + return d + def updates(prog, tag, summary): log.msg("%d%%: %s" % (prog, summary)) - if not self.circuit_timeout: - self.circuit_timeout = 90 - - class TimeoutError(Exception): - pass - - def stupid_timer(secs, e=None): - def decorator(func): - def _timer(signum, frame): - raise TimeoutError, e - def wrapper(*args, **kwargs): - signal.signal(signal.SIGALRM, _timer) - signal.alarm(secs) - try: - res = func(*args, **kwargs) - finally: - signal.alarm(0) - return res - return wraps(func)(wrapper) - return decorator + def write_torrc(conf, data_dir=None): + """ + Create a torrc in our data_directory. If we don't yet have a + data_directory, create a temporary one. Any temporary files or + folders are added to delete_list. + + :return: delete_list, data_dir, torrc + """ + delete_list = [] + + if data_dir is None: + data_dir = tempfile.mkdtemp(prefix='bridget-tordata') + delete_list.append(data_dir) + conf.DataDirectory = data_dir + #conf.__OwningControllerProcess = os.getpid() + + (fd, torrc) = tempfile.mkstemp(dir=data_dir) + delete_list.append(torrc) + os.write(fd, conf.create_torrc()) + os.close(fd) + return torrc, data_dir, delete_list - @stupid_timer(self.circuit_timeout) - def _launch_tor_and_report_bridge_status(_config, - _reactor, - _updates, - _binary, - _callback, - _callback_arg, - _errback): - """The grossest Python function you've ever seen.""" - - log.msg("Starting Tor ...") - ## :return: a Deferred which callbacks with a TorProcessProtocol - ## connected to the fully-bootstrapped Tor; this has a - ## txtorcon.TorControlProtocol instance as .protocol. - proc_proto = launch_tor(_config, _reactor, progress_updates=_updates, - tor_binary=_binary).addCallback(_callback, - _callback_arg).addErrback(_errback) - return proc_proto - ## now build circuits - - - - if len(args) == 0: - log.msg("Bridget can't run without bridges or relays to test!") - log.msg("Exiting ...") - return sys.exit() - else: - log.msg("Bridget: initiating test ... ") - self.london_bridge = [] - self.working_bridge = [] - if len(self.bridge_list) >= 1: - self.untested_bridge_count = len(self.bridge_list) - self.burnt_bridge_count = len(self.london_bridge) - self.current_bridge = None + log.msg("Bridget: initiating test ... ") + + while self.bridges_remaining() > 0: + self.current_bridge = self.bridges.pop() + #current_ip = self.current_bridge.split(':', 1)[0] + #print "CURRENT BRIDGE IP %s" % current_ip + + if not self.config.config.has_key('Bridge'): + self.config.Bridge = self.current_bridge + (torrc, self.data_directory, to_delete) = write_torrc( + self.config, self.data_directory) + + log.msg("Starting Tor ...") + log.msg("Using the following as our torrc:\n%s" + % self.config.create_torrc()) + report = {'tor_config': self.config.create_torrc()} + + state = start_tor(reactor, updates, torrc, to_delete, + self.control_port, self.tor_binary, + self.data_directory) + state.addCallback(setup_done) + state.addErrback(setup_fail) + state.addCallback(remove_relays, self.bridges) + state.addErrback(remove_relays_fail) + state.addCallback(state_attach, self.bridges) + ## XXX write state_attach_fail() + state.addErrback(state_attach_fail) - ## while bridges are in the bucket - while self.config.UseBridges and self.untested_bridge_count > 0: + ## XXX see txtorcon.TorControlProtocol.add_event_listener we + ## may not need full CustomCircuit class + ## o if bridges and relays, use one bridge then build a circuit + ## from three relays + ## o if bridges only, try one bridge at a time, but don't build + ## circuits, just return + ## o if relays only, build circuits from relays + else: + log.msg("We now have %d untested bridges..." + % self.bridges_remaining()) try: - ## if the user wanted to use pluggable transports - assert self.pt_use is True - except AssertionError as no_use_pt: - ## if the user didn't want to use pluggable transports - log.msg("Configuring Bridge line without pluggable transport") - log.msg("%s" % no_use_pt) - self.pt_good = False + state.addCallback(reconfigure_bridge, self.current_bridge, + self.use_pt, self.pt_type) + state.addErrback(reconfigure_fail) + except TimeoutError: + log.msg("Adding %s to unreachable bridges..." + % self.current_bridge) + self.bridges_down.append(self.current_bridge) else: - ## user wanted PT, and we recognized transport - if self.ctp is not None: - try: - assert self.ctp is str - except AssertionError as not_a_string: - log.msg("Error: ClientTransportPlugin string unusable: %s" - % not_a_string) - log.msg(" Exiting ...") - sys.exit() - else: - ## configure the transport - self.config.ClientTransportPlugin = self.ctp - self.pt_good = True - ## user wanted PT, but we didn't recognize it - else: - log.msg("Error: Unable to use ClientTransportPlugin %s %s" - % (self.pt_type, self.pt_exec)) - log.msg(" Exiting...") - sys.exit() - ## whether or not we're using a pluggable transport, we need - ## to set the Bridge line - finally: - log.msg("We now have %d bridges in our list..." - % self.untested_bridge_count) - self.current_bridge = self.bridge_list.pop() - self.untested_bridge_count -= 1 - - log.msg("Current Bridge: %s" % self.current_bridge) - - if self.pt_good: - self.config.Bridge = self.pt_type +" "+ self.current_bridge - else: - self.config.Bridge = self.current_bridge - - log.msg("Using the following as our torrc:\n%s" - % self.config.create_torrc()) - report = {'tor_config': self.config.create_torrc()} - self.config.save() + log.msg("Adding %s to reachable bridges..." + % self.current_bridge) + self.bridges_up.append(self.current_bridge) - try: - _launch_tor_and_report_bridge_status(self.config, - reactor, - updates, - self.tor_binary, - bootstrap, - self.config, - setup_fail) - except TimeoutError: - log.msg("Adding %s to bad bridges..." % self.current_bridge) - self.london_bridge.append(self.current_bridge) - else: - log.msg("Adding %s to good bridges..." % self.current_bridge) - self.working_bridge.append(self.current_bridge) + reactor.run() + ## now build circuits - d = defer.Deferred() - d.addCallback(log.msg, 'Working Bridges:\n%s\nUnreachable Bridges:\n%s\n' - % (self.working_bridge, self.london_bridge)) - return d - - def control(self, exp_res): - exp_res ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) ## ISIS' NOTES ## ----------- -## self.config.save() only needs to be called if Tor is already running. -## -## to test gid, uid, and euid: -## with open('/proc/self/state') as uidfile: -## print uidfile.read(1000) -## ## TODO: -## o add option for any kwarg=arg self.config setting ## o cleanup documentation ## x add DataDirectory option ## o check if bridges are public relays diff --git a/ooni/utils/circuit.py b/ooni/utils/circuit.py index bb1ab11..0c77dd1 100644 --- a/ooni/utils/circuit.py +++ b/ooni/utils/circuit.py @@ -13,6 +13,9 @@ from zope.interface import implements from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +from ooni.utils import log + +import random class CustomCircuit(CircuitListenerMixin): @@ -28,7 +31,7 @@ class CustomCircuit(CircuitListenerMixin): def waiting_on(self, circuit): for (circid, d) in self.waiting_circuits: if circuit.id == circid: - return true + return True return False def circuit_extend(self, circuit, router): @@ -36,8 +39,7 @@ class CustomCircuit(CircuitListenerMixin): if circuit.purpose != 'GENERAL': return if self.waiting_on(circuit): - log.msg("Circuit %d (%s)" - % (circuit.id, router.id_hex)) + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) def circuit_built(self, circuit): "ICircuitListener" @@ -47,7 +49,7 @@ class CustomCircuit(CircuitListenerMixin): log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) for (circid, d) in self.waiting_circuits: if circid == circuit.id: - self.waiting_circuits.remove(circid, d) + self.waiting_circuits.remove((circid, d)) d.callback(circuit) def circuit_failed(self, circuit, reason): @@ -57,7 +59,7 @@ class CustomCircuit(CircuitListenerMixin): circid, d = None, None for x in self.waiting_circuits: if x[0] == circuit.id: - circid, d, stream_cc = x + circid, d = x if d is None: raise Exception("Expected to find circuit.") @@ -71,36 +73,32 @@ class CustomCircuit(CircuitListenerMixin): TorInfo.dump(self) def request_circuit_build(self, deferred): - entries = self.state.entry_guards.value() - relays = self.state.routers.values() - log.msg("We have these nodes listed as entry guards:") - log.msg("%s" % entries) - log.msg("We have these nodes listed as relays:") - log.msg("%s" % relays) - path = [random.choice(entries), - random.choice(relays), - random.choice(relays)] + if self.state.relays_remaining() > 0: + first, middle,last = (self.state.relays.pop() + for i in range(3)) + else: + first = random.choice(self.state.entry_guards.values()) + middle, last = (random.choice(self.state.routers.values()) + for i in range(2)) + path = [first, middle, last] + log.msg("Requesting a circuit: %s" - % '-->'.join(map(lambda x: x.location.countrycode, - path))) + % '->'.join(map(lambda node: node, path))) class AppendWaiting: def __init__(self, attacher, deferred): self.attacher = attacher self.d = deferred - - def __call__(self, circuit): + def __call__(self, circ): """ Return from build_circuit is a Circuit, however, we want to wait until it is built before we can issue an attach on it and callback to the Deferred we issue here. """ - log.msg("Circuit %s is in progress ..." % circuit.id) - self.attacher.waiting_circuits.append((circuit.id, - self.d)) - - fin = self.state.build_circuit(path) - fin.addCallback(AppendWaiting(self, deferred_to_callback)) - fin.addErrback(log.err) - return fin + log.msg("Circuit %s is in progress ..." % circ.id) + self.attacher.waiting_circuits.append((circ.id, self.d)) + + return self.state.build_circuit(path).addCallback( + AppendWaiting(self, deferred_to_callback)).addErrback( + log.err)

1 0

[ooni-probe/master] * Moving to using @defer.inlineCallbacks for rewriting the torrc Bridge line.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 4a6b164cbefb5dca519ece6cfd62486c8a14206e Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 20 03:38:29 2012 +0000 * Moving to using @defer.inlineCallbacks for rewriting the torrc Bridge line. NOT DONE YET. NOT TESTED YET. * Moving to bridges being assets and relays being a list attached to our TorState() instance, so that we can pass it to CustomCircuit. TODO: Make the CustomCircuit() instance handle iteration through relay list, and provide access to objects for unreachable and reachable relays. We should probably check how onionoo handles this. --- ooni/plugins/bridget.py | 134 +++++++++++++++++++++++++++++++---------------- ooni/utils/circuit.py | 18 +++--- 2 files changed, 97 insertions(+), 55 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 192c2dd..07d7269 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -341,55 +341,91 @@ class BridgetTest(OONITest): rmtree(temp, ignore_errors=True) #(a)timer(self.circuit_timeout) + @defer.inlineCallbacks def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): - """Rewrite the Bridge line in our torrc.""" - log.msg("Current Bridge: %s" % bridge) - if use_pt is False: - state.protocol.set_conf('Bridge', bridge) - elif use_pt and pt_type is not None: - state.protocol.set_conf('Bridge', pt_type +' '+ bridge) - else: - raise PTNotFoundException - return state.callback + """ + Rewrite the Bridge line in our torrc. If use of pluggable + transports was specified, rewrite the line as: + Bridge <transport_type> <ip>:<orport> + Otherwise, rewrite in the standard form. + """ + log.msg("Current Bridge: %s" % bridge) + try: + if use_pt is False: + reset_tor = yield state.protocol.set_conf('Bridge', bridge) + elif use_pt and pt_type is not None: + reset_tor = yield state.protocol.set_conf('Bridge', + pt_type +' '+ bridge) + else: + raise PTNotFoundException + + controller_response = reset_tor.callback + if not controller_response: + defer.returnValue((state.callback, None)) + else: + defer.returnValue((state.callback, controller_response)) + except Exception, e: + log.msg("Reconfiguring torrc Bridge line failed with %s" % bridge) def reconfigure_fail(*param): log.msg("Reconfiguring TorConfig with parameters %s failed" % param) reactor.stop() - def remove_relays(state, bridge_list): + @defer.inlineCallbacks + def remove_public_relays(state, bridges): """ Remove bridges from our bridge list which are also listed as public relays. """ - ips = map(lambda addr: addr.split(':',1)[0], bridge_list) - both = set(state.relays.values()).intersection(ips) - if len(both) > 0: - for bridge in both: - bridge_list.remove(bridge) - return state.callback + IPs = map(lambda addr: addr.split(':',1)[0], bridges) + both = set(state.routers.values()).intersection(IPs) - def remove_relays_fail(state): - log.msg("Unable to remove public relays from the bridge list.") + def __remove_line__(node, bridges=bridges): + for line in bridges: + if line.startswith(node): + log.msg("Removing %s because it is a public relay" % node) + bridges.remove(line) - def setup_fail(args): - log.msg("Setup Failed.") - report.update({'failed': args}) + if len(both) > 0: + try: + updated = yield map(lambda node: __remove_line__(node), both) + if not updated: + ## XXX do these need to be state.callback? + defer.returnValue(state) + else: + defer.returnValue(state) + except Exception, e: + log.msg("Unable to remove public relays from bridge list:\n%s" + % both) + log.err(e) + + def setup_fail(proto, bridge_list, relay_list): + log.err("Setup Failed: %s" % proto) + log.err("We were given bridge list:\n%s\nAnd our relay list:\n%s\n" + % (bridge_list, relay_list)) + report.update({'setup_fail': 'FAILED', + 'proto': proto, + 'bridge_list': bridge_list, + 'relay_list': relay_list}) reactor.stop() - def setup_done(proto): + def setup_done(proto, bridge_list, relay_list): log.msg("Setup Complete: %s" % proto) state = TorState(proto.tor_protocol) state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) - report.update({'success': args}) - - def start_tor(reactor, update, torrc, to_delete, control_port, - tor_binary, data_directory): + if bridge_list is not None: + state.post_bootstrap.addCallback(remove_public_relays, bridge_list) + if relay_list is not None: + raise NotImplemented + #report.update({'success': args}) + + def start_tor(reactor, update, torrc, to_delete, control_port, tor_binary, + data_directory, bridge_list=None, relay_list=None): """ Create a TCP4ClientEndpoint at our control_port, and connect it to our reactor and a spawned Tor process. Compare with :meth:`txtorcon.launch_tor` for differences. """ - ## XXX do we need self.reactor? end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port) connection_creator = partial(end_point.connect, TorProtocolFactory()) process_protocol = TorProcessProtocol(connection_creator, updates) @@ -406,9 +442,9 @@ class BridgetTest(OONITest): except RuntimeError, e: process_protocol.connected_cb.errback(e) finally: - return process_protocol.connected_cb + return process_protocol.connected_cb, bridge_list, relay_list - def state_complete(state): + def state_complete(state, bridge_list=None, relay_list=None): """Called when we've got a TorState.""" log.msg("We've completely booted up a Tor version %s at PID %d" % (state.protocol.version, state.tor_pid)) @@ -418,7 +454,12 @@ class BridgetTest(OONITest): for circ in state.circuits.values(): log.msg("%s" % circ) - return state + if bridge_list is not None and relay_list is None: + return state, bridge_list + elif bridge_list is None and relay_list is not None: + raise NotImplemented + else: + return state, None def state_attach(state, relay_list): log.msg("Setting up custom circuit builder...") @@ -426,12 +467,12 @@ class BridgetTest(OONITest): state.set_attacher(attacher, reactor) state.add_circuit_listener(attacher) - for circ in state.circuits.values(): - for relay in circ.path: - try: - relay_list.remove(relay) - except KeyError: - continue + #for circ in state.circuits.values(): + # for relay in circ.path: + # try: + # relay_list.remove(relay) + # except KeyError: + # continue ## XXX how do we attach to circuits with bridges? d = defer.Deferred() attacher.request_circuit_build(d) @@ -457,7 +498,6 @@ class BridgetTest(OONITest): data_dir = mkdtemp(prefix='bridget-tordata') delete_list.append(data_dir) conf.DataDirectory = data_dir - #conf.__OwningControllerProcess = os.getpid() (fd, torrc) = mkstemp(dir=data_dir) delete_list.append(torrc) @@ -468,8 +508,16 @@ class BridgetTest(OONITest): log.msg("Bridget: initiating test ... ") - while self.bridges_remaining() > 0: - self.current_bridge = self.bridges.pop() + #while self.bridges_remaining() > 0: + while args['bridge']: + + #self.current_bridge = self.bridges.pop() + self.current_bridge = args['bridge'] + try: + self.bridges.remove(self.current_bridge) + except ValueError, ve: + log.err(ve) + if self.config.config.has_key('Bridge'): log.msg("We now have %d untested bridges..." % self.bridges_remaining()) @@ -479,9 +527,6 @@ class BridgetTest(OONITest): self.pt_type) reconf.addErrback(reconfigure_fail) state.chainDeferred(reconf) - #reconfigure_bridge(state, self.current_bridge, - # self.use_pt, self.pt_type) - #current_ip = self.current_bridge.split(':', 1)[0] else: self.config.Bridge = self.current_bridge @@ -499,10 +544,7 @@ class BridgetTest(OONITest): state.addCallback(setup_done) state.addErrback(setup_fail) state.addBoth(remove_relays, self.bridges) - #state.addCallback(remove_relays, self.bridges) - #state.addErrback(remove_relays_fail) - #state.addCallback(state_attach, self.bridges) - #state.addErrback(state_attach_fail) + return state ## XXX see txtorcon.TorControlProtocol.add_event_listener we diff --git a/ooni/utils/circuit.py b/ooni/utils/circuit.py index 0c77dd1..6ee0720 100644 --- a/ooni/utils/circuit.py +++ b/ooni/utils/circuit.py @@ -3,27 +3,27 @@ # ---------- # Utilities for working with Tor circuits. # -# This code is largely taken from the txtorcon documentation, and as -# such any and all credit should go to meejah. +# This code is largely taken from attach_streams_by_country.py in the txtorcon +# documentation, and as such any and all credit should go to meejah. Minor +# adjustments have been made to use OONI's logging system, and to build custom +# circuits without actually attaching streams. # -# :author: Mike Warren, Isis Lovecruft +# :author: Meejah, Isis Lovecruft # :license: see included license file # :version: 0.1.0-alpha # -from zope.interface import implements + +import random from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +from ooni.lib.txtorcon import TorInfo from ooni.utils import log - -import random +from zope.interface import implements class CustomCircuit(CircuitListenerMixin): implements(IStreamAttacher) - from txtorcon.interface import IRouterContainer - from txtorcon.interface import ICircuitContainer - def __init__(self, state): self.state = state self.waiting_circuits = []

1 0

[ooni-probe/master] Remove now unused stuff from oonicli
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit b7c5c1fec12da443a30f0cbda1ebb44a8ba7612e Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 20 19:34:48 2012 +0000 Remove now unused stuff from oonicli --- ooni/oonicli.py | 204 +------------------------------------------------------ 1 files changed, 3 insertions(+), 201 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index f0287b4..1e7613b 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -34,68 +34,6 @@ from twisted.python.compat import set from twisted.trial import itrial from twisted.trial import runner as irunner - -def _parseLocalVariables(line): - """ - Accepts a single line in Emacs local variable declaration format and - returns a dict of all the variables {name: value}. - Raises ValueError if 'line' is in the wrong format. - - See http://www.gnu.org/software/emacs/manual/html_node/File-Variables.html - """ - paren = '-*-' - start = line.find(paren) + len(paren) - end = line.rfind(paren) - if start == -1 or end == -1: - raise ValueError("%r not a valid local variable declaration" % (line,)) - items = line[start:end].split(';') - localVars = {} - for item in items: - if len(item.strip()) == 0: - continue - split = item.split(':') - if len(split) != 2: - raise ValueError("%r contains invalid declaration %r" - % (line, item)) - localVars[split[0].strip()] = split[1].strip() - return localVars - - -def loadLocalVariables(filename): - """ - Accepts a filename and attempts to load the Emacs variable declarations - from that file, simulating what Emacs does. - - See http://www.gnu.org/software/emacs/manual/html_node/File-Variables.html - """ - f = file(filename, "r") - lines = [f.readline(), f.readline()] - f.close() - for line in lines: - try: - return _parseLocalVariables(line) - except ValueError: - pass - return {} - - -def getTestModules(filename): - testCaseVar = loadLocalVariables(filename).get('test-case-name', None) - if testCaseVar is None: - return [] - return testCaseVar.split(',') - - -def isTestFile(filename): - """ - Returns true if 'filename' looks like a file containing unit tests. - False otherwise. Doesn't care whether filename exists. - """ - basename = os.path.basename(filename) - return (basename.startswith('test_') - and os.path.splitext(basename)[1] == ('.py')) - - class Options(usage.Options, app.ReactorSelectionMixin): synopsis = """%s [options] [[file|package|module|TestCase|testmethod]...] """ % (os.path.basename(sys.argv[0]),) @@ -104,37 +42,16 @@ class Options(usage.Options, app.ReactorSelectionMixin): "network tests. These are loaded from modules, packages and" "files listed on the command line") - optFlags = [["help", "h"], - ["rterrors", "e", "realtime errors, print out tracebacks as " - "soon as they occur"], - ["debug", "b", "Run tests in the Python debugger. Will load " - "'.pdbrc' from current directory if it exists."], - ["debug-stacktraces", "B", "Report Deferred creation and " - "callback stack traces"], - ["nopm", None, "don't automatically jump into debugger for " - "postmorteming of exceptions"], - ["force-gc", None, "Have OONI run gc.collect() before and " - "after each test case."], - ["unclean-warnings", None, - "Turn dirty reactor errors into warnings"], - ["no-recurse", "N", "Don't recurse into packages"], - ['help-reporters', None, - "Help on available output plugins (reporters)"] - ] + optFlags = [["help", "h"]] optParameters = [ ["reportfile", "o", "report.yaml", "report file name"], ["logfile", "l", "test.log", "log file name"], ['temp-directory', None, '_ooni_temp', - 'Path to use as working directory for tests.'], - ['reporter', None, 'default', - 'The reporter to use for this test run. See --help-reporters for ' - 'more info.']] + 'Path to use as working directory for tests.'] + ] compData = usage.Completions( - optActions={"tbformat": usage.CompleteList(["plain", "emacs", "cgitb"]), - "logfile": usage.CompleteFiles(descr="log file name"), - }, extraActions=[usage.CompleteFiles( "*.py", descr="file | module | package | TestCase | testMethod", repeat=True)], @@ -146,55 +63,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): self['test'] = None usage.Options.__init__(self) - - def coverdir(self): - """ - Return a L{FilePath} representing the directory into which coverage - results should be written. - """ - coverdir = 'coverage' - result = FilePath(self['temp-directory']).child(coverdir) - print "Setting coverage directory to %s." % (result.path,) - return result - - - def opt_coverage(self): - """ - Generate coverage information in the I{coverage} file in the - directory specified by the I{trial-temp} option. - """ - import trace - self.tracer = trace.Trace(count=1, trace=0) - sys.settrace(self.tracer.globaltrace) - - - def opt_testmodule(self, filename): - """ - Filename to grep for test cases (-*- test-case-name) - """ - # If the filename passed to this parameter looks like a test module - # we just add that to the test suite. - # - # If not, we inspect it for an Emacs buffer local variable called - # 'test-case-name'. If that variable is declared, we try to add its - # value to the test suite as a module. - # - # This parameter allows automated processes (like Buildbot) to pass - # a list of files to OONI with the general expectation of "these files, - # whatever they are, will get tested" - if not os.path.isfile(filename): - sys.stderr.write("File %r doesn't exist\n" % (filename,)) - return - - filename = os.path.abspath(filename) - self['test'] = filename - - if isTestFile(filename): - self['tests'].add(filename) - else: - self['tests'].update(getTestModules(filename)) - - def opt_spew(self): """ Print an insanely verbose log of everything that happens. Useful @@ -202,61 +70,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): """ sys.settrace(spewer) - - def opt_help_reporters(self): - synopsis = ("OONI's output can be customized using plugins called " - "Reporters. You can\nselect any of the following " - "reporters using --reporter=<foo>\n") - print synopsis - for p in plugin.getPlugins(itrial.IReporter): - print ' ', p.longOpt, '\t', p.description - print - sys.exit(0) - - - def opt_disablegc(self): - """ - Disable the garbage collector - """ - gc.disable() - - - def opt_tbformat(self, opt): - """ - Specify the format to display tracebacks with. Valid formats are - 'plain', 'emacs', and 'cgitb' which uses the nicely verbose stdlib - cgitb.text function - """ - try: - self['tbformat'] = TBFORMAT_MAP[opt] - except KeyError: - raise usage.UsageError( - "tbformat must be 'plain', 'emacs', or 'cgitb'.") - - - def opt_recursionlimit(self, arg): - """ - see sys.setrecursionlimit() - """ - try: - sys.setrecursionlimit(int(arg)) - except (TypeError, ValueError): - raise usage.UsageError( - "argument to recursionlimit must be an integer") - - - def opt_without_module(self, option): - """ - Fake the lack of the specified modules, separated with commas. - """ - for module in option.split(","): - if module in sys.modules: - warnings.warn("Module '%s' already imported, " - "disabling anyway." % (module,), - category=RuntimeWarning) - sys.modules[module] = None - - def parseArgs(self, *args): try: self['test'] = args[0] @@ -265,18 +78,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): def postOptions(self): - # Only load reporters now, as opposed to any earlier, to avoid letting - # application-defined plugins muck up reactor selecting by importing - # t.i.reactor and causing the default to be installed. self['reporter'] = reporter.OONIReporter - if 'tbformat' not in self: - self['tbformat'] = 'default' - if self['nopm']: - if not self['debug']: - raise usage.UsageError("you must specify --debug when using " - "--nopm ") - failure.DO_POST_MORTEM = False def run(): if len(sys.argv) == 1: @@ -287,7 +90,6 @@ def run(): except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) - file_name = os.path.abspath('nettests/simpletest.py') classes = runner.findTestClassesFromFile(config['test']) casesList, options = runner.loadTestsAndOptions(classes) for idx, cases in enumerate(casesList):

1 0

[ooni-probe/master] * Still working on the @defer.inlineCallbacks structure; I think I should
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit c4978ff0e750d044b3efd9c211dc3cfa80cc2ec4 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 20 14:18:38 2012 +0000 * Still working on the @defer.inlineCallbacks structure; I think I should use DeferredLists for each of the sets of Deferreds, and then pile all the DeferredLists onto TorState(). --- ooni/plugins/bridget.py | 124 ++++++++++++++++++++++++++++++++++++---------- ooni/utils/circuit.py | 35 ++++++++------ 2 files changed, 117 insertions(+), 42 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 07d7269..d82f2e7 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -360,16 +360,25 @@ class BridgetTest(OONITest): raise PTNotFoundException controller_response = reset_tor.callback - if not controller_response: - defer.returnValue((state.callback, None)) + + #if not controller_response: + # defer.returnValue((state.callback, None)) + #else: + # defer.returnValue((state.callback, controller_response)) + if controller_response == 'OK': + defer.returnValue(state.callback) else: - defer.returnValue((state.callback, controller_response)) + log.msg("TorControlProtocol responded with error:\n%s\n" + % controller_response) + defer.returnValue(state.callback) + except Exception, e: - log.msg("Reconfiguring torrc Bridge line failed with %s" % bridge) + log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s\n" + % (bridge, e)) - def reconfigure_fail(*param): - log.msg("Reconfiguring TorConfig with parameters %s failed" % param) - reactor.stop() + def reconfigure_fail(state, bridge, bad): + log.msg("Reconfiguring TorConfig with parameters %s failed" % state) + bad.append(bridge) @defer.inlineCallbacks def remove_public_relays(state, bridges): @@ -399,6 +408,44 @@ class BridgetTest(OONITest): % both) log.err(e) + def request_circuit_build(state, deferred, path=None): + if path is None: + if relays_remaining() > 0: + first, middle,last = (state.relays.pop() + for i in range(3)) + else: + first = random.choice(state.entry_guards.values()) + middle, last = (random.choice(state.routers.values()) + for i in range(2)) + path = [first, middle, last] + else: + assert type(path) is list, "Circuit path must be a list of routers!" + assert len(path) >= 3, "Circuits must be at least three hops!" + + log.msg("Requesting a circuit: %s" + % '->'.join(map(lambda node: node, path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + def __call__(self, circ): + """ + Return from build_circuit is a Circuit, however, + we want to wait until it is built before we can + issue an attach on it and callback to the Deferred + we issue here. + """ + log.msg("Circuit %s is in progress ..." % circ.id) + self.attacher.waiting_circuits.append((circ.id, self.d)) + + return self.state.build_circuit(path).addCallback( + AppendWaiting(self, deferred)).addErrback( + log.err) + + def attacher_extend_circuit(attacher, deferred, router): + + def setup_fail(proto, bridge_list, relay_list): log.err("Setup Failed: %s" % proto) log.err("We were given bridge list:\n%s\nAnd our relay list:\n%s\n" @@ -508,27 +555,17 @@ class BridgetTest(OONITest): log.msg("Bridget: initiating test ... ") - #while self.bridges_remaining() > 0: - while args['bridge']: + while self.bridges_remaining() > 0: + + self.current_bridge = self.bridges.pop() + #self.current_bridge = args['bridge'] - #self.current_bridge = self.bridges.pop() - self.current_bridge = args['bridge'] try: self.bridges.remove(self.current_bridge) except ValueError, ve: log.err(ve) - if self.config.config.has_key('Bridge'): - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - reconf = defer.Deferred() - reconf.addCallback(reconfigure_bridge, state, - self.current_bridge, self.use_pt, - self.pt_type) - reconf.addErrback(reconfigure_fail) - state.chainDeferred(reconf) - - else: + if not self.config.config.has_key('Bridge'): self.config.Bridge = self.current_bridge (torrc, self.data_directory, to_delete) = write_torrc( self.config, self.data_directory) @@ -538,14 +575,48 @@ class BridgetTest(OONITest): % self.config.create_torrc()) report = {'tor_config': self.config.create_torrc()} - state = start_tor(reactor, updates, torrc, to_delete, - self.control_port, self.tor_binary, - self.data_directory) + state = start_tor(reactor, updates, torrc, to_delete, + self.control_port, + self.tor_binary, + self.data_directory, + bridge_list=self.bridges) state.addCallback(setup_done) state.addErrback(setup_fail) - state.addBoth(remove_relays, self.bridges) + state.addCallback(remove_public_relays, self.bridges) + + else: + log.msg("We now have %d untested bridges..." + % self.bridges_remaining()) + reconf = defer.Deferred() + reconf.addCallback(reconfigure_bridge, state, + self.current_bridge, self.use_pt, + self.pt_type) + reconf.addErrback(reconfigure_fail, state, + self.current_bridge, self.bridges_down) + state.chainDeferred(reconf) + #all = [] + #reconf = reconfigure_bridge(state, self.current_bridge, + # self.use_pt, self.pt_type) + #reconf.addCallback(reconfigure_done) + #reconf.addErrback(reconfigure_fail) + #state.DeferredList(all) + + while self.relays_remaining() >= 3: + #path = list(self.relays.pop() for i in range(3)) + #log.msg("Trying path %s" % '->'.join(map(lambda node: node, path))) + self.current_relay = self.relays.pop() + for circ in state.circuits.values(): + for node in circ.path: + if node == self.current_relay: + self.relays_up.append(self.current_relay) + try: + ext = attacher_extend_circuit(state.attacher, circ, + self.current_relay) + return state + ## still need to attach attacher to state + ## then build circuits ## XXX see txtorcon.TorControlProtocol.add_event_listener we ## may not need full CustomCircuit class @@ -569,7 +640,6 @@ class BridgetTest(OONITest): # self.bridges_up.append(self.current_bridge) reactor.run() - ## now build circuits ## So that getPlugins() can register the Test: diff --git a/ooni/utils/circuit.py b/ooni/utils/circuit.py index 6ee0720..d725879 100644 --- a/ooni/utils/circuit.py +++ b/ooni/utils/circuit.py @@ -9,7 +9,7 @@ # circuits without actually attaching streams. # # :author: Meejah, Isis Lovecruft -# :license: see included license file +# :license: see included LICENSE file # :version: 0.1.0-alpha # @@ -24,9 +24,10 @@ from zope.interface import implements class CustomCircuit(CircuitListenerMixin): implements(IStreamAttacher) - def __init__(self, state): + def __init__(self, state, relays=None): self.state = state self.waiting_circuits = [] + self.relays = relays def waiting_on(self, circuit): for (circid, d) in self.waiting_circuits: @@ -54,12 +55,12 @@ class CustomCircuit(CircuitListenerMixin): def circuit_failed(self, circuit, reason): if self.waiting_on(circuit): - log.msg("A circuit we requested %s failed for reason %s" + log.msg("Circuit %s failed for reason %s" % (circuit.id, reason)) circid, d = None, None - for x in self.waiting_circuits: - if x[0] == circuit.id: - circid, d = x + for c in self.waiting_circuits: + if c[0] == circuit.id: + circid, d = c if d is None: raise Exception("Expected to find circuit.") @@ -72,15 +73,19 @@ class CustomCircuit(CircuitListenerMixin): #router.update() ## XXX can i use without args? no. TorInfo.dump(self) - def request_circuit_build(self, deferred): - if self.state.relays_remaining() > 0: - first, middle,last = (self.state.relays.pop() - for i in range(3)) + def request_circuit_build(self, deferred, path=None): + if path is None: + if self.state.relays_remaining() > 0: + first, middle,last = (self.state.relays.pop() + for i in range(3)) + else: + first = random.choice(self.state.entry_guards.values()) + middle, last = (random.choice(self.state.routers.values()) + for i in range(2)) + path = [first, middle, last] else: - first = random.choice(self.state.entry_guards.values()) - middle, last = (random.choice(self.state.routers.values()) - for i in range(2)) - path = [first, middle, last] + assert type(path) is list, "Circuit path must be a list of routers!" + assert len(path) >= 3, "Circuits must be at least three hops!" log.msg("Requesting a circuit: %s" % '->'.join(map(lambda node: node, path))) @@ -100,5 +105,5 @@ class CustomCircuit(CircuitListenerMixin): self.attacher.waiting_circuits.append((circ.id, self.d)) return self.state.build_circuit(path).addCallback( - AppendWaiting(self, deferred_to_callback)).addErrback( + AppendWaiting(self, deferred)).addErrback( log.err)

1 0

[ooni-probe/master] Implement working trial inspired OONIprobe refactoring
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit cbfcbdd0344d8e9a80d565ba01b64c7a40c97352 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 20 19:26:46 2012 +0000 Implement working trial inspired OONIprobe refactoring --- ooni/nettest.py | 41 +++---- ooni/oonicli.py | 66 ++++------- ooni/plugoo/tests.py | 1 + ooni/runner.py | 330 ++++++++++++------------------------------------- 4 files changed, 122 insertions(+), 316 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 4ab1e0f..0302d25 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -1,5 +1,4 @@ import itertools - from twisted.python import log from twisted.trial import unittest, itrial @@ -42,35 +41,29 @@ class TestSuiteFactory(object): self._idx += 1 return new_test_suite -class TestSuite(pyunit.TestSuite): - def __init__(self, tests=()): - self._tests = [] - self.input = None - self._idx = 0 - self.addTests(tests) - - def __repr__(self): - return "<%s input=%s tests=%s>" % (self.__class__, - self.input, self._tests) - - def run(self, result): - """ - Call C{run} on every member of the suite. - """ - # we implement this because Python 2.3 unittest defines this code - # in __call__, whereas 2.4 defines the code in run. - for i, test in enumerate(self._tests): +class InputTestSuite(pyunit.TestSuite): + def run(self, result, idx=0): + self._idx = idx + while self._tests: if result.shouldStop: break - test.input = self.input - test._idx = self._idx + i - test(result) - + test = self._tests.pop(0) + try: + test.input = self.input + test._idx = self._idx + print "IDX: %s" % self._idx + test(result) + except: + test(result) + self._idx += 1 return result class TestCase(unittest.TestCase): name = "DefaultTestName" - inputs = [None] + inputs = [] + + def getOptions(self): + return {'inputs': self.inputs} def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index be73d30..f0287b4 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -15,6 +15,15 @@ import sys, os, random, gc, time, warnings +import unittest +import inspect + +from ooni.input import InputUnitFactory +from ooni.reporter import ReporterFactory +from ooni.nettest import InputTestSuite +from ooni.plugoo import tests +from ooni import nettest, runner, reporter + from twisted.internet import defer from twisted.application import app from twisted.python import usage, reflect, failure, log @@ -24,7 +33,6 @@ from twisted.python.util import spewer from twisted.python.compat import set from twisted.trial import itrial from twisted.trial import runner as irunner -from ooni import runner, reporter def _parseLocalVariables(line): @@ -132,11 +140,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): repeat=True)], ) - fallbackReporter = reporter.OONIReporter tracer = None def __init__(self): - self['tests'] = set() + self['test'] = None usage.Options.__init__(self) @@ -178,7 +185,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): if not os.path.isfile(filename): sys.stderr.write("File %r doesn't exist\n" % (filename,)) return + filename = os.path.abspath(filename) + self['test'] = filename + if isTestFile(filename): self['tests'].add(filename) else: @@ -248,7 +258,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): - self['tests'].update(args) + try: + self['test'] = args[0] + except: + raise usage.UsageError("No test filename specified!") def postOptions(self): @@ -265,39 +278,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): "--nopm ") failure.DO_POST_MORTEM = False - -def _initialDebugSetup(config): - # do this part of debug setup first for easy debugging of import failures - if config['debug']: - failure.startDebugMode() - if config['debug'] or config['debug-stacktraces']: - defer.setDebugging(True) - - -def _getSuitesAndInputs(config): - #loader = irunner.TestLoader() - loader = runner.NetTestLoader() - recurse = not config['no-recurse'] - print "loadByNames %s" % config['tests'] - inputs, suites = loader.loadByNamesWithInput(config['tests'], recurse) - return inputs, suites - -def _makeRunner(config): - mode = None - if config['debug']: - mode = runner.OONIRunner.DEBUG - print "using %s" % config['reporter'] - return runner.OONIRunner(config['reporter'], - reportfile=config["reportfile"], - mode=mode, - logfile=config['logfile'], - tracebackFormat=config['tbformat'], - realTimeErrors=config['rterrors'], - uncleanWarnings=config['unclean-warnings'], - workingDirectory=config['temp-directory'], - forceGarbageCollection=config['force-gc']) - - def run(): if len(sys.argv) == 1: sys.argv.append("--help") @@ -307,10 +287,10 @@ def run(): except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) - _initialDebugSetup(config) - trialRunner = _makeRunner(config) - inputs, testSuites = _getSuitesAndInputs(config) - log.startLogging(sys.stdout) - for i, suite in enumerate(testSuites): - test_result = trialRunner.run(suite, inputs[i]) + file_name = os.path.abspath('nettests/simpletest.py') + classes = runner.findTestClassesFromFile(config['test']) + casesList, options = runner.loadTestsAndOptions(classes) + for idx, cases in enumerate(casesList): + orunner = runner.ORunner(cases, options[idx]) + orunner.run() diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 2b1e87c..e4ee187 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -133,6 +133,7 @@ class OONITest(object): @param args: the asset(s) lines that we are working on. """ self.start_time = date.now() + print "FOWID" log.msg("Starting test %s" % self.__class__) return self._do_experiment(args) diff --git a/ooni/runner.py b/ooni/runner.py index 604942d..ddde83b 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -13,9 +13,16 @@ from twisted.trial.runner import filenameToModule, _importFromFile from ooni.reporter import ReporterFactory from ooni.input import InputUnitFactory +from ooni.nettest import InputTestSuite from ooni import nettest from ooni.plugoo import tests as oonitests +def isTestCase(thing): + try: + return issubclass(thing, unittest.TestCase) + except TypeError: + return False + def isLegacyTest(obj): """ Returns True if the test in question is written using the OONITest legacy @@ -23,7 +30,10 @@ def isLegacyTest(obj): We do this for backward compatibility of the OONIProbe API. """ try: - return issubclass(obj, oonitests.OONITest) + if issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest: + return True + else: + return False except TypeError: return False @@ -36,267 +46,89 @@ def adaptLegacyTest(obj, inputs=[None]): older test cases compatible with the new OONI. """ class LegacyOONITest(nettest.TestCase): - inputs = [1] + inputs = [None] original_test = obj + @defer.inlineCallbacks def test_start_legacy_test(self): print "bla bla bla" - my_test = self.original_test() - print my_test - print "foobat" - my_test.startTest(self.input) - print "HHAHAHA" + print self.original_test + my_test = self.original_test(None, None, None) + yield my_test.startTest(self.input) return LegacyOONITest -class LoggedSuite(nettest.TestSuite): - """ - Any errors logged in this suite will be reported to the L{TestResult} - object. - """ - - def run(self, result): - """ - Run the suite, storing all errors in C{result}. If an error is logged - while no tests are running, then it will be added as an error to - C{result}. - - @param result: A L{TestResult} object. - """ - observer = unittest._logObserver - observer._add() - super(LoggedSuite, self).run(result) - observer._remove() - for error in observer.getErrors(): - result.addError(TestHolder(NOT_IN_TEST), error) - observer.flushErrors() +def findTestClassesFromFile(filename): + classes = [] + print "FILENAME %s" % filename + module = filenameToModule(filename) + for name, val in inspect.getmembers(module): + if isTestCase(val): + classes.append(val) + elif isLegacyTest(val): + classes.append(adaptLegacyTest(val)) + return classes -class OONISuite(nettest.TestSuite): - """ - Suite to wrap around every single test in a C{trial} run. Used internally - by OONI to set up things necessary for OONI tests to work, regardless of - what context they are run in. - """ +def makeTestCases(klass, tests, methodPrefix): + cases = [] + for test in tests: + cases.append(klass(methodPrefix+test)) + return cases - def __init__(self, tests=()): - suite = LoggedSuite(tests) - super(OONISuite, self).__init__([suite]) - - def _bail(self): - from twisted.internet import reactor - d = defer.Deferred() - reactor.addSystemEventTrigger('after', 'shutdown', - lambda: d.callback(None)) - reactor.fireSystemEvent('shutdown') # radix's suggestion - # As long as TestCase does crap stuff with the reactor we need to - # manually shutdown the reactor here, and that requires util.wait - # :( - # so that the shutdown event completes - nettest.TestCase('mktemp')._wait(d) - - def run(self, result): - try: - nettest.TestSuite.run(self, result) - finally: - self._bail() - - -class NetTestLoader(TestLoader): - """ - Reponsible for finding the modules that can work as tests and running them. - If we detect that a certain test is written using the legacy OONI API we - will wrap it around a next gen class to make it work here too. - - XXX This class needs to be cleaned up a *lot* of all the things we actually - don't need. - """ +def loadTestsAndOptions(classes): methodPrefix = 'test' - modulePrefix = 'test_' - - def __init__(self): - self.suiteFactory = nettest.TestSuite - self._importErrors = [] - - def findTestClasses(self, module): - classes = [] - for name, val in inspect.getmembers(module): - if isTestCase(val): - classes.append(val) - # This is here to allow backward compatibility with legacy OONI - # tests. - elif isLegacyTest(val): - print "adapting! %s" % val - val = adaptLegacyTest(val) - classes.append(val) - return classes - - def loadClass(self, klass): - """ - Given a class which contains test cases, return a sorted list of - C{TestCase} instances. - """ - if not (isinstance(klass, type) or isinstance(klass, types.ClassType)): - raise TypeError("%r is not a class" % (klass,)) - if not isTestCase(klass): - raise ValueError("%r is not a test case" % (klass,)) - names = self.getTestCaseNames(klass) - tests = [] - for name in names: - tests.append(self._makeCase(klass, self.methodPrefix+name)) - - suite = self.suiteFactory(tests) - print "**+*" - print tests - print "**+*" - - return suite - loadTestsFromTestCase = loadClass - - def findAllInputs(self, thing): - testClasses = self.findTestClasses(thing) - # XXX will there ever be more than 1 test class with inputs? - for klass in testClasses: - try: - inputs = klass.inputs - except: - pass - return inputs - - def loadByNamesWithInput(self, names, recurse=False): - """ - Construct a OONITestSuite containing all the tests found in 'names', where - names is a list of fully qualified python names and/or filenames. The - suite returned will have no duplicate tests, even if the same object - is named twice. - - This test suite will have set the attribute inputs to the inputs found - inside of the tests. - """ - inputs = [] - things = [] - errors = [] - for name in names: - try: - thing = self.findByName(name) - things.append(thing) - except: - errors.append(ErrorHolder(name, failure.Failure())) - suites = [] - for thing in self._uniqueTests(things): - inputs.append(self.findAllInputs(thing)) - suite = self.loadAnything(thing, recurse) - suites.append(suite) - - suites.extend(errors) - return inputs, suites - -class OONIRunner(object): - """ - A specialised runner that is used by the ooniprobe frontend to run tests. - Heavily inspired by the trial TrialRunner class. - """ - - DEBUG = 'debug' - DRY_RUN = 'dry-run' - - def _getDebugger(self): - dbg = pdb.Pdb() + suiteFactory = InputTestSuite + options = [] + testCases = [] + for klass in classes: try: - import readline - except ImportError: - print "readline module not available" - sys.exc_clear() - for path in ('.pdbrc', 'pdbrc'): - if os.path.exists(path): - try: - rcFile = file(path, 'r') - except IOError: - sys.exc_clear() - else: - dbg.rcLines.extend(rcFile.readlines()) - return dbg - - - def _setUpTestdir(self): - self._tearDownLogFile() - currentDir = os.getcwd() - base = filepath.FilePath(self.workingDirectory) - testdir, self._testDirLock = util._unusedTestDirectory(base) - os.chdir(testdir.path) - return currentDir - - - def _tearDownTestdir(self, oldDir): - os.chdir(oldDir) - self._testDirLock.unlock() - - - _log = log - def _makeResult(self): - reporter = self.reporterFactory(self.stream, self.tbformat, - self.rterrors, self._log) - if self.uncleanWarnings: - reporter = UncleanWarningsReporterWrapper(reporter) - return reporter - - def __init__(self, reporterFactory, - reportfile="report.yaml", - mode=None, - logfile='test.log', - stream=sys.stdout, - profile=False, - tracebackFormat='default', - realTimeErrors=False, - uncleanWarnings=False, - workingDirectory=None, - forceGarbageCollection=False): - self.reporterFactory = reporterFactory - self._reportfile = reportfile - self.logfile = logfile - self.mode = mode - self.stream = stream - self.tbformat = tracebackFormat - self.rterrors = realTimeErrors - self.uncleanWarnings = uncleanWarnings - self._result = None - self.workingDirectory = workingDirectory or '_trial_temp' - self._logFileObserver = None - self._logFileObject = None - self._forceGarbageCollection = forceGarbageCollection - if profile: - self.run = util.profiled(self.run, 'profile.data') - - def _tearDownLogFile(self): - if self._logFileObserver is not None: - log.removeObserver(self._logFileObserver.emit) - self._logFileObserver = None - if self._logFileObject is not None: - self._logFileObject.close() - self._logFileObject = None - - def _setUpLogFile(self): - self._tearDownLogFile() - if self.logfile == '-': - logFile = sys.stdout + k = klass() + options.append(k.getOptions()) + except AttributeError: + options.append([]) + + tests = reflect.prefixedMethodNames(klass, methodPrefix) + if tests: + cases = makeTestCases(klass, tests, methodPrefix) + testCases.append(cases) else: - logFile = file(self.logfile, 'a') - self._logFileObject = logFile - self._logFileObserver = log.FileLogObserver(logFile) - log.startLoggingWithObserver(self._logFileObserver.emit, 0) + options.pop() + + return testCases, options + +class ORunner(object): + def __init__(self, cases, options=None): + self.baseSuite = InputTestSuite + self.cases = cases + self.options = options + self.inputs = options['inputs'] + self.reporterFactory = ReporterFactory(open('foo.log', 'a+'), + testSuite=self.baseSuite(self.cases)) + + def runWithInputUnit(self, inputUnit): + idx = 0 + result = self.reporterFactory.create() + for input in inputUnit: + suite = self.baseSuite(self.cases) + suite.input = input + suite(result, idx) + + # XXX refactor all of this index bullshit to avoid having to pass + # this index around. Probably what I want to do is go and make + # changes to report to support the concept of having multiple runs + # of the same test. + # We currently need to do this addition in order to get the number + # of times the test cases that have run inside of the test suite. + idx += (suite._idx - idx) + + result.done() + + def run(self): + self.reporterFactory.writeHeader() + + for inputUnit in InputUnitFactory(self.inputs): + self.runWithInputUnit(inputUnit) - def run(self, tests, inputs=[None]): - """ - Run the test or suite and return a result object. - """ - reporterFactory = ReporterFactory(open(self._reportfile, 'a+'), - testSuite=tests) - reporterFactory.writeHeader() - for inputUnit in InputUnitFactory(inputs): - testSuiteFactory = nettest.TestSuiteFactory(inputUnit, tests, nettest.TestSuite) - testUnitReport = reporterFactory.create() - for suite in testSuiteFactory: - suite(testUnitReport) - testUnitReport.done()

1 0

[ooni-probe/master] Remove log file
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 7150e97e4e6237777ff1d343ade7953dea0af413 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 21 11:41:51 2012 +0000 Remove log file --- ooni/ooniprobe.log | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/ooni/ooniprobe.log b/ooni/ooniprobe.log deleted file mode 100644 index 700e820..0000000 --- a/ooni/ooniprobe.log +++ /dev/null @@ -1 +0,0 @@ -2012-05-31 02:27:28,010 ooniprobe INFO Started ooni-probe

1 0

[ooni-probe/master] Implement backward compatibility of new ooniprobe with legacy tests
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 7643541a424655c689e252783a50037846368773 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 21 10:40:41 2012 +0000 Implement backward compatibility of new ooniprobe with legacy tests * Reporting system works with old tests * Properly document some of the functions --- ooni/input.py | 17 +++++------- ooni/nettest.py | 44 ++++----------------------------- ooni/oonicli.py | 8 ++++-- ooni/plugins/httpt.py | 9 ++++--- ooni/plugoo/tests.py | 7 ++--- ooni/reporter.py | 30 ++++++++++++++++++++--- ooni/runner.py | 64 +++++++++++++++++++++++++++++++++++++++---------- 7 files changed, 102 insertions(+), 77 deletions(-) diff --git a/ooni/input.py b/ooni/input.py index b931b82..69507b4 100644 --- a/ooni/input.py +++ b/ooni/input.py @@ -9,7 +9,7 @@ class InputUnitFactory(object): """ inputUnitSize = 3 def __init__(self, inputs=[]): - self._inputs = inputs + self._inputs = iter(inputs) self._idx = 0 self._ended = False @@ -21,16 +21,13 @@ class InputUnitFactory(object): raise StopIteration last_element_idx = self._idx + self.inputUnitSize - input_unit_elements = self._inputs[self._idx:last_element_idx] - try: - # XXX hack to fail when we reach the end of the list - antani = self._inputs[last_element_idx] - except: - if len(input_unit_elements) > 0: + input_unit_elements = [] + for i in xrange(last_element_idx): + try: + input_unit_elements.append(self._inputs.next()) + except: self._ended = True - return InputUnit(input_unit_elements) - else: - raise StopIteration + break self._idx += self.inputUnitSize diff --git a/ooni/nettest.py b/ooni/nettest.py index 0302d25..f3476cf 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -4,44 +4,11 @@ from twisted.trial import unittest, itrial pyunit = __import__('unittest') -def _iterateTests(testSuiteOrCase): +class InputTestSuite(pyunit.TestSuite): """ - Iterate through all of the test cases in C{testSuiteOrCase}. + This in an extension of a unittest test suite. It adds support for inputs + and the tracking of current index via idx. """ - try: - suite = iter(testSuiteOrCase) - except TypeError: - yield testSuiteOrCase - else: - for test in suite: - for subtest in _iterateTests(test): - yield subtest - - -class TestSuiteFactory(object): - def __init__(self, inputUnit, tests, basesuite): - self._baseSuite = basesuite - self._inputUnit = inputUnit - self._idx = 0 - self.tests = tests - - def __iter__(self): - return self - - def next(self): - try: - next_input = self._inputUnit.next() - print "Now dealing with %s %s" % (next_input, self._idx) - except: - raise StopIteration - new_test_suite = self._baseSuite(self.tests) - new_test_suite.input = next_input - new_test_suite._idx = self._idx - - self._idx += 1 - return new_test_suite - -class InputTestSuite(pyunit.TestSuite): def run(self, result, idx=0): self._idx = idx while self._tests: @@ -51,7 +18,6 @@ class InputTestSuite(pyunit.TestSuite): try: test.input = self.input test._idx = self._idx - print "IDX: %s" % self._idx test(result) except: test(result) @@ -59,8 +25,8 @@ class InputTestSuite(pyunit.TestSuite): return result class TestCase(unittest.TestCase): - name = "DefaultTestName" - inputs = [] + name = "DefaultOONITestCase" + inputs = [None] def getOptions(self): return {'inputs': self.inputs} diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 1e7613b..f14bf6e 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -73,10 +73,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): try: self['test'] = args[0] + self['subArgs'] = args[1:] except: raise usage.UsageError("No test filename specified!") - def postOptions(self): self['reporter'] = reporter.OONIReporter @@ -90,9 +90,11 @@ def run(): except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) - classes = runner.findTestClassesFromFile(config['test']) + classes = runner.findTestClassesFromConfig(config) + casesList, options = runner.loadTestsAndOptions(classes) for idx, cases in enumerate(casesList): - orunner = runner.ORunner(cases, options[idx]) + orunner = runner.ORunner(cases, options[idx], config) orunner.run() + diff --git a/ooni/plugins/httpt.py b/ooni/plugins/httpt.py index 46f3b17..358f1ea 100644 --- a/ooni/plugins/httpt.py +++ b/ooni/plugins/httpt.py @@ -65,13 +65,14 @@ class httptTest(http.HTTPTest): def processRedirect(self, location): self.result['redirect'] = None - if self.local_options['rules']: + try: + rules_file = self.local_options['rules'] import yaml - rules = yaml.load(open(self.local_options['rules'])) + rules = yaml.load(open(rules_file)) log.msg("Testing rules %s" % rules) redirect = self.testRules(rules, location) self.result['redirect'] = redirect - else: + except TypeError: log.msg("No rules file. Got a redirect, but nothing to do.") @@ -90,4 +91,4 @@ class httptTest(http.HTTPTest): # We need to instantiate it otherwise getPlugins does not detect it # XXX Find a way to load plugins without instantiating them. -httpt = httptTest(None, None, None) +#httpt = httptTest(None, None, None) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index e4ee187..37e0664 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -25,19 +25,19 @@ class OONITest(object): name = "oonitest" # By default we set this to False, meaning that we don't block blocking = False - reactor = None + reactor = reactor tool = False ended = False def __init__(self, local_options, global_options, report, ooninet=None, - reactor=None): + my_reactor=reactor): # These are the options that are read through the tests suboptions self.local_options = local_options # These are the options global to all of OONI self.global_options = global_options self.report = report #self.ooninet = ooninet - self.reactor = reactor + self.reactor = my_reactor self.result = {} self.initialize() @@ -133,7 +133,6 @@ class OONITest(object): @param args: the asset(s) lines that we are working on. """ self.start_time = date.now() - print "FOWID" log.msg("Starting test %s" % self.__class__) return self._do_experiment(args) diff --git a/ooni/reporter.py b/ooni/reporter.py index 14297cd..4b3ed6f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -11,6 +11,10 @@ from twisted.trial import unittest, reporter, runner pyunit = __import__('unittest') class OReporter(pyunit.TestResult): + """ + This is an extension of the unittest TestResult. It adds support for + reporting to yaml format. + """ def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, publisher=None, testSuite=None): super(OReporter, self).__init__() @@ -45,6 +49,10 @@ class OReporter(pyunit.TestResult): class ReporterFactory(OReporter): + """ + This is a reporter factory. It emits new instances of Reports. It is also + responsible for writing the OONI Report headers. + """ def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, publisher=None, testSuite=None): super(ReporterFactory, self).__init__(stream=stream, @@ -77,6 +85,14 @@ class ReporterFactory(OReporter): class OONIReporter(OReporter): + """ + This is a special reporter that has knowledge about the fact that there can + exist more test runs of the same kind per run. + These multiple test runs are kept track of through idx. + + An instance of such reporter should be created per InputUnit. Every input + unit will invoke size_of_input_unit * test_cases times startTest(). + """ def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, publisher=None): super(OONIReporter, self).__init__(stream=stream, @@ -119,10 +135,16 @@ class OONIReporter(OReporter): idx = self.getTestIndex(test) self._tests[idx]['lastTime'] = self._getTime() - self._tests[idx]['testStarted'] - # XXX I put a dict() here so that the object is re-instantiated and I - # actually end up with the report I want. This could either be a - # python bug or a yaml bug. - self._tests[idx]['report'] = dict(test.report) + # This is here for allowing reporting of legacy tests. + # XXX In the future this should be removed. + try: + self._tests[idx]['report'] = list(test.legacy_report) + except: + # XXX I put a dict() here so that the object is re-instantiated and I + # actually end up with the report I want. This could either be a + # python bug or a yaml bug. + self._tests[idx]['report'] = dict(test.report) + def done(self): """ diff --git a/ooni/runner.py b/ooni/runner.py index ddde83b..0bcafb0 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -15,6 +15,7 @@ from ooni.reporter import ReporterFactory from ooni.input import InputUnitFactory from ooni.nettest import InputTestSuite from ooni import nettest +from ooni.utils import log from ooni.plugoo import tests as oonitests def isTestCase(thing): @@ -37,7 +38,7 @@ def isLegacyTest(obj): except TypeError: return False -def adaptLegacyTest(obj, inputs=[None]): +def adaptLegacyTest(obj, config): """ We take a legacy OONITest class and convert it into a nettest.TestCase. This allows backward compatibility of old OONI tests. @@ -45,30 +46,62 @@ def adaptLegacyTest(obj, inputs=[None]): XXX perhaps we could implement another extra layer that makes the even older test cases compatible with the new OONI. """ + class legacy_reporter(object): + def __init__(self, report_target): + self.report_target = report_target + + def __call__(self, what): + self.report_target.append(what) + class LegacyOONITest(nettest.TestCase): - inputs = [None] - original_test = obj + try: + name = obj.shortName + except: + name = "LegacyOONITest" + + originalTest = obj + + subOptions = obj.options() + subOptions.parseOptions(config['subArgs']) + + test_class = obj(None, None, None, None) + test_class.local_options = subOptions + assets = test_class.load_assets() + + # XXX here we are only taking assets that are set to one item only. + for key, inputs in assets.items(): + pass + + inputs = inputs + local_options = subOptions @defer.inlineCallbacks def test_start_legacy_test(self): - print "bla bla bla" - print self.original_test - my_test = self.original_test(None, None, None) - yield my_test.startTest(self.input) + + self.legacy_report = [] + + my_test = self.originalTest(None, None, None) + my_test.report = legacy_reporter(self.legacy_report) + args = {} + args[self.key] = self.input + result = yield my_test.startTest(args) + print "Finished!" + print result return LegacyOONITest -def findTestClassesFromFile(filename): +def findTestClassesFromConfig(config): + filename = config['test'] + classes = [] - print "FILENAME %s" % filename module = filenameToModule(filename) for name, val in inspect.getmembers(module): if isTestCase(val): classes.append(val) elif isLegacyTest(val): - classes.append(adaptLegacyTest(val)) + classes.append(adaptLegacyTest(val, config)) return classes def makeTestCases(klass, tests, methodPrefix): @@ -99,13 +132,17 @@ def loadTestsAndOptions(classes): return testCases, options class ORunner(object): - def __init__(self, cases, options=None): + def __init__(self, cases, options=None, config=None): self.baseSuite = InputTestSuite self.cases = cases self.options = options self.inputs = options['inputs'] - self.reporterFactory = ReporterFactory(open('foo.log', 'a+'), - testSuite=self.baseSuite(self.cases)) + try: + reportFile = open(config['reportfile'], 'a+') + except: + reportFile = open('report.yaml', 'a+') + self.reporterFactory = ReporterFactory(reportFile, + testSuite=self.baseSuite(self.cases)) def runWithInputUnit(self, inputUnit): idx = 0 @@ -126,6 +163,7 @@ class ORunner(object): result.done() def run(self): + log.start() self.reporterFactory.writeHeader() for inputUnit in InputUnitFactory(self.inputs):

1 0

[ooni-probe/master] Make the ascii art thing only be a special command line option
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 05b8d5b809db6ee34ee76c9e0c1a67d163b586ef Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 21 12:14:44 2012 +0000 Make the ascii art thing only be a special command line option --- ooni/oonicli.py | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index f14bf6e..e5c990f 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -42,7 +42,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): "network tests. These are loaded from modules, packages and" "files listed on the command line") - optFlags = [["help", "h"]] + optFlags = [["help", "h"], + ] optParameters = [ ["reportfile", "o", "report.yaml", "report file name"], @@ -63,6 +64,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): self['test'] = None usage.Options.__init__(self) + def opt_asciilulz(self): + from ooni.utils import logo + print logo.getlogo() + def opt_spew(self): """ Print an insanely verbose log of everything that happens. Useful @@ -73,6 +78,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): try: self['test'] = args[0] + self['subArgs'] = args[1:] except: raise usage.UsageError("No test filename specified!") @@ -97,4 +103,3 @@ def run(): orunner = runner.ORunner(cases, options[idx], config) orunner.run() -

1 0