October 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] * Modified the ValueChecker class to have static methods.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit d448f256ee5993237d9182164ba0efae808c18b6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Sep 28 04:58:00 2012 +0000 * Modified the ValueChecker class to have static methods. * Switched to dictionaries for self.relays and self.bridges so that all methods can access various values without having to constantly change parameters. * Fixed a couple assertions. --- ooni/plugins/bridget.py | 352 ++++++++++++++++++++++------------------------ ooni/utils/config.py | 9 +- ooni/utils/onion.py | 55 +++++--- 3 files changed, 210 insertions(+), 206 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index c749b90..208b4c3 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -12,25 +12,26 @@ # :licence: see included LICENSE # :version: 0.1.0-alpha -from __future__ import with_statement -from functools import partial -from random import randint -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import defer, error, reactor -from zope.interface import implements - -from ooni.utils import log, date -from ooni.utils.config import ValueChecker -from ooni.utils.onion import parse_data_dir -from ooni.utils.onion import TxtorconImportError -from ooni.utils.onion import PTNoBridgesException, PTNotFoundException -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset, MissingAssetException +from __future__ import with_statement +from functools import partial +from random import randint import os import sys +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor +from zope.interface import implements + +from ooni.utils import log, date +from ooni.utils.config import ValueChecker +from ooni.utils.onion import parse_data_dir +from ooni.utils.onion import TxtorconImportError +from ooni.utils.onion import PTNoBridgesException, PTNotFoundException +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset, MissingAssetException + class RandomPortException(Exception): """Raised when using a random port conflicts with configured ports.""" @@ -40,12 +41,9 @@ class RandomPortException(Exception): class BridgetArgs(usage.Options): """Commandline options.""" - global vc - vc = ValueChecker - allowed = "Port to use for Tor's %s, must be between 1024 and 65535." - sock_check = vc(allowed % "SocksPort").port_check - ctrl_check = vc(allowed % "ControlPort").port_check + sock_check = ValueChecker(allowed % "SocksPort").port_check + ctrl_check = ValueChecker(allowed % "ControlPort").port_check optParameters = [ ['bridges', 'b', None, @@ -69,7 +67,7 @@ class BridgetArgs(usage.Options): raise MissingAssetException( "Bridget can't run without bridges or relays to test!") if self['transport']: - vc().uid_check( + ValueChecker.uid_check( "Can't run bridget as root with pluggable transports!") if not self['bridges']: raise PTNoBridgesException @@ -77,9 +75,9 @@ class BridgetArgs(usage.Options): if self['random']: raise RandomPortException if self['datadir']: - vc().dir_check(self['datadir']) + ValueChecker.dir_check(self['datadir']) if self['torpath']: - vc().file_check(self['torpath']) + ValueChecker.file_check(self['torpath']) class BridgetAsset(Asset): """Class for parsing bridget Assets ignoring commented out lines.""" @@ -126,29 +124,11 @@ class BridgetTest(OONITest): running, so we need to deal with most of the TorConfig() only once, before the experiment runs. """ - self.d = defer.Deferred() - self.socks_port = 9049 self.control_port = 9052 self.circuit_timeout = 90 self.tor_binary = '/usr/sbin/tor' self.data_directory = None - self.use_pt = False - self.pt_type = None - - ## XXX we should do report['bridges_up'].append(self.current_bridge) - self.bridges, self.bridges_up, self.bridges_down = ([] for i in range(3)) - self.bridges_remaining = lambda: len(self.bridges) - self.bridges_down_count = lambda: len(self.bridges_down) - self.current_bridge = None - - self.relays, self.relays_up, self.relays_down = ([] for i in range(3)) - self.relays_remaining = lambda: len(self.relays) - self.relays_down_count = lambda: len(self.relays_down) - self.current_relay = None - - ## Make sure we don't use self.load_assets() for now: - self.assets = {} def __make_asset_list__(opt, lst): log.msg("Loading information from %s ..." % opt) @@ -159,52 +139,71 @@ class BridgetTest(OONITest): else: lst.append(line.replace('\n','')) + def __count_remaining__(which): + total, reach, unreach = map(lambda x: which[x], + ['all', 'reachable', 'unreachable']) + count = len(total) - reach() - unreach() + return count + + ## XXX should we do report['bridges_up'].append(self.bridges['current']) + self.bridges = {} + self.bridges['all'], self.bridges['up'], self.bridges['down'] = \ + ([] for i in range(3)) + self.bridges['reachable'] = lambda: len(self.bridges['up']) + self.bridges['unreachable'] = lambda: len(self.bridges['down']) + self.bridges['remaining'] = lambda: __count_remaining__(self.bridges) + self.bridges['current'] = None + self.bridges['pt_type'] = None + self.bridges['use_pt'] = False + + self.relays = {} + self.relays['all'], self.relays['up'], self.relays['down'] = \ + ([] for i in range(3)) + self.relays['reachable'] = lambda: len(self.relays['up']) + self.relays['unreachable'] = lambda: len(self.relays['down']) + self.relays['remaining'] = lambda: __count_remaining__(self.relays) + self.relays['current'] = None + if self.local_options: try: from ooni.lib.txtorcon import TorConfig except ImportError: raise TxtorconImportError - - options = self.local_options - config = self.config = TorConfig() + else: + self.config = TorConfig() + finally: + options = self.local_options if options['bridges']: self.config.UseBridges = 1 - __make_asset_list__(options['bridges'], self.bridges) - + __make_asset_list__(options['bridges'], self.bridges['all']) if options['relays']: - ## first hop must be in TorState().guards to build circuits + ## first hop must be in TorState().guards self.config.EntryNodes = ','.join(relay_list) - __make_asset_list__(options['relays'], self.relays) - + __make_asset_list__(options['relays'], self.relays['all']) if options['socks']: self.socks_port = options['socks'] if options['control']: self.control_port = options['control'] - if options['random']: log.msg("Using randomized ControlPort and SocksPort ...") self.socks_port = randint(1024, 2**16) self.control_port = randint(1024, 2**16) - if options['torpath']: self.tor_binary = options['torpath'] - if options['datadir']: self.data_directory = parse_data_dir(options['datadir']) - else: - self.data_directory = None - if options['transport']: - self.use_pt = True - log.msg("Using ClientTransportPlugin %s" - % options['transport']) - [self.pt_type, pt_exec] = options['transport'].split(' ', 1) - ## ClientTransportPlugin transport exec pathtobinary [options] ## XXX we need a better way to deal with all PTs - if self.pt_type == "obfs2": - self.config.ClientTransportPlugin = self.pt_type+" "+pt_exec + log.msg("Using ClientTransportPlugin %s" % options['transport']) + self.bridges['use_pt'] = True + [self.bridges['pt_type'], pt_exec] = \ + options['transport'].split(' ', 1) + + if self.bridges['pt_type'] == "obfs2": + self.config.ClientTransportPlugin = \ + self.bridges['pt_type'] + " " + pt_exec else: raise PTNotFoundException @@ -212,11 +211,21 @@ class BridgetTest(OONITest): self.config.ControlPort = self.control_port self.config.CookieAuthentication = 1 - def load_assets(self): + def __load_assets__(self): """ Load bridges and/or relays from files given in user options. Bridges should be given in the form IP:ORport. We don't want to load these as assets, because it's inefficient to start a Tor process for each one. + + We cannot use the Asset model, because that model calls + self.experiment() with the current Assets, which would be one relay + and one bridge, then it gives the defer.Deferred returned from + self.experiment() to self.control(), which means that, for each + (bridge, relay) pair, experiment gets called again, which instantiates + an additional Tor process that attempts to bind to the same + ports. Thus, additionally instantiated Tor processes return with + RuntimeErrors, which break the final defer.chainDeferred.callback(), + sending it into the errback chain. """ assets = {} if self.local_options: @@ -229,50 +238,40 @@ class BridgetTest(OONITest): return assets def experiment(self, args): - """ - We cannot use the Asset model, because that model calls - self.experiment() with the current Assets, which would be one relay - and one bridge, then it gives the defer.Deferred returned from - self.experiment() to self.control(), which means that, for each - (bridge, relay) pair, experiment gets called again, which instantiates - an additional Tor process that attempts to bind to the same - ports. Thus, additionally instantiated Tor processes return with - RuntimeErrors, which break the final defer.chainDeferred.callback(), - sending it into the errback chain. - - if bridges: - 1. configure first bridge line - 2a. configure data_dir, if it doesn't exist - 2b. write torrc to a tempfile in data_dir - 3. start tor } if any of these - 4. remove bridges which are public relays } fail, add current - 5. SIGHUP for each bridge } bridge to unreach- - } able bridges. - if relays: - 1a. configure the data_dir, if it doesn't exist - 1b. write torrc to a tempfile in data_dir - 2. start tor - 3. remove any of our relays which are already part of current - circuits - 4a. attach CustomCircuit() to self.state - 4b. RELAY_EXTEND for each relay } if this fails, add - } current relay to list - } of unreachable relays - 5. - if bridges and relays: - 1. configure first bridge line - 2a. configure data_dir if it doesn't exist - 2b. write torrc to a tempfile in data_dir - 3. start tor - 4. remove bridges which are public relays - 5. remove any of our relays which are already part of current - circuits - 6a. attach CustomCircuit() to self.state - 6b. for each bridge, build three circuits, with three - relays each - 6c. RELAY_EXTEND for each relay } if this fails, add - } current relay to list - } of unreachable relays + """ + if bridges: + 1. configure first bridge line + 2a. configure data_dir, if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor } if any of these + 4. remove bridges which are public relays } fail, add current + 5. SIGHUP for each bridge } bridge to unreach- + } able bridges. + if relays: + 1a. configure the data_dir, if it doesn't exist + 1b. write torrc to a tempfile in data_dir + 2. start tor + 3. remove any of our relays which are already part of current + circuits + 4a. attach CustomCircuit() to self.state + 4b. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays + 5. + if bridges and relays: + 1. configure first bridge line + 2a. configure data_dir if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor + 4. remove bridges which are public relays + 5. remove any of our relays which are already part of current + circuits + 6a. attach CustomCircuit() to self.state + 6b. for each bridge, build three circuits, with three + relays each + 6c. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays :param args: The :class:`BridgetAsset` line currently being used. Except that it @@ -290,59 +289,52 @@ class BridgetTest(OONITest): log.err(tie) sys.exit() + def reconfigure_done(state, bridges): + log.msg("Reconfiguring with 'Bridge %s' successful" + % bridges['current']) + bridges['up'].append(bridges['current']) + return state + + def reconfigure_fail(state, bridges): + log.msg("Reconfiguring TorConfig with parameters %s failed" + % state) + bridges['down'].append(bridges['current']) + return state + @defer.inlineCallbacks - def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): + def reconfigure_bridge(state, bridges): """ Rewrite the Bridge line in our torrc. If use of pluggable transports was specified, rewrite the line as: Bridge <transport_type> <ip>:<orport> Otherwise, rewrite in the standard form. """ - log.msg("Current Bridge: %s" % bridge) + log.msg("Current Bridge: %s" % bridges['current']) + log.msg("We now have %d bridges remaining to test..." + % bridges['remaining']()) try: - if use_pt is False: - reset_tor = yield state.protocol.set_conf('Bridge', - bridge) - elif use_pt and pt_type is not None: - reset_tor = yield state.protocol.set_conf( - 'Bridge', pt_type +' '+ bridge) + if bridges['use_pt'] is False: + controller_response = yield state.protocol.set_conf( + 'Bridge', bridges['current']) + elif bridges['use_pt'] and bridges['pt_type'] is not None: + controller_reponse = yield state.protocol.set_conf( + 'Bridge', bridges['pt_type'] +' '+ bridges['current']) else: raise PTNotFoundException - controller_response = reset_tor.callback - controller_response.addCallback(reconfigure_done, - bridge, - reachable) - controller_response.addErrback(reconfigure_fail, - bridge, - reachable) - - #if not controller_response: - # defer.returnValue((state.callback, None)) - #else: - # defer.returnValue((state.callback, controller_response)) if controller_response == 'OK': - defer.returnValue((state, controller_response)) + finish = reconfigure_done(state, bridges) else: - log.msg("TorControlProtocol responded with error:\n%s" - % controller_response) - defer.returnValue((state.callback, None)) - - except Exception, e: - log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s" - % (bridge, e)) - defer.returnValue((state.callback, e)) + log.err("SETCONF for %s responded with error:\n %s" + % (bridges['current'], controller_response)) + finish = reconfigure_fail(state, bridges) - def reconfigure_done(state, bridge, reachable): - log.msg("Reconfiguring with 'Bridge %s' successful" % bridge) - reachable.append(bridge) - return state + defer.returnValue(finish) - def reconfigure_fail(state, bridge, unreachable): - log.msg("Reconfiguring TorConfig with parameters %s failed" - % state) - unreachable.append(bridge) - return state + except Exception, e: + log.err("Reconfiguring torrc with Bridge line %s failed:\n%s" + % (bridges['current'], e)) + defer.returnValue(None) def attacher_extend_circuit(attacher, deferred, router): ## XXX todo write me @@ -373,25 +365,21 @@ class BridgetTest(OONITest): log.err("Attaching custom circuit builder failed: %s" % state) - ## Start the experiment - log.msg("Bridget: initiating test ... ") - all_of_the_bridges = self.bridges - all_of_the_relays = self.relays ## Local copy of orginal lists + log.msg("Bridget: initiating test ... ") ## Start the experiment + all_of_the_bridges = self.bridges['all'] + all_of_the_relays = self.relays['all'] ## Local copy of orginal lists - if self.bridges_remaining() >= 1 and not 'Bridge' in self.config.config: - ## XXX we should do self.bridges[0] + self.bridges[1:] - initial_bridge = all_of_the_bridges.pop() - self.config.Bridge = initial_bridge - self.config.save() ## avoid starting several processes + if self.bridges['remaining']() >= 1 and not 'Bridge' in self.config.config: + self.bridges['current'] = self.bridges['all'][0] + self.config.Bridge = self.bridges['current'] + ## avoid starting several + self.config.save() ## processes assert self.config.config.has_key('Bridge'), "NO BRIDGE" - state = start_tor(self.reactor, self.config, - self.control_port, self.tor_binary, - self.data_directory).addCallbacks( - setup_done, - errback=setup_fail) - state.addCallback(remove_public_relays, - self.bridges) + state = start_tor(reactor, self.config, self.control_port, + self.tor_binary, self.data_directory) + state.addCallbacks(setup_done, setup_fail) + state.addCallback(remove_public_relays, self.bridges) #controller = singleton_semaphore(bootstrap) #controller = x().addCallback(singleton_semaphore, tor) @@ -404,17 +392,12 @@ class BridgetTest(OONITest): log.debug("Current callbacks on TorState():\n%s" % state.callbacks) log.debug("TorState():\n%s" % state) - if self.bridges_remaining() > 0: + if self.bridges['remaining']() >= 2: all = [] - for bridge in self.bridges: - #self.current_bridge = bridge + for bridge in self.bridges['all'][1:]: + self.bridges['current'] = bridge new = defer.Deferred() - new.addCallback(reconfigure_bridge, state, bridge, - self.bridges_remaining(), - self.bridges_up, - self.bridges_down, - use_pt=self.use_pt, - pt_type=self.pt_type) + new.addCallback(reconfigure_bridge, state, self.bridges) all.append(new) #state.chainDeferred(defer.DeferredList(all)) @@ -426,38 +409,39 @@ class BridgetTest(OONITest): # % controller.callbacks) state.chainDeferred(check_remaining) - if self.relays_remaining() > 0: - while self.relays_remaining() >= 3: + if self.relays['remaining']() > 0: + while self.relays['remaining']() >= 3: #path = list(self.relays.pop() for i in range(3)) #log.msg("Trying path %s" % '->'.join(map(lambda node: # node, path))) - self.current_relay = self.relays.pop() + self.relays['current'] = self.relays['all'].pop() for circ in state.circuits.values(): for node in circ.path: - if node == self.current_relay: - self.relays_up.append(self.current_relay) + if node == self.relays['current']: + self.relays['up'].append(self.relays['current']) if len(circ.path) < 3: try: ext = attacher_extend_circuit(state.attacher, circ, - self.current_relay) + self.relays['current']) ext.addCallback(attacher_extend_circuit_done, state.attacher, circ, - self.current_relay) + self.relays['current']) except Exception, e: - log.msg("Extend circuit failed: %s" % e) + log.err("Extend circuit failed: %s" % e) else: continue - state.callback(all) + #state.callback(all) self.reactor.run() - #return state + return state def startTest(self, args): self.start_time = date.now() - log.msg("Starting %s" % self.shortName) - self.do_science = self.experiment(args) - self.do_science.addCallback(self.finished).addErrback(log.err) - return self.do_science + self.laboratory = defer.Deferred() + self.laboratory.addCallbacks(self.experiment, errback=log.err) + self.laboratory.addCallbacks(self.finished, errback=log.err) + self.laboratory.callback(args) + return self.laboratory ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) diff --git a/ooni/utils/config.py b/ooni/utils/config.py index 7c28f33..e429ef6 100644 --- a/ooni/utils/config.py +++ b/ooni/utils/config.py @@ -85,7 +85,8 @@ class ValueChecker(object): raise ValueError("Port out of range") log.err() - def uid_check(self, error_message): + @staticmethod + def uid_check(error_message): """ Check that we're not root. If we are, setuid(1000) to normal user if we're running on a posix-based system, and if we're on Windows just @@ -105,13 +106,15 @@ class ValueChecker(object): else: sys.exit(0) - def dir_check(self, d): + @staticmethod + def dir_check(d): """Check that the given directory exists.""" if not os.path.isdir(d): raise ValueError("%s doesn't exist, or has wrong permissions" % d) - def file_check(self, f): + @staticmethod + def file_check(f): """Check that the given file exists.""" if not os.path.isfile(f): raise ValueError("%s does not exist, or has wrong permissions" diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 82f013f..6f6b355 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -68,17 +68,31 @@ def parse_data_dir(data_dir): :return: The absolute path of :param:data_dir. """ - import os + from os import path, getcwd + import sys + + try: + assert isinstance(data_dir, str), \ + "Parameter type(data_dir) must be str" + except AssertionError, ae: + log.err(ae) if data_dir.startswith('~'): - data_dir = os.path.expanduser(data_dir) + data_dir = path.expanduser(data_dir) elif data_dir.startswith('/'): - data_dir = os.path.join(os.getcwd(), data_dir) + data_dir = path.join(getcwd(), data_dir) elif data_dir.startswith('./'): - data_dir = os.path.abspath(data_dir) + data_dir = path.abspath(data_dir) + else: + data_dir = path.join(getcwd(), data_dir) + + try: + assert path.isdir(data_dir), "Could not find %s" % data_dir + except AssertionError, ae: + log.err(ae) + sys.exit(1) else: - data_dir = os.path.join(os.getcwd(), data_dir) - return data_dir + return data_dir def write_torrc(conf, data_dir=None): """ @@ -137,11 +151,10 @@ def remove_node_from_list(node, list): if item.startswith(node): ## due to the :<port>. try: log.msg("Removing %s because it is a public relay" % node) - list.remove(line) + list.remove(item) except ValueError, ve: log.err(ve) -(a)defer.inlineCallbacks def remove_public_relays(state, bridges): """ Remove bridges from our bridge list which are also listed as public @@ -151,25 +164,26 @@ def remove_public_relays(state, bridges): XXX Does state.router.values() have all of the relays in the consensus, or just the ones we know about so far? + + XXX FIXME: There is a problem in that Tor needs a Bridge line to already be + configured in order to bootstrap. However, after bootstrapping, we grab the + microdescriptors of all the relays and check if any of our bridges are + listed as public relays. Because of this, the first bridge does not get + checked for being a relay. """ - IPs = map(lambda addr: addr.split(':',1)[0], bridges) + IPs = map(lambda addr: addr.split(':',1)[0], bridges['all']) both = set(state.routers.values()).intersection(IPs) if len(both) > 0: try: - updated = yield map(lambda node: remove_node_from_list(node), - both) + updated = map(lambda node: remove_node_from_list(node), both) if not updated: - ## XXX do these need to be state.callback? defer.returnValue(state) else: defer.returnValue(state) except Exception, e: - log.msg("Removing public relays from bridge list failed:\n%s" - % both) - log.err(e) - except ValueError, ve: - log.err(ve) + log.err("Removing public relays %s from bridge list failed:\n%s" + % (both, e)) #(a)defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, @@ -259,6 +273,7 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, transport = reactor.spawnProcess(process_protocol, tor_binary, args=(tor_binary,'-f',torrc), + env={'HOME':data_dir}, path=data_dir) transport.closeStdin() except RuntimeError, e: @@ -337,8 +352,10 @@ class CustomCircuit(CircuitListenerMixin): for i in range(2)) path = [first, middle, last] else: - assert type(path) is list, "Circuit path must be a list of relays!" - assert len(path) >= 3, "Circuits must be at least three hops!" + assert isinstance(path, list), \ + "Circuit path must be a list of relays!" + assert len(path) >= 3, \ + "Circuit path must be at least three hops!" log.msg("Requesting a circuit: %s" % '->'.join(map(lambda node: node, path)))

1 0

[ooni-probe/master] * Moved bootstrap and remove_public_relay functions to Tor utilities file.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit efc1206e8a099e699996c3a45742d4f7500a714f Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 27 01:19:49 2012 +0000 * Moved bootstrap and remove_public_relay functions to Tor utilities file. * Still sorting out the chainDeferred(defer.DeferredList(list)) problems. --- ooni/plugins/bridget.py | 136 +++++++++++++++++------------------------------ ooni/utils/onion.py | 107 ++++++++++++++++++++++++++----------- 2 files changed, 123 insertions(+), 120 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 99a7d05..c749b90 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -279,8 +279,9 @@ class BridgetTest(OONITest): in Bridget it doesn't, so it should be ignored and avoided. """ try: - from ooni.utils.process import singleton_semaphore - from ooni.utils.onion import start_tor, setup_done, setup_fail + from ooni.utils import process + from ooni.utils.onion import start_tor, remove_public_relays + from ooni.utils.onion import setup_done, setup_fail from ooni.utils.onion import CustomCircuit from ooni.lib.txtorcon import TorConfig, TorState except ImportError: @@ -289,18 +290,6 @@ class BridgetTest(OONITest): log.err(tie) sys.exit() - ''' - ## XXX qu'est-que fuck? ou est utiliser ce fonction? - def bootstrap(ctrl): - """ - Launch a Tor process with the TorConfig instance returned from - initialize() and write_torrc(). - """ - conf = TorConfig(ctrl) - conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) - log.msg("Tor process connected, bootstrapping ...") - ''' - @defer.inlineCallbacks def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): """ @@ -312,7 +301,7 @@ class BridgetTest(OONITest): log.msg("Current Bridge: %s" % bridge) try: if use_pt is False: - reset_tor = yield state.protocol.set_conf('Bridge', + reset_tor = yield state.protocol.set_conf('Bridge', bridge) elif use_pt and pt_type is not None: reset_tor = yield state.protocol.set_conf( @@ -331,62 +320,29 @@ class BridgetTest(OONITest): #if not controller_response: # defer.returnValue((state.callback, None)) #else: - # defer.returnValue((state.callback, controller_response)) + # defer.returnValue((state.callback, controller_response)) if controller_response == 'OK': defer.returnValue((state, controller_response)) else: log.msg("TorControlProtocol responded with error:\n%s" % controller_response) defer.returnValue((state.callback, None)) - + except Exception, e: log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s" % (bridge, e)) defer.returnValue((state.callback, e)) - def reconfigure_done(response, bridge, reachable): + def reconfigure_done(state, bridge, reachable): log.msg("Reconfiguring with 'Bridge %s' successful" % bridge) reachable.append(bridge) + return state - def reconfigure_fail(response, bridge, unreachable): + def reconfigure_fail(state, bridge, unreachable): log.msg("Reconfiguring TorConfig with parameters %s failed" % state) unreachable.append(bridge) - - @defer.inlineCallbacks - def remove_public_relays(state, bridges): - """ - Remove bridges from our bridge list which are also listed as - public relays. - """ - IPs = map(lambda addr: addr.split(':',1)[0], bridges) - both = set(state.routers.values()).intersection(IPs) - - def __remove_line__(node, bridges=bridges): - for line in bridges: - if line.startswith(node): - try: - log.msg("Removing %s because it is a public relay" - % node) - bridges.remove(line) - except ValueError, ve: - log.debug(ve) - - if len(both) > 0: - try: - updated = yield map(lambda node: - __remove_line__(node), both) - if not updated: - ## XXX do these need to be state.callback? - defer.returnValue(state) - else: - defer.returnValue(state) - except Exception, e: - log.msg("Removing public relays from bridge list failed:\n%s" - % both) - log.err(e) - except ValueError, ve: - log.err(ve) + return state def attacher_extend_circuit(attacher, deferred, router): ## XXX todo write me @@ -417,54 +373,58 @@ class BridgetTest(OONITest): log.err("Attaching custom circuit builder failed: %s" % state) + ## Start the experiment log.msg("Bridget: initiating test ... ") - x = defer.Deferred - - if self.bridges_remaining() > 0 and not 'Bridge' in self.config.config: - self.config.Bridge = self.bridges.pop() - - ## Necessary for avoiding starting several processes: - self.config.save() + all_of_the_bridges = self.bridges + all_of_the_relays = self.relays ## Local copy of orginal lists + + if self.bridges_remaining() >= 1 and not 'Bridge' in self.config.config: + ## XXX we should do self.bridges[0] + self.bridges[1:] + initial_bridge = all_of_the_bridges.pop() + self.config.Bridge = initial_bridge + self.config.save() ## avoid starting several processes assert self.config.config.has_key('Bridge'), "NO BRIDGE" - tor = start_tor(self.reactor, - self.config, - self.control_port, - self.tor_binary, - self.data_directory).addCallback( - setup_done).addErrback( - setup_fail) - self.tor_process_semaphore = True + state = start_tor(self.reactor, self.config, + self.control_port, self.tor_binary, + self.data_directory).addCallbacks( + setup_done, + errback=setup_fail) + state.addCallback(remove_public_relays, + self.bridges) - run_once = x().addCallback(singleton_semaphore, tor) - run_once.addErrback(setup_fail) + #controller = singleton_semaphore(bootstrap) + #controller = x().addCallback(singleton_semaphore, tor) + #controller.addErrback(setup_fail) - filter_bridges = x().addCallback(remove_public_relays, self.bridges) + #filter_bridges = remove_public_relays(self.bridges) - state = defer.gatherResults([run_once, filter_bridges], consumeErrors=True) + #bootstrap = defer.gatherResults([controller, filter_bridges], + # consumeErrors=True) log.debug("Current callbacks on TorState():\n%s" % state.callbacks) + log.debug("TorState():\n%s" % state) if self.bridges_remaining() > 0: all = [] for bridge in self.bridges: - self.current_bridge = bridge - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - reconf = x().addCallback(reconfigure_bridge, state, - self.current_bridge, - self.use_pt, - self.pt_type) - reconf.addCallback(reconfigure_done, self.current_bridge, - self.bridges_up) - reconf.addErrback(reconfigure_fail, self.current_bridge, - self.bridges_down) - all.append(reconf) + #self.current_bridge = bridge + new = defer.Deferred() + new.addCallback(reconfigure_bridge, state, bridge, + self.bridges_remaining(), + self.bridges_up, + self.bridges_down, + use_pt=self.use_pt, + pt_type=self.pt_type) + all.append(new) #state.chainDeferred(defer.DeferredList(all)) #state.chainDeferred(defer.gatherResults(all, consumeErrors=True)) - n_plus_one_bridges = defer.gatherResults(all, consumeErrors=True) - state.chainDeferred(n_plus_one_bridges) - log.debug("Current callbacks on TorState():\n%s" % state.callbacks) + check_remaining = defer.DeferredList(all, consumeErrors=True) + + #controller.chainDeferred(check_remaining) + #log.debug("Current callbacks on TorState():\n%s" + # % controller.callbacks) + state.chainDeferred(check_remaining) if self.relays_remaining() > 0: while self.relays_remaining() >= 3: diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 81e4ea3..82f013f 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -13,6 +13,7 @@ # :copyright: copyright (c) 2012 The Tor Project, Inc. # :version: 0.1.0-alpha # +# XXX TODO add report keys for onion methods import random @@ -31,25 +32,32 @@ def setup_done(proto): def setup_fail(proto): log.err("Setup Failed: %s" % proto) - report.update({'setup_fail': proto}) + #report.update({'setup_fail': proto}) reactor.stop() def state_complete(state): """Called when we've got a TorState.""" log.msg("We've completely booted up a Tor version %s at PID %d" % (state.protocol.version, state.tor_pid)) - log.msg("This Tor has the following %d Circuits:" % len(state.circuits)) for circ in state.circuits.values(): log.msg("%s" % circ) - - #return state + return state def updates(_progress, _tag, _summary): """Log updates on the Tor bootstrapping process.""" log.msg("%d%%: %s" % (_progress, _summary)) +def bootstrap(ctrl): + """ + Bootstrap Tor from an instance of + :class:`ooni.lib.txtorcon.TorControlProtocol`. + """ + conf = TorConfig(ctrl) + conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) + log.msg("Tor process connected, bootstrapping ...") + def parse_data_dir(data_dir): """ Parse a string that a has been given as a DataDirectory and determine @@ -124,7 +132,46 @@ def delete_files_or_dirs(delete_list): except OSError: rmtree(temp, ignore_errors=True) +def remove_node_from_list(node, list): + for item in list: ## bridges don't match completely + if item.startswith(node): ## due to the :<port>. + try: + log.msg("Removing %s because it is a public relay" % node) + list.remove(line) + except ValueError, ve: + log.err(ve) + @defer.inlineCallbacks +def remove_public_relays(state, bridges): + """ + Remove bridges from our bridge list which are also listed as public + relays. This must be called after Tor has fully bootstrapped and we have a + :class:`ooni.lib.txtorcon.TorState` with the + :attr:`ooni.lib.txtorcon.TorState.routers` attribute assigned. + + XXX Does state.router.values() have all of the relays in the consensus, or + just the ones we know about so far? + """ + IPs = map(lambda addr: addr.split(':',1)[0], bridges) + both = set(state.routers.values()).intersection(IPs) + + if len(both) > 0: + try: + updated = yield map(lambda node: remove_node_from_list(node), + both) + if not updated: + ## XXX do these need to be state.callback? + defer.returnValue(state) + else: + defer.returnValue(state) + except Exception, e: + log.msg("Removing public relays from bridge list failed:\n%s" + % both) + log.err(e) + except ValueError, ve: + log.err(ve) + +#(a)defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, report=None, progress=updates, process_cb=setup_done, process_eb=setup_fail): @@ -193,39 +240,35 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, reactor.addSystemEventTrigger('before', 'shutdown', partial(delete_files_or_dirs, to_delete)) + #try: + # transport = yield reactor.spawnProcess(process_protocol, + # tor_binary, + # args=(tor_binary,'-f',torrc), + # env={'HOME': data_dir}, + # path=data_dir) + # if transport: + # transport.closeStdin() + #except RuntimeError as e: + # log.err("Starting Tor failed: %s" % e) + # process_protocol.connected_cb.errback(e) + #except NotImplementedError, e: + # url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" + # log.err("Running bridget on Windows requires pywin32: %s" % url) + # process_protocol.connected_cb.errback(e) try: - transport = yield reactor.spawnProcess(process_protocol, - tor_binary, - args=(tor_binary,'-f',torrc), - env={'HOME': data_dir}, - path=data_dir) - if transport: - transport.closeStdin() - except RuntimeError as e: - log.err("Starting Tor failed: %s" % e) - process_protocol.connected_cb.errback(e) - except NotImplementedError, e: - url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" - log.err("Running bridget on Windows requires pywin32: %s" % url) + transport = reactor.spawnProcess(process_protocol, + tor_binary, + args=(tor_binary,'-f',torrc), + path=data_dir) + transport.closeStdin() + except RuntimeError, e: + log.err(e) process_protocol.connected_cb.errback(e) - #proc_proto = process_protocol.connected_cb - #proc_proto.addCallback(process_cb) - #proc_proto.addErrback(process_eb) - # - #d = yield process_protocol.connected_cb.addCallback( - # process_cb).addErrback( - # process_eb) + return process_protocol.connected_cb + #d = yield process_protocol.connected_cb - #d.addCallback(process_cb) - #d.addErrback(process_eb) - # #defer.returnValue(d) - - d = yield process_protocol.connected_cb - defer.returnValue(d) - - #return process_protocol.connected_cb.addCallback(process_cb).addErrback(process_eb) class CustomCircuit(CircuitListenerMixin): implements(IStreamAttacher)

1 0

[ooni-probe/master] Add some documentation on writing OONI Tests using
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit ffc44ce1cbc595ce8cbcd6522e8962f37b7e6869 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 12:00:07 2012 +0000 Add some documentation on writing OONI Tests using the new framework. --- docs/source/conf.py | 7 ++- docs/source/index.rst | 100 +++++++++++------------------------------ docs/source/install.rst | 55 ++++++++++++++++++++++ docs/source/tutorial.rst | 1 - docs/source/writing_tests.rst | 80 ++++++++++++++++++++++++++------ docs/writing_tests.md | 17 ------- ooni/nettest.py | 43 +++++++++--------- 7 files changed, 174 insertions(+), 129 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 49b96d4..e066613 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,7 +16,9 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, + os.path.join(os.path.dirname(__file__), '..', '..')) + # -- General configuration ----------------------------------------------------- @@ -25,7 +27,8 @@ import sys, os # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', +'sphinx.ext.viewcode', 'sphinx.ext.autodoc'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/source/index.rst b/docs/source/index.rst index 6e95ee7..34ca87c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,96 +1,50 @@ .. OONI documentation master file. -========== -About OONI -========== +Welcome to the OONI documentation! +================================== + The Net interprets censorship as damage and routes around it. + John Gilmore; TIME magazine (6 December 1993) -Dependencies -************ +OONI, the Open Observatory of Network Interference, is a global observation +network which aims is to collect high quality data using open methodologies, +using Free and Open Source Software (FL/OSS) to share observations and data +about the various types, methods, and amounts of network tampering in the world. -* Twisted: http://twistedmatrix.com/trac/ -* PyYAML: http://pyyaml.org/ -* Scapy: http://www.secdev.org/projects/scapy/ - * pypcap: http://code.google.com/p/pypcap/ - * libdnet: http://code.google.com/p/libdnet/ -*Optional* +Getting started +*************** -* BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +If you choose to use virtualenv to setup your development environment you will +need to do the following:: -Installation -************ + virtualenv ENV + source ENV/bin/activate + pip install twisted Scapy pyyaml -On debian you can install all the dependecies with apt-get with this command: +To get the latest version of scapy you will need mercurial. You can then install +it with:: - apt-get install python-twisted python-twisted-names python-yaml python-scapy python-beautifulsoup - -*The "hard" way* - -This involves installing the dependencies installable via easy_install/pip and -the ones that are not by building them from source. - -"simple" dependencies via easy_install: - - sudo easy_install pyyaml - sudo easy_install twisted - sudo easy_install beautifulsoup - -"simple" dependencies via pip: - - sudo pip install pyyaml - sudo pip install twisted - sudo pip install beautifulsoup + pip install hg+http://hg.secdev.org/scapy -libdnet: +On debian you can install all the dependecies with apt-get with this command:: - wget http://libdnet.googlecode.com/files/libdnet-1.12.tgz - tar xzf libdnet-1.12.tgz - cd libdnet-1.12 - ./configure && make - cd python/ - sudo python setup.py install - cd ../../ && rm -rf libdnet-1.12* - -pypcap: - - svn checkout http://pypcap.googlecode.com/svn/trunk/ pypcap-read-only - cd pypcap-read-only/ - sudo pip install pyrex - make - sudo python setup.py install - cd ../ && rm -rf pypcap-read-only - -scapy: - - wget http://www.secdev.org/projects/scapy/files/scapy-latest.zip - unzip scapy-latest.zip - cd scapy-2.2.0/ - sudo python setup.py install - cd ../ && rm -rf scapy-* - -Running -******* + apt-get install python-twisted python-twisted-names python-yaml python-scapy python-beautifulsoup -To run ooni probe do +Once you have installed all the dependencies OONI tests can be run like so:: -$ export PYTHONPATH=`pwd` + bin/ooniprobe path/to/test.py --cmd1 foo --cmd2 bar -$ python ooni/ooniprobe.py +Contents +******** .. toctree:: - :maxdepth: 2 + :maxdepth: 2 + :glob: - intro + install tutorial writing_tests - ... - -Indices and tables -================== -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/source/install.rst b/docs/source/install.rst new file mode 100644 index 0000000..a2352d0 --- /dev/null +++ b/docs/source/install.rst @@ -0,0 +1,55 @@ + +Installing OONI +=============== + +Currently no installation documentation is present, since OONI is not meant to +be installed and should be handled with care. + +Dependencies +************ + +OONI depends on the following pieces of software. + +* Twisted: http://twistedmatrix.com/trac/ +* PyYAML: http://pyyaml.org/ +* Scapy: http://www.secdev.org/projects/scapy/ + * pypcap: http://code.google.com/p/pypcap/ + * libdnet: http://code.google.com/p/libdnet/ + +*Optional* + +* BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ + +Manual installation of scapy +---------------------------- + +It is optimal to install scapy, libdnet and pypcap from source. This can be +done with the following code snippets. + +libdnet:: + + wget http://libdnet.googlecode.com/files/libdnet-1.12.tgz + tar xzf libdnet-1.12.tgz + cd libdnet-1.12 + ./configure && make + cd python/ + sudo python setup.py install + cd ../../ && rm -rf libdnet-1.12* + +pypcap:: + + svn checkout http://pypcap.googlecode.com/svn/trunk/ pypcap-read-only + cd pypcap-read-only/ + sudo pip install pyrex + make + sudo python setup.py install + cd ../ && rm -rf pypcap-read-only + +scapy:: + + wget http://www.secdev.org/projects/scapy/files/scapy-latest.zip + unzip scapy-latest.zip + cd scapy-2.2.0/ + sudo python setup.py install + cd ../ && rm -rf scapy-* + diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index b13b687..15afad7 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -1,4 +1,3 @@ -======== Tutorial ======== diff --git a/docs/source/writing_tests.rst b/docs/source/writing_tests.rst index 1aa4b7c..a18b096 100644 --- a/docs/source/writing_tests.rst +++ b/docs/source/writing_tests.rst @@ -1,26 +1,76 @@ -.. OONI documentation master file. - -================== Writing OONI tests ================== -OONIProbe tests can be written in two modes: blocking or non-blocking. -Going the blocking route is not advised and all tests in the end should end up -being written in the non-blocking way. +The OONI testing API is heavily influenced and partially based on the python +:class:`unittest` module and :class:`twsted.trial`. + + +Test Cases +---------- + +The atom of OONI Testing is called a Test Case. A test case class may contain +multiple Test Functions. + +.. autoclass:: ooni.nettest.TestCase + +:class:`ooni.nettest.TestCase` is a subclass of :class:`unittest.TestCase` so +the assert methods that apply to :class:`unittest.TestCase` will also apply to +:class:`ooni.nettest.TestCase`. + +If the test you plan to write is not listed on the `Tor OONI trac page +<https://trac.torproject.org/projects/tor/wiki/doc/OONI/Tests>`_, you should +add it to the list and following the `test template +<https://trac.torproject.org/projects/tor/wiki/doc/OONI/Tests/TestTemplate>`_ +write up a description about it. + + +Inputs +------ + +Inputs are what is given as input to every iteration of the Test Case. You have +100 inputs, then every test case will be run 100 times. + +To configure a static set of inputs you should define the +:class:`ooni.nettest.TestCase` attribute ``inputs``. The test will be run ``len(inputs)`` times. Any iterable object is a valid ``inputs`` attribute. + +If you would like to have inputs be determined from a user specified input +file, then you must set the ``inputFile`` attribute. This is an array that +specifies what command line option may be used to control this value. + +By default the ``inputProcessor`` is set to read the file line by line and +strip newline characters. To change this behavior you must set the +``inputProcessor`` attribute to a function that takes as arugment a file +descriptor and yield the next item. The default ``inputProcessor`` looks like +this:: + + + def lineByLine(fp): + for x in fp.readlines(): + yield x.strip() + fp.close() + + +Test Functions +-------------- + +These shall be defined inside of your :class:`ooni.nettest.TestCase` subclass. +These will be class methods. -A good way to understand how to write a test is also to take a look at the OONI -Test Interface in the following files: +To add data to the test report you may write directly to the report object like +so:: -* ooni/plugoo/interface.py + def my_test_function(): + result = do_something() + self.report['something'] = result -* ooni/plugoo/tests.py +OONI will then handle the writing of the data to the final test report. -Writing non-blocking tests --------------------------- +To access the current input you can use the ``input`` attribute, for example:: -To bootstrap the process of creating a new tests you can run the scaffolding -script in ooni/scaffolding.py. + def my_test_with_input(): + do_something_with_input(self.input) -This will create a new plugin with the specified name inside of ooni/plugins/. +This will at each iteration over the list of inputs do something with the +input. diff --git a/docs/writing_tests.md b/docs/writing_tests.md deleted file mode 100644 index 460620a..0000000 --- a/docs/writing_tests.md +++ /dev/null @@ -1,17 +0,0 @@ -= Writing OONI Tests = - -There are two modes of writing tests. The first way is by making the Test -blocking and have each instance of a test run in a separate thread. The other -is by making them non-blocking using the twisted deferred non blocking pattern. - -The first kind of mechanism relied of writing a test that uses blocking code -and each instance of it is run inside of a separate thread. Beware that your -blocking code must be thread safe to run properly (as is obvious :P) - -The other menthod invloves having some knowledge about twisted. The test you -will write will be written in twisted and should implement async style non -blocking architecture. - -It is recommended that tests are written using the second pattern and the first -should only be used for runnign tests that have been previously written. - diff --git a/ooni/nettest.py b/ooni/nettest.py index 2a02a43..c0c7406 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -34,39 +34,40 @@ class TestCase(unittest.TestCase): This is the monad of the OONI nettest universe. When you write a nettest you will subclass this object. - _inputs_ can be set to a static set of inputs. All the tests (the methods - starting with the "test_" prefix) will be run once per input. At every run - the _input_ attribute of the TestCase instance will be set to the value of - the current iteration over inputs. Any python iterable object can be set - to inputs. + * inputs: can be set to a static set of inputs. All the tests (the methods + starting with the "test_" prefix) will be run once per input. At every run + the _input_ attribute of the TestCase instance will be set to the value of + the current iteration over inputs. Any python iterable object can be set + to inputs. - _inputFile_ attribute should be set to an array containing the command line - argument that should be used as the input file. Such array looks like this: + * inputFile: attribute should be set to an array containing the command line + argument that should be used as the input file. Such array looks like + this: - ["commandlinearg", "c", "The description"] + ``["commandlinearg", "c", "The description"]`` - The second value of such arrray is the shorthand for the command line arg. - The user will then be able to specify inputs to the test via: + The second value of such arrray is the shorthand for the command line arg. + The user will then be able to specify inputs to the test via: - ooniprobe mytest.py --commandlinearg path/to/file.txt + ``ooniprobe mytest.py --commandlinearg path/to/file.txt`` - or + or - ooniprobe mytest.py -c path/to/file.txt + ``ooniprobe mytest.py -c path/to/file.txt`` - _inputProcessor_ should be set to a function that takes as argument an - open file descriptor and it will yield the input to be passed to the test - instance. + * inputProcessor: should be set to a function that takes as argument an + open file descriptor and it will yield the input to be passed to the test + instance. - _name_ should be set to the name of the test. + * name: should be set to the name of the test. - _author_ should contain the name and contact details for the test author. - The format for such string is as follows: + * author: should contain the name and contact details for the test author. + The format for such string is as follows: - The Name <email(a)example.com> + ``The Name <email(a)example.com>`` - _version_ is the version string of the test. + * version: is the version string of the test. """ name = "IDidNotChangeTheName" author = "John Doe <foo(a)example.com>"

1 0

[ooni-probe/master] * Added more general Tor utility functions to ooni.utils.onion.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit f88900672897e858ec85da4f5da343cf99ba28d6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Sep 26 12:43:56 2012 +0000 * Added more general Tor utility functions to ooni.utils.onion. * Added decorator-based timeout function for blocking methods to ooni.utils.timer. * Moved general Exception classes to appropriate files. * Fixed another bug in the Options type enforcer. --- ooni/plugins/bridget.py | 212 ++++++++++++++-------------------------------- ooni/plugoo/assets.py | 5 + ooni/plugoo/tests.py | 1 - ooni/utils/config.py | 65 ++++++++++++++ ooni/utils/onion.py | 90 ++++++++++++-------- ooni/utils/process.py | 54 ++++++++++++ ooni/utils/timer.py | 36 ++++++++ 7 files changed, 281 insertions(+), 182 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 31537ec..99a7d05 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -12,135 +12,48 @@ # :licence: see included LICENSE # :version: 0.1.0-alpha -from __future__ import with_statement -from random import randint -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import defer, error, reactor -from zope.interface import implements - -from ooni.utils import log -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset +from __future__ import with_statement +from functools import partial +from random import randint +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor +from zope.interface import implements + +from ooni.utils import log, date +from ooni.utils.config import ValueChecker +from ooni.utils.onion import parse_data_dir +from ooni.utils.onion import TxtorconImportError +from ooni.utils.onion import PTNoBridgesException, PTNotFoundException +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset, MissingAssetException import os import sys -def timer(secs, e=None): - import signal - import functools.wraps - def decorator(func): - def _timer(signum, frame): - raise TimeoutError, e - def wrapper(*args, **kwargs): - signal.signal(signal.SIGALRM, _timer) - signal.alarm(secs) - try: - res = func(*args, **kwargs) - finally: - signal.alarm(0) - return res - return functools.wraps(func)(wrapper) - return decorator - - -class MissingAssetException(Exception): - """Raised when neither there are neither bridges nor relays to test.""" - def __init__(self): - log.msg("Bridget can't run without bridges or relays to test!") - return sys.exit() - -class PTNoBridgesException(Exception): - """Raised when a pluggable transport is specified, but no bridges.""" - def __init__(self): - log.msg("Pluggable transport requires the bridges option") - return sys.exit() - -class PTNotFoundException(Exception): - def __init__(self, transport_type): - m = "Pluggable Transport type %s was unaccounted " % transport_type - m += "for, please contact isis(at)torproject(dot)org and it will " - m += "get included." - log.msg("%s" % m) - return sys.exit() - -class ValueChecker(object): - def port_check(self, port): - """Check that given ports are in the allowed range.""" - port = int(port) - if port not in range(1024, 65535): - raise ValueError("Port out of range") - log.err() - - sock_check, ctrl_check = port_check, port_check - allowed = "must be between 1024 and 65535." - sock_check.coerceDoc = "Port to use for Tor's SocksPort, " +allowed - ctrl_check.coerceDoc = "Port to use for Tor's ControlPort, " +allowed - - def uid_check(self, pluggable_transport): - """ - Check that we're not root when trying to use pluggable transports. If - we are, setuid to normal user (1000) if we're running on a posix-based - system, and if we're Windows just tell the user that we can't be run - as root with the specified options and then exit. - """ - uid, gid = os.getuid(), os.getgid() - if uid == 0 and gid == 0: - log.msg("Error: Running bridget as root with transports not allowed.") - if os.name == 'posix': - log.msg("Dropping privileges to normal user...") - os.setgid(1000) - os.setuid(1000) - else: - sys.exit(0) - - def dir_check(self, d): - """Check that the given directory exists.""" - if not os.path.isdir(d): - raise ValueError("%s doesn't exist, or has wrong permissions" % d) - - def file_check(self, f): - """Check that the given file exists.""" - if not os.path.isfile(f): - raise ValueError("%s does not exist, or has wrong permissions" % f) - class RandomPortException(Exception): """Raised when using a random port conflicts with configured ports.""" def __init__(self): log.msg("Unable to use random and specific ports simultaneously") return sys.exit() -class TimeoutError(Exception): - """Raised when a timer runs out.""" - pass - -class TxtorconImportError(ImportError): - """Raised when /ooni/lib/txtorcon cannot be imported from.""" - cwd, tx = os.getcwd(), 'lib/txtorcon/torconfig.py' - try: - log.msg("Unable to import from ooni.lib.txtorcon") - if cwd.endswith('ooni'): - check = os.path.join(cwd, tx) - else: - check = os.path.join(cwd, 'ooni/'+tx) - assert isfile(check) - except: - log.msg("Error: Some OONI libraries are missing!") - log.msg("Please go to /ooni/lib/ and do \"make all\"") - class BridgetArgs(usage.Options): """Commandline options.""" global vc - vc = ValueChecker() + vc = ValueChecker + allowed = "Port to use for Tor's %s, must be between 1024 and 65535." + sock_check = vc(allowed % "SocksPort").port_check + ctrl_check = vc(allowed % "ControlPort").port_check + optParameters = [ ['bridges', 'b', None, 'File listing bridge IP:ORPorts to test'], ['relays', 'f', None, 'File listing relay IPs to test'], - ['socks', 's', 9049, None, vc.sock_check], - ['control', 'c', 9052, None, vc.ctrl_check], + ['socks', 's', 9049, None, sock_check], + ['control', 'c', 9052, None, ctrl_check], ['torpath', 'p', None, 'Path to the Tor binary to use'], ['datadir', 'd', None, @@ -153,18 +66,20 @@ class BridgetArgs(usage.Options): def postOptions(self): if not self['bridges'] and not self['relays']: - raise MissingAssetException + raise MissingAssetException( + "Bridget can't run without bridges or relays to test!") if self['transport']: - vc.uid_check(self['transport']) + vc().uid_check( + "Can't run bridget as root with pluggable transports!") if not self['bridges']: raise PTNoBridgesException if self['socks'] or self['control']: if self['random']: raise RandomPortException if self['datadir']: - vc.dir_check(self['datadir']) + vc().dir_check(self['datadir']) if self['torpath']: - vc.file_check(self['torpath']) + vc().file_check(self['torpath']) class BridgetAsset(Asset): """Class for parsing bridget Assets ignoring commented out lines.""" @@ -211,6 +126,8 @@ class BridgetTest(OONITest): running, so we need to deal with most of the TorConfig() only once, before the experiment runs. """ + self.d = defer.Deferred() + self.socks_port = 9049 self.control_port = 9052 self.circuit_timeout = 90 @@ -219,6 +136,7 @@ class BridgetTest(OONITest): self.use_pt = False self.pt_type = None + ## XXX we should do report['bridges_up'].append(self.current_bridge) self.bridges, self.bridges_up, self.bridges_down = ([] for i in range(3)) self.bridges_remaining = lambda: len(self.bridges) self.bridges_down_count = lambda: len(self.bridges_down) @@ -229,6 +147,9 @@ class BridgetTest(OONITest): self.relays_down_count = lambda: len(self.relays_down) self.current_relay = None + ## Make sure we don't use self.load_assets() for now: + self.assets = {} + def __make_asset_list__(opt, lst): log.msg("Loading information from %s ..." % opt) with open(opt) as opt_file: @@ -238,17 +159,6 @@ class BridgetTest(OONITest): else: lst.append(line.replace('\n','')) - def __parse_data_dir__(data_dir): - if data_dir.startswith('~'): - data_dir = os.path.expanduser(data_dir) - elif data_dir.startswith('/'): - data_dir = os.path.join(os.getcwd(), data_dir) - elif data_dir.startswith('./'): - data_dir = os.path.abspath(data_dir) - else: - data_dir = os.path.join(os.getcwd(), data_dir) - return data_dir - if self.local_options: try: from ooni.lib.txtorcon import TorConfig @@ -281,7 +191,7 @@ class BridgetTest(OONITest): self.tor_binary = options['torpath'] if options['datadir']: - self.data_directory = __parse_data_dir__(options['datadir']) + self.data_directory = parse_data_dir(options['datadir']) else: self.data_directory = None @@ -294,7 +204,7 @@ class BridgetTest(OONITest): ## ClientTransportPlugin transport exec pathtobinary [options] ## XXX we need a better way to deal with all PTs if self.pt_type == "obfs2": - config.ClientTransportPlugin = self.pt_type + " " + pt_exec + self.config.ClientTransportPlugin = self.pt_type+" "+pt_exec else: raise PTNotFoundException @@ -302,7 +212,6 @@ class BridgetTest(OONITest): self.config.ControlPort = self.control_port self.config.CookieAuthentication = 1 - ''' def load_assets(self): """ Load bridges and/or relays from files given in user options. Bridges @@ -318,7 +227,6 @@ class BridgetTest(OONITest): assets.update({'relay': BridgetAsset(self.local_options['relays'])}) return assets - ''' def experiment(self, args): """ @@ -367,13 +275,14 @@ class BridgetTest(OONITest): } of unreachable relays :param args: - The :class:`BridgetAsset` line currently being used. + The :class:`BridgetAsset` line currently being used. Except that it + in Bridget it doesn't, so it should be ignored and avoided. """ try: - from ooni.utils.onion import start_tor, singleton_semaphore - from ooni.utils.onion import setup_done, setup_fail - from ooni.utils.onion import CustomCircuit - from ooni.lib.txtorcon import TorConfig, TorState + from ooni.utils.process import singleton_semaphore + from ooni.utils.onion import start_tor, setup_done, setup_fail + from ooni.utils.onion import CustomCircuit + from ooni.lib.txtorcon import TorConfig, TorState except ImportError: raise TxtorconImportError except TxtorconImportError, tie: @@ -509,7 +418,7 @@ class BridgetTest(OONITest): log.msg("Bridget: initiating test ... ") - d = defer.Deferred + x = defer.Deferred if self.bridges_remaining() > 0 and not 'Bridge' in self.config.config: self.config.Bridge = self.bridges.pop() @@ -527,13 +436,13 @@ class BridgetTest(OONITest): setup_fail) self.tor_process_semaphore = True - run_once = d().addCallback(singleton_semaphore, tor) + run_once = x().addCallback(singleton_semaphore, tor) run_once.addErrback(setup_fail) - only_bridges = d().addCallback(remove_public_relays, self.bridges) + filter_bridges = x().addCallback(remove_public_relays, self.bridges) - state = defer.gatherResults([run_once, only_bridges], consumeErrors=True) - log.debug("%s" % state.callbacks) + state = defer.gatherResults([run_once, filter_bridges], consumeErrors=True) + log.debug("Current callbacks on TorState():\n%s" % state.callbacks) if self.bridges_remaining() > 0: all = [] @@ -541,7 +450,7 @@ class BridgetTest(OONITest): self.current_bridge = bridge log.msg("We now have %d untested bridges..." % self.bridges_remaining()) - reconf = d().addCallback(reconfigure_bridge, state, + reconf = x().addCallback(reconfigure_bridge, state, self.current_bridge, self.use_pt, self.pt_type) @@ -549,10 +458,14 @@ class BridgetTest(OONITest): self.bridges_up) reconf.addErrback(reconfigure_fail, self.current_bridge, self.bridges_down) - all.append(reconf) - state.chainDeferred(defer.DeferredList(all)) - log.debug("%s" % state.callbacks) - + all.append(reconf) + + #state.chainDeferred(defer.DeferredList(all)) + #state.chainDeferred(defer.gatherResults(all, consumeErrors=True)) + n_plus_one_bridges = defer.gatherResults(all, consumeErrors=True) + state.chainDeferred(n_plus_one_bridges) + log.debug("Current callbacks on TorState():\n%s" % state.callbacks) + if self.relays_remaining() > 0: while self.relays_remaining() >= 3: #path = list(self.relays.pop() for i in range(3)) @@ -575,11 +488,16 @@ class BridgetTest(OONITest): else: continue - #reactor.run() - return state + state.callback(all) + self.reactor.run() + #return state - def control(self, experiment_result, args): - experiment_result.callback + def startTest(self, args): + self.start_time = date.now() + log.msg("Starting %s" % self.shortName) + self.do_science = self.experiment(args) + self.do_science.addCallback(self.finished).addErrback(log.err) + return self.do_science ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) diff --git a/ooni/plugoo/assets.py b/ooni/plugoo/assets.py index eded997..3974230 100644 --- a/ooni/plugoo/assets.py +++ b/ooni/plugoo/assets.py @@ -54,3 +54,8 @@ class Asset: except: raise StopIteration +class MissingAssetException(Exception): + """Raised when an Asset necessary for running the Test is missing.""" + def __init__(self, error_message): + log.msg(error_message) + return sys.exit() diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 42f9542..92bf88f 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -38,7 +38,6 @@ class OONITest(object): #self.ooninet = ooninet self.reactor = reactor self.result = {} - self.initialize() self.assets = self.load_assets() diff --git a/ooni/utils/config.py b/ooni/utils/config.py index ab43e66..7c28f33 100644 --- a/ooni/utils/config.py +++ b/ooni/utils/config.py @@ -1,4 +1,6 @@ import ConfigParser +import os + from ooni.utils import Storage class Config(Storage): @@ -51,3 +53,66 @@ class Config(Storage): self._cfgparser.write(cfgfile) finally: cfgfile.close() + +class ValueChecker(object): + """ + A class for general purpose value checks on commandline parameters + passed to subclasses of :class:`twisted.python.usage.Options`. + """ + def __init__(self, coerce_doc=None): + self.coerce_doc = coerce_doc + + def port_check(self, port, range_min=1024, range_max=65535): + """ + Check that given ports are in the allowed range for an unprivileged + user. + + :param port: + The port to check. + :param range_min: + The minimum allowable port number. + :param range_max: + The minimum allowable port number. + :param coerce_doc: + The documentation string to show in the optParameters menu, see + :class:`twisted.python.usage.Options`. + """ + if self.coerce_doc is not None: + coerceDoc = self.coerce_doc + + assert type(port) is int + if port not in range(range_min, range_max): + raise ValueError("Port out of range") + log.err() + + def uid_check(self, error_message): + """ + Check that we're not root. If we are, setuid(1000) to normal user if + we're running on a posix-based system, and if we're on Windows just + tell the user that we can't be run as root with the specified options + and then exit. + + :param error_message: + The string to log as an error message when the uid check fails. + """ + uid, gid = os.getuid(), os.getgid() + if uid == 0 and gid == 0: + log.msg(error_message) + if os.name == 'posix': + log.msg("Dropping privileges to normal user...") + os.setgid(1000) + os.setuid(1000) + else: + sys.exit(0) + + def dir_check(self, d): + """Check that the given directory exists.""" + if not os.path.isdir(d): + raise ValueError("%s doesn't exist, or has wrong permissions" + % d) + + def file_check(self, f): + """Check that the given file exists.""" + if not os.path.isfile(f): + raise ValueError("%s does not exist, or has wrong permissions" + % f) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 5f15e90..81e4ea3 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -44,12 +44,34 @@ def state_complete(state): for circ in state.circuits.values(): log.msg("%s" % circ) - return state + #return state def updates(_progress, _tag, _summary): """Log updates on the Tor bootstrapping process.""" log.msg("%d%%: %s" % (_progress, _summary)) +def parse_data_dir(data_dir): + """ + Parse a string that a has been given as a DataDirectory and determine + its absolute path on the filesystem. + + :param data_dir: + A directory for Tor's DataDirectory, to be parsed. + :return: + The absolute path of :param:data_dir. + """ + import os + + if data_dir.startswith('~'): + data_dir = os.path.expanduser(data_dir) + elif data_dir.startswith('/'): + data_dir = os.path.join(os.getcwd(), data_dir) + elif data_dir.startswith('./'): + data_dir = os.path.abspath(data_dir) + else: + data_dir = os.path.join(os.getcwd(), data_dir) + return data_dir + def write_torrc(conf, data_dir=None): """ Create a torrc in our data_dir. If we don't yet have a data_dir, create a @@ -103,38 +125,6 @@ def delete_files_or_dirs(delete_list): rmtree(temp, ignore_errors=True) @defer.inlineCallbacks -def singleton_semaphore(deferred_process_init, callbacks=[], errbacks=[]): - """ - Initialize a process only once, and do not return until - that initialization is complete. - - :param deferred_process_init: - A deferred which returns a connected process via - :meth:`twisted.internet.reactor.spawnProcess`. - :param callbacks: - A list of callback functions to add to the initialized processes' - deferred. - :param errbacks: - A list of errback functions to add to the initialized processes' - deferred. - :return: - The final state of the :param deferred_process_init: after the - callback chain has completed. This should be a fully initialized - process connected to a :class:`twisted.internet.reactor`. - """ - assert type(callbacks) is list - assert type(errbacks) is list - - for cb in callbacks: - deferred_process_init.addCallback(cb) - for eb in errbacks: - deferred_process_init.addErrback(eb) - - only_once = defer.DeferredSemaphore(1) - singleton = yield only_once.run(deferred_process_init) - defer.returnValue(singleton) - -(a)defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, report=None, progress=updates, process_cb=setup_done, process_eb=setup_fail): @@ -237,7 +227,6 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, #return process_protocol.connected_cb.addCallback(process_cb).addErrback(process_eb) - class CustomCircuit(CircuitListenerMixin): implements(IStreamAttacher) @@ -328,3 +317,36 @@ class CustomCircuit(CircuitListenerMixin): return self.state.build_circuit(path).addCallback( AppendWaiting(self, deferred)).addErrback( log.err) + +class TxtorconImportError(ImportError): + """Raised when /ooni/lib/txtorcon cannot be imported from.""" + from os import getcwd, path + + cwd, tx = getcwd(), 'lib/txtorcon/torconfig.py' + try: + log.msg("Unable to import from ooni.lib.txtorcon") + if cwd.endswith('ooni'): + check = path.join(cwd, tx) + elif cwd.endswith('utils'): + check = path.join(cwd, '../'+tx) + else: + check = path.join(cwd, 'ooni/'+tx) + assert path.isfile(check) + except: + log.msg("Error: Some OONI libraries are missing!") + log.msg("Please go to /ooni/lib/ and do \"make all\"") + +class PTNoBridgesException(Exception): + """Raised when a pluggable transport is specified, but not bridges.""" + def __init__(self): + log.msg("Pluggable transport requires the bridges option") + return sys.exit() + +class PTNotFoundException(Exception): + def __init__(self, transport_type): + m = "Pluggable Transport type %s was unaccounted " % transport_type + m += "for, please contact isis(at)torproject(dot)org and it will " + m += "get included." + log.msg("%s" % m) + return sys.exit() + diff --git a/ooni/utils/process.py b/ooni/utils/process.py new file mode 100644 index 0000000..e9f56a0 --- /dev/null +++ b/ooni/utils/process.py @@ -0,0 +1,54 @@ +# +# process.py +# ---------- +# OONI utilities for dealing with starting and stopping processes. +# +# :author: Isis Lovecruft +# :version: 0.1.0-pre-alpha +# :license: see include LICENSE file +# :copyright: copyright (c) 2012, Isis Lovecruft, The Tor Project Inc. +# + +from twisted.internet import defer + +(a)defer.inlineCallbacks +def singleton_semaphore(deferred_process_init, + callbacks=[], errbacks=[], + max_inits=1): + """ + Initialize a process only once, and do not return until + that initialization is complete. If the keyword parameter max_inits= + is given, run the process a maximum of that number of times. + + :param deferred_process_init: + A deferred which returns a connected process via + :meth:`twisted.internet.reactor.spawnProcess`. + :param callbacks: + A list of callback functions to add to the initialized processes' + deferred. + :param errbacks: + A list of errback functions to add to the initialized processes' + deferred. + :param max_inits: + An integer specifying the maximum number of allowed + initializations for :param:deferred_process_init. If no maximum + is given, only one instance (a singleton) of the process is + initialized. + :return: + The final state of the :param deferred_process_init: after the + callback chain has completed. This should be a fully initialized + process connected to a :class:`twisted.internet.reactor`. + """ + assert type(callbacks) is list + assert type(errbacks) is list + assert type(max_inits) is int + + for cb in callbacks: + deferred_process_init.addCallback(cb) + for eb in errbacks: + deferred_process_init.addErrback(eb) + + only_this_many = defer.DeferredSemaphore(max_inits) + singleton = yield only_this_many.run(deferred_process_init) + defer.returnValue(singleton) + diff --git a/ooni/utils/timer.py b/ooni/utils/timer.py new file mode 100644 index 0000000..8ad9b79 --- /dev/null +++ b/ooni/utils/timer.py @@ -0,0 +1,36 @@ +# +# timer.py +# ---------- +# OONI utilities for adding timeouts to functions and to Deferreds. +# +# :author: Isis Lovecruft +# :version: 0.1.0-pre-alpha +# :license: see include LICENSE file +# :copyright: copyright (c) 2012, Isis Lovecruft, The Tor Project Inc. +# + + +class TimeoutError(Exception): + """Raised when a timer runs out.""" + pass + +def timeout(secs, e=None): + """ + A decorator for blocking methods to cause them to timeout. + """ + import signal + import functools.wraps + def decorator(func): + def _timeout(signum, frame): + raise TimeoutError, e + + def wrapper(*args, **kwargs): + signal.signal(signal.SIGALRM, _timeout) + signal.alarm(secs) + try: + res = func(*args, **kwargs) + finally: + signal.alarm(0) + return res + return functools.wraps(func)(wrapper) + return decorator

1 0

[ooni-probe/master] Merge branch 'nettest'
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 16c3cfedbd80d23337579d6d5fc3f9bff367e714 Merge: 2e49b73 d621155 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 12:06:54 2012 +0000 Merge branch 'nettest' Conflicts: ooni/plugins/new_bridget.py .gitignore | 1 + README.md | 10 + bin/ooniprobe | 16 +- docs/design.dia | Bin 1706 -> 1706 bytes docs/source/conf.py | 7 +- docs/source/index.rst | 100 ++---- docs/source/install.rst | 55 +++ docs/source/tutorial.rst | 1 - docs/source/writing_tests.rst | 88 ++++- docs/writing_tests.md | 17 - ooni/input.py | 65 ++++ ooni/nettest.py | 90 +++++ ooni/oonicli.py | 105 ++++++ ooni/oonitests/bridget.py | 373 -------------------- ooni/plugins/httpt.py | 9 +- ooni/plugins/new_bridget.py | 4 +- ooni/plugoo/tests.py | 5 +- ooni/protocols/http.py | 17 +- ooni/reporter.py | 199 +++++++++++ ooni/runner.py | 191 ++++++++++ 20 files changed, 852 insertions(+), 501 deletions(-) diff --cc README.md index 7d6e37a,123d837..23deb32 --- a/README.md +++ b/README.md @@@ -32,8 -26,18 +32,18 @@@ If you see some errors see INSTALL to i To list the help for a specific test: - python ooniprobe.py httpt --help + $ python ooniprobe.py httpt --help + ## Virtualenv way (Recommended) + + virtualenv2 ENV/ + source ENV/bin/activate + pip install twisted Scapy + + To install the most up to date scapy version (requires mercurial): + + pip install hg+http://hg.secdev.org/scapy + # More details diff --cc ooni/ooniprobe.py index 86f4e2d,f80669e..86f4e2d mode 100755,100644..100644 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py diff --cc ooni/plugins/new_bridget.py index 736b084,b26455e..673cacc --- a/ooni/plugins/new_bridget.py +++ b/ooni/plugins/new_bridget.py @@@ -1,421 -1,105 +1,421 @@@ -""" -This is a self genrated test created by scaffolding.py. -you will need to fill it up with all your necessities. -Safe hacking :). -""" -from exceptions import Exception -from datetime import datetime -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import reactor, task - -from ooni.utils import log -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset - -from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState -from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor - -class bridgetArgs(usage.Options): - optParameters = [['bridges', 'b', None, 'List of bridges to scan'], - ['relays', 'f', None, 'List of relays to scan'], - ['resume', 'r', 0, 'Resume at this index'], - ['timeout', 't', 5, 'Timeout in seconds after which to consider a bridge not working'] - ] - -class bridgetTest(OONITest): +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# +-----------+ +# | BRIDGET | +# | +----------------------------------------------+ +# +--------| Use a slave Tor process to test making a Tor | +# | connection to a list of bridges or relays. | +# +----------------------------------------------+ +# +# :authors: Arturo Filasto, Isis Lovecruft +# :licence: see included LICENSE +# :version: 0.1.0-alpha + +from __future__ import with_statement +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor + +import random +import sys + +try: + from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +except: + print "BridgeT requires txtorcon: https://github.com/meejah/txtorcon.git" + print "Your copy of OONI should have it included, if you're seeing this" + print "message, please file a bug report." + log.msg ("Bridget: Unable to import from ooni.lib.txtorcon") + +from ooni.utils import log +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset + + +class BridgetArgs(usage.Options): + + def portCheck(number): + number = int(number) + if number not in range(1024, 65535): + raise ValueError("Port out of range") + portCheck.coerceDoc = "Ports must be between 1024 and 65535" + + optParameters = [ + ['bridges', 'b', None, + 'List of bridges to scan <IP>:<ORport>'], + ['relays', 'f', None, + 'List of relays to scan <IP>'], + ['socks', 's', 9049, portCheck, + 'Tor SocksPort to use'], + ['control', 'c', 9052, portCheck, + 'Tor ControlPort to use'], + ['tor-path', 'p', None, + 'Path to the Tor binary to use'], + ['data-dir', 'd', None, + 'Tor DataDirectory to use'], + ['transport', 't', None, + 'Tor ClientTransportPlugin'], + ['resume', 'r', 0, + 'Resume at this index']] + optFlags = [['random', 'x', 'Randomize control and socks ports']] + + def postOptions(self): + ## We can't test pluggable transports without bridges + if self['transport'] and not self['bridges']: + e = "Pluggable transport requires the bridges option" + raise usage.UsageError, e + ## We can't use random and static port simultaneously + if self['socks'] and self['control']: + if self['random']: + e = "Unable to use random and specific ports simultaneously" + raise usageError, e + +class CustomCircuit(CircuitListenerMixin): + implements(IStreamAttacher) + + from txtorcon.interface import IRouterContainer, ICircuitContainer + + def __init__(self, state): + self.state = state + self.waiting_circuits = [] + + def waiting_on(self, circuit): + for (circid, d) in self.waiting_circuits: + if circuit.id == circid: + return true + return False + + def circuit_extend(self, circuit, router): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + if self.waiting_on(circuit): + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) + + def circuit_built(self, circuit): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + + log.msg("Circuit %s built ..." % circuit.id) + log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) + + for (circid, d) in self.waiting_circuits: + if circid == circuit.id: + self.waiting_circuits.remove(circid, d) + d.callback(circuit) + + def circuit_failed(self, circuit, reason): + if self.waiting_on(circuit): + log.msg("A circuit we requested %s failed for reason %s" % + (circuit.id, reason)) + circid, d = None, None + for x in self.waiting_circuits: + if x[0] == circuit.id: + circid, d, stream_cc = x + if d is None: + raise Exception("Expected to find circuit.") + + self.waiting_circuits.remove((circid, d)) + log.msg("Trying to build a circuit for %s" % circid) + self.request_circuit_build(d) + + def check_circuit_route(self, circuit, router): + if router in circuit.path: + #router.update() ## XXX can i use without args? no. + TorInfo.dump(self) + + def request_circuit_build(self, deferred): + entries = self.state.entry_guards.value() + relays = self.state.routers.values() + + log.msg("We have these nodes listed as entry guards:") + log.msg("%s" % entries) + log.msg("We have these nodes listed as relays:") + log.msg("%s" % relays) + + path = [random.choice(entries), + random.choice(relays), + random.choice(relays)] + + log.msg("Requesting a circuit: %s" + % '-->'.join(map(lambda x: x.location.countrycode, path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + + def __call__(self, circuit): + """ + Return from build_circuit is a Circuit, however, we want to + wait until it is built before we can issue an attach on it and + callback to the Deferred we issue here. + """ + log.msg("Circuit %s is in progress ..." % circuit.id) + self.attacher.waiting_circuits.append((circuit.id, self.d)) + + return self.state.build_circuit(path).addCallback(AppendWaiting(self, deferred_to_callback)).addErrback(log.err) + +class BridgetAsset(Asset): + """ + Class for parsing bridge assets so that they can be commented out. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + if line.startswith('#'): + return + else: + return line.replace('\n','') + +class BridgetTest(OONITest): + """ + XXX fill me in + + :ivar config: + An :class:`ooni.lib.txtorcon.TorConfig` instance. + :ivar relay_list: + A list of all provided relays to test. We have to do this because + txtorcon.TorState().entry_guards won't build a custom circuit if the + first hop isn't in the torrc's EntryNodes. + :ivar bridge_list: + A list of all provided bridges to test. + :ivar socks_port: + Integer for Tor's SocksPort. + :ivar control_port: + Integer for Tor's ControlPort. + :ivar plug_transport: + String defining the Tor's ClientTransportPlugin, for testing + a bridge's pluggable transport functionality. + :ivar tor_binary: + Path to the Tor binary to use, e.g. \'/usr/sbin/tor\' + """ implements(IPlugin, ITest) - shortName = "bridget" - description = "Use a Tor process to test connecting to bridges and relays" + shortName = "newbridget" + description = "bridget" requirements = None - options = bridgetArgs + options = BridgetArgs blocking = False - def experiment(self, args): - log.msg("Doing test") - last_update = datetime.now() - tor_log = [] - - def check_timeout(): - log.msg("Checking for timeout") - time_since_update = datetime.now() - last_update - if time_since_update.seconds > self.local_options['timeout']: - log.msg("Timed out when connecting to %s" % args) - l.stop() - self.result['reason'] = 'timeout' - d.errback(args) - return + def initialize(self): + """ + Extra initialization steps. We only want one child Tor process + running, so we need to deal with the creation of TorConfig() only + once, before the experiment runs. + """ + self.socks_port = 9049 + self.control_port = 9052 + self.tor_binary = '/usr/sbin/tor' + self.data_directory = None - def updates(prog, tag, summary): - tor_log.append((prog, tag, summary)) - last_update = datetime.now() - log.msg("%d%%: %s" % (prog, summary)) + if self.local_options: + try: + from ooni.lib.txtorcon import TorConfig + except: + e = "Could not import TorConfig class from txtorcon!" + raise ImportError, e - def setup_failed(failure): - log.msg("Setup Failed.") - if not self.result['reason']: - self.result['reason'] = 'unknown' - self.result['input'] = args - self.result['result'] = 'failed' - self.result['tor_log'] = tor_log - return + options = self.local_options + self.config = TorConfig() - def setup_complete(proto): - log.msg("Setup Complete.") - self.result['input'] = args - self.result['result'] = 'success' - return + ## Don't run the experiment if we don't have anything to test + if not options['bridges'] and not options['relays']: + self.suicide = True + + if options['bridges']: + self.config.UseBridges = 1 - config = TorConfig() - import random - config.SocksPort = random.randint(1024, 2**16) - config.ControlPort = random.randint(1024, 2**16) + if options['relays']: + ## Stupid hack for testing only relays: + ## Tor doesn't use EntryNodes when UseBridges is enabled, but + ## config.state.entry_guards needs to include the first hop to + ## build a custom circuit. + self.config.EntryNodes = ','.join(relay_list) - if 'bridge' in args: - config.UseBridges = 1 - config.Bridge = args['bridge'] + if options['socks']: + self.socks_port = options['socks'] - config.save() + if options['control']: + self.control_port = options['control'] - print config.config - self.result['tor_config'] = config.config - log.msg("Starting Tor connecting to %s" % args['bridge']) + if options['random']: + log.msg("Using randomized ControlPort and SocksPort ...") + self.socks_port = random.randint(1024, 2**16) + self.control_port = random.randint(1024, 2**16) - l = task.LoopingCall(check_timeout) - l.start(1.0) + if options['tor-path']: + self.tor_binary = options['tor-path'] - d = launch_tor(config, self.reactor, control_port=config.ControlPort, progress_updates=updates) - d.addCallback(setup_complete) - d.addErrback(setup_failed) - return d + if options['data-dir']: + self.config.DataDirectory = options['data-dir'] + + if options['transport']: + ## ClientTransportPlugin transport socks4|socks5 IP:PORT + ## ClientTransportPlugin transport exec path-to-binary [options] + if not options['bridges']: + e = "You must use the bridge option to test a transport." + raise usage.UsageError("%s" % e) + + log.msg("Using pluggable transport ...") + ## XXX fixme there's got to be a better way to check the exec + assert type(options['transport']) is str + self.config.ClientTransportPlugin = options['transport'] + + self.config.SocksPort = self.socks_port + self.config.ControlPort = self.control_port + self.config.save() def load_assets(self): - assets = {} + """ + Load bridges and/or relays from files given in user options. Bridges + should be given in the form IP:ORport. We don't want to load these as + assets, because it's inefficient to start a Tor process for each one. + """ + assets = {} + self.bridge_list = [] + self.relay_list = [] + + ## XXX fix me + ## we should probably find a more memory nice way to load addresses, + ## in case the files are really large if self.local_options: if self.local_options['bridges']: - assets.update({'bridge': Asset(self.local_options['bridges'])}) - elif self.local_options['relays']: - assets.update({'relay': Asset(self.local_options['relay'])}) + log.msg("Loading bridge information from %s ..." + % self.local_options['bridges']) + with open(self.local_options['bridges']) as bridge_file: + for line in bridge_file.readlines(): + if line.startswith('#'): + continue + else: + self.bridge_list.append(line.replace('\n','')) + assets.update({'bridges': self.bridge_list}) + + if self.local_options['relays']: + log.msg("Loading relay information from %s ..." + % self.local_options['relays']) + with open(options['relays']) as relay_file: + for line in relay_file.readlines(): + if line.startswith('#'): + continue + else: + self.relay_list.append(line.replace('\n','')) + assets.update({'relays': self.relay_list}) return assets -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -bridget = bridgetTest(None, None, None) + def experiment(self, args): + """ + XXX fill me in + + :param args: + The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being + used. + :meth launch_tor: + Returns a Deferred which callbacks with a + :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol + <TorProcessProtocol>` connected to the fully-bootstrapped Tor; + this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol + <TorControlProtocol>` instance as .protocol. + """ + from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState + from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor + + def bootstrap(ctrl): + """ + Launch a Tor process with the TorConfig instance returned from + initialize(). + """ + conf = TorConfig(ctrl) + conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) + log.msg("Tor process connected, bootstrapping ...") + + def reconf_controller(conf, bridge): + ## if bridges and relays, use one bridge then build a circuit + ## from three relays + conf.Bridge = bridge + ## XXX do we need a SIGHUP to restart? + + ## XXX see txtorcon.TorControlProtocol.add_event_listener we + ## may not need full CustomCircuit class + + ## if bridges only, try one bridge at a time, but don't build + ## circuits, just return + ## if relays only, build circuits from relays + + def reconf_fail(args): + log.msg("Reconfiguring Tor config with args %s failed" % args) + reactor.stop() + + def setup_fail(args): + log.msg("Setup Failed.") + report.update({'failed': args}) + reactor.stop() + + def setup_done(proto): + log.msg("Setup Complete: %s" % proto) + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) + report.update({'success': args}) + + def updates(prog, tag, summary): + log.msg("%d%%: %s" % (prog, summary)) + + if len(args) == 0: + log.msg("Bridget needs lists of bridges and/or relays to test!") + log.msg("Exiting ...") + d = sys.exit() + return d + + else: + if len(self.bridge_list) >= 1: + for bridge in self.bridge_list: + try: + print "BRIDGE IS %s" % bridge + reconf_controller(self.config, bridge) + except: + reconf_fail(bridge) + + log.msg("Bridget: initiating test ... ") + log.msg("Using the following as our torrc:\n%s" + % self.config.create_torrc()) + report = {'tor_config': self.config.config} + log.msg("Starting Tor ...") + + ## :return: a Deferred which callbacks with a TorProcessProtocol + ## connected to the fully-bootstrapped Tor; this has a + ## txtorcon.TorControlProtocol instance as .protocol. + d = launch_tor(self.config, + reactor, + progress_updates=updates, + tor_binary=self.tor_binary) + d.addCallback(bootstrap, self.config) + d.addErrback(setup_fail) + ## now build circuits + + #print "Tor process ID: %s" % d.transport.pid + return d + +## So that getPlugins() can register the Test: +bridget = BridgetTest(None, None, None) + +## ISIS' NOTES +## ----------- +## self.config.save() only needs to be called if Tor is already running. +## +## need to add transport type to torrc Bridge line: +## Bridge <transport> IP:ORPort <fingerprint> +## +## TODO: +## o add option for any kwarg=arg self.config setting +## o cleanup documentation +## o check if bridges are public relays +## o take bridge_desc file as input, also be able to give same +## format as output +## o change the stupid name +## +## FIX: +## data directory is not found, or permissions aren't right

1 0

[ooni-probe/master] Implement the first Test Template based on the new API.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit f358389ab248fdd4d6ced99e9e2b9f6265cdd38b Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 14:13:54 2012 +0000 Implement the first Test Template based on the new API. * Fix some bugs --- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 10 ++-- ooni/plugoo/work.py | 2 - ooni/runner.py | 8 ++- ooni/templates/http.py | 147 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 11 deletions(-) diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index b48ffdf..1bfdac0 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -4,7 +4,7 @@ import os import yaml import itertools -from utils import log, date, net +from ooni.utils import log, date, net class Report: """This is the ooni-probe reporting mechanism. It allows diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 75c23f7..21ac7bf 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -8,10 +8,10 @@ from twisted.internet import reactor, defer, threads ## XXX why is this imported and not used? from twisted.python import failure -from utils import log, date -from plugoo import assets, work -from plugoo.reports import Report -from plugoo.interface import ITest +from ooni.utils import log, date +from ooni.plugoo import assets, work +from ooni.plugoo.reports import Report +from ooni.plugoo.interface import ITest class OONITest(object): @@ -58,7 +58,7 @@ class OONITest(object): return {} def __repr__(self): - return "<OONITest %s %s %s>" % (self.local_options, + return "<OONITest %s %s %s>" % (self.local_options, self.global_options, self.assets) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index e32d76d..db88fbf 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -19,8 +19,6 @@ from zope.interface import Interface, Attribute from twisted.python import failure from twisted.internet import reactor, defer -from plugoo.nodes import LocalNode - class Worker(object): """ This is the core of OONI. It takes as input Work Units and diff --git a/ooni/runner.py b/ooni/runner.py index ff6b47e..1a9b4ad 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -5,7 +5,8 @@ import time import inspect from twisted.internet import defer, reactor -from twisted.python import reflect, log, failure +from twisted.python import reflect, failure +from twisted.python import log from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader from twisted.trial.runner import isPackage, isTestCase, ErrorHolder @@ -15,7 +16,7 @@ from ooni.reporter import ReporterFactory from ooni.input import InputUnitFactory from ooni.nettest import InputTestSuite from ooni import nettest -from ooni.utils import log +#from ooni.utils import log from ooni.plugoo import tests as oonitests def isTestCase(thing): @@ -182,7 +183,8 @@ class ORunner(object): result.done() def run(self): - log.start() + log.startLogging(sys.stdout) + #log.start(True) self.reporterFactory.writeHeader() for inputUnit in InputUnitFactory(self.inputs): diff --git a/ooni/templates/__init__.py b/ooni/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ooni/templates/http.py b/ooni/templates/http.py new file mode 100644 index 0000000..222e44c --- /dev/null +++ b/ooni/templates/http.py @@ -0,0 +1,147 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import random + +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer +from twisted.web.http_headers import Headers + +from ooni.nettest import TestCase +from ooni.utils import log + +useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + +class BodyReceiver(protocol.Protocol): + def __init__(self, finished): + self.finished = finished + self.data = "" + + def dataReceived(self, bytes): + self.data += bytes + + def connectionLost(self, reason): + self.finished.callback(self.data) + +class HTTPTest(TestCase): + """ + A utility class for dealing with HTTP based testing. It provides methods to + be overriden for dealing with HTTP based testing. + The main functions to look at are processResponseBody and + processResponseHeader that are invoked once the headers have been received + and once the request body has been received. + """ + randomizeUA = True + followRedirects = False + + def setUp(self): + from twisted.web.client import Agent + from twisted.internet import reactor + import yaml + self.agent = Agent(reactor) + if self.followRedirects: + from twisted.web.client import RedirectAgent + self.agent = RedirectAgent(self.agent) + self.request = {} + self.response = {} + + def _processResponseBody(self, data): + self.response['body'] = data + self.report['response'] = self.response + + self.processResponseBody(data) + + def processResponseBody(self, data): + """ + This should handle all the response body smushing for getting it ready + to be passed onto the control. + + @param data: The content of the body returned. + """ + pass + + def processResponseHeaders(self, headers): + """ + This should take care of dealing with the returned HTTP headers. + + @param headers: The content of the returned headers. + """ + pass + + def processRedirect(self, location): + """ + Handle a redirection via a 3XX HTTP status code. + + @param location: the url that is being redirected to. + """ + pass + + def doRequest(self, url): + d = self.build_request(url) + def finished(data): + return data + + d.addCallback(self._cbResponse) + d.addCallback(finished) + return d + + def test_http(self): + log.msg("Running experiment") + + if self.input: + url = self.input + else: + raise Exception("No input supplied") + + self.mainDefer = self.doRequest(url) + return self.mainDefer + + def _cbResponse(self, response): + self.response['headers'] = response.headers + self.response['code'] = response.code + self.response['length'] = response.length + self.response['version'] = response.length + + if str(self.response['code']).startswith('3'): + self.processRedirect(response.headers.getRawHeaders('Location')[0]) + + self.processResponseHeaders(self.response['headers']) + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished)) + finished.addCallback(self._processResponseBody) + + return finished + + def randomize_useragent(self): + user_agent = random.choice(useragents) + self.request['headers']['User-Agent'] = [user_agent] + + def build_request(self, url, method="GET", headers=None, body=None): + self.request['method'] = method + self.request['url'] = url + self.request['headers'] = headers if headers else {} + self.request['body'] = body + if self.randomizeUA: + self.randomize_useragent() + + self.report['request'] = self.request + self.report['url'] = url + return self.agent.request(self.request['method'], self.request['url'], + Headers(self.request['headers']), + self.request['body']) +

1 0

[ooni-probe/master] * Need to fix the Deferred class decorator timeout.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 365bd5a1a631191808a110ded51f520806ab342e Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Sep 28 05:37:49 2012 +0000 * Need to fix the Deferred class decorator timeout. --- ooni/plugins/bridget.py | 20 +++++++++++--------- ooni/utils/timer.py | 31 +++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 208b4c3..538e2f3 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -282,6 +282,7 @@ class BridgetTest(OONITest): from ooni.utils.onion import start_tor, remove_public_relays from ooni.utils.onion import setup_done, setup_fail from ooni.utils.onion import CustomCircuit + from ooni.utils.timer import timeout from ooni.lib.txtorcon import TorConfig, TorState except ImportError: raise TxtorconImportError @@ -365,19 +366,19 @@ class BridgetTest(OONITest): log.err("Attaching custom circuit builder failed: %s" % state) - log.msg("Bridget: initiating test ... ") ## Start the experiment - all_of_the_bridges = self.bridges['all'] - all_of_the_relays = self.relays['all'] ## Local copy of orginal lists + log.msg("Bridget: initiating test ... ") ## Start the experiment - if self.bridges['remaining']() >= 1 and not 'Bridge' in self.config.config: + if self.bridges['remaining']() >= 1 \ + and not 'Bridge' in self.config.config: self.bridges['current'] = self.bridges['all'][0] self.config.Bridge = self.bridges['current'] - ## avoid starting several - self.config.save() ## processes + ## avoid starting several + self.config.save() ## processes assert self.config.config.has_key('Bridge'), "NO BRIDGE" - state = start_tor(reactor, self.config, self.control_port, - self.tor_binary, self.data_directory) + state = timeout(self.circuit_timeout)(start_tor( + reactor, self.config, self.control_port, + self.tor_binary, self.data_directory)) state.addCallbacks(setup_done, setup_fail) state.addCallback(remove_public_relays, self.bridges) @@ -396,7 +397,8 @@ class BridgetTest(OONITest): all = [] for bridge in self.bridges['all'][1:]: self.bridges['current'] = bridge - new = defer.Deferred() + #new = defer.Deferred() + new = defer.waitForDeferred(state) new.addCallback(reconfigure_bridge, state, self.bridges) all.append(new) diff --git a/ooni/utils/timer.py b/ooni/utils/timer.py index 8ad9b79..24074f9 100644 --- a/ooni/utils/timer.py +++ b/ooni/utils/timer.py @@ -9,28 +9,39 @@ # :copyright: copyright (c) 2012, Isis Lovecruft, The Tor Project Inc. # - class TimeoutError(Exception): """Raised when a timer runs out.""" pass -def timeout(secs, e=None): +def timeout(seconds, e=None): """ - A decorator for blocking methods to cause them to timeout. + A decorator for blocking methods to cause them to timeout. Can be used + like this: + @timeout(30) + def foo(): + for x in xrange(1000000000): + print x + or like this: + ridiculous = timeout(30)(foo) + + :param seconds: + Number of seconds to wait before raising :class:`TimeoutError`. + :param e: + Error message to pass to :class:`TimeoutError`. Default None. """ - import signal - import functools.wraps + from signal import alarm, signal, SIGALRM + from functools import wraps + def decorator(func): def _timeout(signum, frame): raise TimeoutError, e - def wrapper(*args, **kwargs): - signal.signal(signal.SIGALRM, _timeout) - signal.alarm(secs) + signal(SIGALRM, _timeout) + alarm(seconds) try: res = func(*args, **kwargs) finally: - signal.alarm(0) + alarm(0) return res - return functools.wraps(func)(wrapper) + return wraps(func)(wrapper) return decorator

1 0

[ooni-probe/master] Fix bug in tests
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit d621155c27143c6fad8743d0bfbdc15727a3b0db Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 12:06:00 2012 +0000 Fix bug in tests --- ooni/plugoo/tests.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 37e0664..4e7e094 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -30,14 +30,14 @@ class OONITest(object): ended = False def __init__(self, local_options, global_options, report, ooninet=None, - my_reactor=reactor): + reactor=reactor): # These are the options that are read through the tests suboptions self.local_options = local_options # These are the options global to all of OONI self.global_options = global_options self.report = report #self.ooninet = ooninet - self.reactor = my_reactor + self.reactor = reactor self.result = {} self.initialize()

1 0

[ooni-probe/master] Add note about backward compatibility.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 7c645d40702c21c45059a01c67fe67150bd109d5 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 12:01:39 2012 +0000 Add note about backward compatibility. --- docs/source/writing_tests.rst | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diff --git a/docs/source/writing_tests.rst b/docs/source/writing_tests.rst index a18b096..3d58dab 100644 --- a/docs/source/writing_tests.rst +++ b/docs/source/writing_tests.rst @@ -74,3 +74,11 @@ To access the current input you can use the ``input`` attribute, for example:: This will at each iteration over the list of inputs do something with the input. +Backward compatibility +---------------------- + +All ooni tests written using the experiment(), control() pattern are supported, +but all new tests should no longer be written using such pattern. + +Code in protocols should be refactored to follow the new API. +

1 0

[ooni-probe/master] Add an example of using the HTTP test template.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 8e8f3502944295b019f919e703c2f1396153b45e Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 14:14:41 2012 +0000 Add an example of using the HTTP test template. --- nettests/example_http.py | 28 ++++++++++++++++++++++++++++ 1 files changed, 28 insertions(+), 0 deletions(-) diff --git a/nettests/example_http.py b/nettests/example_http.py new file mode 100644 index 0000000..7528d48 --- /dev/null +++ b/nettests/example_http.py @@ -0,0 +1,28 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni.templates import http +class Example(http.HTTPTest): + inputs = ['http://google.com/', 'http://wikileaks.org/', + 'http://torproject.org/'] + + def processResponseBody(self, body): + # XXX here shall go your logic + # for processing the body + if 'blocked' in body: + self.report['censored'] = True + else: + self.report['censored'] = False + + def processResponseHeaders(self, headers): + # XXX place in here all the logic for handling the processing of HTTP + # Headers. + if headers.hasHeader('location'): + self.report['redirect'] = True + + server = headers.getRawHeaders("Server") + if server: + self.report['http_server'] = str(server.pop()) +

1 0