commit efc1206e8a099e699996c3a45742d4f7500a714f Author: Isis Lovecruft <isis@torproject.org> Date: Thu Sep 27 01:19:49 2012 +0000 * Moved bootstrap and remove_public_relay functions to Tor utilities file. * Still sorting out the chainDeferred(defer.DeferredList(list)) problems. --- ooni/plugins/bridget.py | 136 +++++++++++++++++------------------------------ ooni/utils/onion.py | 107 ++++++++++++++++++++++++++----------- 2 files changed, 123 insertions(+), 120 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 99a7d05..c749b90 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -279,8 +279,9 @@ class BridgetTest(OONITest): in Bridget it doesn't, so it should be ignored and avoided. """ try: - from ooni.utils.process import singleton_semaphore - from ooni.utils.onion import start_tor, setup_done, setup_fail + from ooni.utils import process + from ooni.utils.onion import start_tor, remove_public_relays + from ooni.utils.onion import setup_done, setup_fail from ooni.utils.onion import CustomCircuit from ooni.lib.txtorcon import TorConfig, TorState except ImportError: @@ -289,18 +290,6 @@ class BridgetTest(OONITest): log.err(tie) sys.exit() - ''' - ## XXX qu'est-que fuck? ou est utiliser ce fonction? - def bootstrap(ctrl): - """ - Launch a Tor process with the TorConfig instance returned from - initialize() and write_torrc(). - """ - conf = TorConfig(ctrl) - conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) - log.msg("Tor process connected, bootstrapping ...") - ''' - @defer.inlineCallbacks def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): """ @@ -312,7 +301,7 @@ class BridgetTest(OONITest): log.msg("Current Bridge: %s" % bridge) try: if use_pt is False: - reset_tor = yield state.protocol.set_conf('Bridge', + reset_tor = yield state.protocol.set_conf('Bridge', bridge) elif use_pt and pt_type is not None: reset_tor = yield state.protocol.set_conf( @@ -331,62 +320,29 @@ class BridgetTest(OONITest): #if not controller_response: # defer.returnValue((state.callback, None)) #else: - # defer.returnValue((state.callback, controller_response)) + # defer.returnValue((state.callback, controller_response)) if controller_response == 'OK': defer.returnValue((state, controller_response)) else: log.msg("TorControlProtocol responded with error:\n%s" % controller_response) defer.returnValue((state.callback, None)) - + except Exception, e: log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s" % (bridge, e)) defer.returnValue((state.callback, e)) - def reconfigure_done(response, bridge, reachable): + def reconfigure_done(state, bridge, reachable): log.msg("Reconfiguring with 'Bridge %s' successful" % bridge) reachable.append(bridge) + return state - def reconfigure_fail(response, bridge, unreachable): + def reconfigure_fail(state, bridge, unreachable): log.msg("Reconfiguring TorConfig with parameters %s failed" % state) unreachable.append(bridge) - - @defer.inlineCallbacks - def remove_public_relays(state, bridges): - """ - Remove bridges from our bridge list which are also listed as - public relays. - """ - IPs = map(lambda addr: addr.split(':',1)[0], bridges) - both = set(state.routers.values()).intersection(IPs) - - def __remove_line__(node, bridges=bridges): - for line in bridges: - if line.startswith(node): - try: - log.msg("Removing %s because it is a public relay" - % node) - bridges.remove(line) - except ValueError, ve: - log.debug(ve) - - if len(both) > 0: - try: - updated = yield map(lambda node: - __remove_line__(node), both) - if not updated: - ## XXX do these need to be state.callback? - defer.returnValue(state) - else: - defer.returnValue(state) - except Exception, e: - log.msg("Removing public relays from bridge list failed:\n%s" - % both) - log.err(e) - except ValueError, ve: - log.err(ve) + return state def attacher_extend_circuit(attacher, deferred, router): ## XXX todo write me @@ -417,54 +373,58 @@ class BridgetTest(OONITest): log.err("Attaching custom circuit builder failed: %s" % state) + ## Start the experiment log.msg("Bridget: initiating test ... ") - x = defer.Deferred - - if self.bridges_remaining() > 0 and not 'Bridge' in self.config.config: - self.config.Bridge = self.bridges.pop() - - ## Necessary for avoiding starting several processes: - self.config.save() + all_of_the_bridges = self.bridges + all_of_the_relays = self.relays ## Local copy of orginal lists + + if self.bridges_remaining() >= 1 and not 'Bridge' in self.config.config: + ## XXX we should do self.bridges[0] + self.bridges[1:] + initial_bridge = all_of_the_bridges.pop() + self.config.Bridge = initial_bridge + self.config.save() ## avoid starting several processes assert self.config.config.has_key('Bridge'), "NO BRIDGE" - tor = start_tor(self.reactor, - self.config, - self.control_port, - self.tor_binary, - self.data_directory).addCallback( - setup_done).addErrback( - setup_fail) - self.tor_process_semaphore = True + state = start_tor(self.reactor, self.config, + self.control_port, self.tor_binary, + self.data_directory).addCallbacks( + setup_done, + errback=setup_fail) + state.addCallback(remove_public_relays, + self.bridges) - run_once = x().addCallback(singleton_semaphore, tor) - run_once.addErrback(setup_fail) + #controller = singleton_semaphore(bootstrap) + #controller = x().addCallback(singleton_semaphore, tor) + #controller.addErrback(setup_fail) - filter_bridges = x().addCallback(remove_public_relays, self.bridges) + #filter_bridges = remove_public_relays(self.bridges) - state = defer.gatherResults([run_once, filter_bridges], consumeErrors=True) + #bootstrap = defer.gatherResults([controller, filter_bridges], + # consumeErrors=True) log.debug("Current callbacks on TorState():\n%s" % state.callbacks) + log.debug("TorState():\n%s" % state) if self.bridges_remaining() > 0: all = [] for bridge in self.bridges: - self.current_bridge = bridge - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - reconf = x().addCallback(reconfigure_bridge, state, - self.current_bridge, - self.use_pt, - self.pt_type) - reconf.addCallback(reconfigure_done, self.current_bridge, - self.bridges_up) - reconf.addErrback(reconfigure_fail, self.current_bridge, - self.bridges_down) - all.append(reconf) + #self.current_bridge = bridge + new = defer.Deferred() + new.addCallback(reconfigure_bridge, state, bridge, + self.bridges_remaining(), + self.bridges_up, + self.bridges_down, + use_pt=self.use_pt, + pt_type=self.pt_type) + all.append(new) #state.chainDeferred(defer.DeferredList(all)) #state.chainDeferred(defer.gatherResults(all, consumeErrors=True)) - n_plus_one_bridges = defer.gatherResults(all, consumeErrors=True) - state.chainDeferred(n_plus_one_bridges) - log.debug("Current callbacks on TorState():\n%s" % state.callbacks) + check_remaining = defer.DeferredList(all, consumeErrors=True) + + #controller.chainDeferred(check_remaining) + #log.debug("Current callbacks on TorState():\n%s" + # % controller.callbacks) + state.chainDeferred(check_remaining) if self.relays_remaining() > 0: while self.relays_remaining() >= 3: diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 81e4ea3..82f013f 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -13,6 +13,7 @@ # :copyright: copyright (c) 2012 The Tor Project, Inc. # :version: 0.1.0-alpha # +# XXX TODO add report keys for onion methods import random @@ -31,25 +32,32 @@ def setup_done(proto): def setup_fail(proto): log.err("Setup Failed: %s" % proto) - report.update({'setup_fail': proto}) + #report.update({'setup_fail': proto}) reactor.stop() def state_complete(state): """Called when we've got a TorState.""" log.msg("We've completely booted up a Tor version %s at PID %d" % (state.protocol.version, state.tor_pid)) - log.msg("This Tor has the following %d Circuits:" % len(state.circuits)) for circ in state.circuits.values(): log.msg("%s" % circ) - - #return state + return state def updates(_progress, _tag, _summary): """Log updates on the Tor bootstrapping process.""" log.msg("%d%%: %s" % (_progress, _summary)) +def bootstrap(ctrl): + """ + Bootstrap Tor from an instance of + :class:`ooni.lib.txtorcon.TorControlProtocol`. + """ + conf = TorConfig(ctrl) + conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) + log.msg("Tor process connected, bootstrapping ...") + def parse_data_dir(data_dir): """ Parse a string that a has been given as a DataDirectory and determine @@ -124,7 +132,46 @@ def delete_files_or_dirs(delete_list): except OSError: rmtree(temp, ignore_errors=True) +def remove_node_from_list(node, list): + for item in list: ## bridges don't match completely + if item.startswith(node): ## due to the :<port>. + try: + log.msg("Removing %s because it is a public relay" % node) + list.remove(line) + except ValueError, ve: + log.err(ve) + @defer.inlineCallbacks +def remove_public_relays(state, bridges): + """ + Remove bridges from our bridge list which are also listed as public + relays. This must be called after Tor has fully bootstrapped and we have a + :class:`ooni.lib.txtorcon.TorState` with the + :attr:`ooni.lib.txtorcon.TorState.routers` attribute assigned. + + XXX Does state.router.values() have all of the relays in the consensus, or + just the ones we know about so far? + """ + IPs = map(lambda addr: addr.split(':',1)[0], bridges) + both = set(state.routers.values()).intersection(IPs) + + if len(both) > 0: + try: + updated = yield map(lambda node: remove_node_from_list(node), + both) + if not updated: + ## XXX do these need to be state.callback? + defer.returnValue(state) + else: + defer.returnValue(state) + except Exception, e: + log.msg("Removing public relays from bridge list failed:\n%s" + % both) + log.err(e) + except ValueError, ve: + log.err(ve) + +#@defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, report=None, progress=updates, process_cb=setup_done, process_eb=setup_fail): @@ -193,39 +240,35 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, reactor.addSystemEventTrigger('before', 'shutdown', partial(delete_files_or_dirs, to_delete)) + #try: + # transport = yield reactor.spawnProcess(process_protocol, + # tor_binary, + # args=(tor_binary,'-f',torrc), + # env={'HOME': data_dir}, + # path=data_dir) + # if transport: + # transport.closeStdin() + #except RuntimeError as e: + # log.err("Starting Tor failed: %s" % e) + # process_protocol.connected_cb.errback(e) + #except NotImplementedError, e: + # url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" + # log.err("Running bridget on Windows requires pywin32: %s" % url) + # process_protocol.connected_cb.errback(e) try: - transport = yield reactor.spawnProcess(process_protocol, - tor_binary, - args=(tor_binary,'-f',torrc), - env={'HOME': data_dir}, - path=data_dir) - if transport: - transport.closeStdin() - except RuntimeError as e: - log.err("Starting Tor failed: %s" % e) - process_protocol.connected_cb.errback(e) - except NotImplementedError, e: - url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" - log.err("Running bridget on Windows requires pywin32: %s" % url) + transport = reactor.spawnProcess(process_protocol, + tor_binary, + args=(tor_binary,'-f',torrc), + path=data_dir) + transport.closeStdin() + except RuntimeError, e: + log.err(e) process_protocol.connected_cb.errback(e) - #proc_proto = process_protocol.connected_cb - #proc_proto.addCallback(process_cb) - #proc_proto.addErrback(process_eb) - # - #d = yield process_protocol.connected_cb.addCallback( - # process_cb).addErrback( - # process_eb) + return process_protocol.connected_cb + #d = yield process_protocol.connected_cb - #d.addCallback(process_cb) - #d.addErrback(process_eb) - # #defer.returnValue(d) - - d = yield process_protocol.connected_cb - defer.returnValue(d) - - #return process_protocol.connected_cb.addCallback(process_cb).addErrback(process_eb) class CustomCircuit(CircuitListenerMixin): implements(IStreamAttacher)