commit cafc88339b058798005766cf97fabd659f0e20f5 Author: Isis Lovecruft isis@torproject.org Date: Sun Sep 23 19:00:33 2012 +0000

* Still squashing bugs. * Fixed the log.debug() function. * Put more general things in the onion.py utility (renamed circuit.py to onion.py). * Updated the installation instructions and documentation on dependencies. * Added my name to the license. --- INSTALL | 6 +++- LICENSE | 2 +- ooni/plugins/bridget.py | 85 ++++++++++++++++++++++------------------------ ooni/utils/log.py | 4 ++- ooni/utils/onion.py | 47 +++++++++++++++++-------- 5 files changed, 82 insertions(+), 62 deletions(-)

diff --git a/INSTALL b/INSTALL index bac1a39..d972d26 100644 --- a/INSTALL +++ b/INSTALL @@ -1,5 +1,5 @@ # Dependencies -* Twisted: http://twistedmatrix.com/trac/ +* Twisted (>12.0.0): http://twistedmatrix.com/trac/ * PyYAML: http://pyyaml.org/ * Scapy: http://www.secdev.org/projects/scapy/ * pypcap: http://code.google.com/p/pypcap/ @@ -7,6 +7,10 @@

Optional: * BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +* Dnspython[1]: http://www.dnspython.org/ +* Paramiko[2]: http://www.lag.net/paramiko/ + +[1][2] These dependencies will soon be removed completely.

# Installation

diff --git a/LICENSE b/LICENSE index f1a9745..a0c3c18 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012, Jacob Appelbaum, Arturo Filastò +Copyright (c) 2012, Jacob Appelbaum, Arturo Filastò, Isis Lovecruft All rights reserved.

Redistribution and use in source and binary forms, with or without diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index c4667e1..cd7ba4e 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -189,7 +189,7 @@ class BridgetTest(OONITest): implements(IPlugin, ITest)

shortName = "bridget" - description = "Use a Tor process to test connecting to bridges and relays" + description = "Use a Tor process to test connecting to bridges or relays" requirements = None options = BridgetArgs blocking = False @@ -218,7 +218,7 @@ class BridgetTest(OONITest): self.relays_down_count = lambda: len(self.relays_down) self.current_relay = None

- def make_asset_list(opt, lst): + def __make_asset_list__(opt, lst): log.msg("Loading information from %s ..." % opt) with open(opt) as opt_file: for line in opt_file.readlines(): @@ -238,12 +238,12 @@ class BridgetTest(OONITest):

if options['bridges']: self.config.UseBridges = 1 - make_asset_list(options['bridges'], self.bridges) + __make_asset_list__(options['bridges'], self.bridges)

if options['relays']: - ## first hop must be in TorState().entry_guards to build circuits + ## first hop must be in TorState().guards to build circuits self.config.EntryNodes = ','.join(relay_list) - make_asset_list(options['relays'], self.relays) + __make_asset_list__(options['relays'], self.relays)

if options['socks']: self.socks_port = options['socks'] @@ -279,6 +279,7 @@ class BridgetTest(OONITest): self.config.ControlPort = self.control_port self.config.CookieAuthentication = 1

+ ''' def load_assets(self): """ Load bridges and/or relays from files given in user options. Bridges @@ -294,6 +295,7 @@ class BridgetTest(OONITest): assets.update({'relay': BridgetAsset(self.local_options['relays'])}) return assets + '''

def experiment(self, args): """ @@ -353,6 +355,7 @@ class BridgetTest(OONITest): log.err(tie) sys.exit()

+ ## XXX qu'est-que fuck? ou est utiliser ce fonction? def bootstrap(ctrl): """ Launch a Tor process with the TorConfig instance returned from @@ -388,19 +391,19 @@ class BridgetTest(OONITest): #else: # defer.returnValue((state.callback, controller_response)) if controller_response == 'OK': - defer.returnValue(state.callback) + defer.returnValue((state.callback, controller_response)) else: - log.msg("TorControlProtocol responded with error:\n%s", - controller_response) - defer.returnValue(state.callback) + log.msg("TorControlProtocol responded with error:\n%s" + % controller_response) + defer.returnValue((state.callback, None))

except Exception, e: - log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s", - bridge, e) + log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s" + % (bridge, e))

def reconfigure_fail(state, bridge, bad): - log.msg("Reconfiguring TorConfig with parameters %s failed", - state) + log.msg("Reconfiguring TorConfig with parameters %s failed" + % state) bad.append(bridge)

@defer.inlineCallbacks @@ -416,8 +419,8 @@ class BridgetTest(OONITest): for line in bridges: if line.startswith(node): try: - log.msg("Removing %s because it is a public relay", - node) + log.msg("Removing %s because it is a public relay" + % node) bridges.remove(line) except ValueError, ve: log.err(ve) @@ -432,8 +435,8 @@ class BridgetTest(OONITest): else: defer.returnValue(state) except Exception, e: - log.msg("Removing public relays from bridge list failed:\n%s", - both) + log.msg("Removing public relays from bridge list failed:\n%s" + % both) log.err(e) except ValueError, ve: log.err(ve) @@ -444,22 +447,22 @@ class BridgetTest(OONITest): raise NotImplemented #attacher.extend_circuit

- def state_complete(state, bridge_list=None, relay_list=None): - """Called when we've got a TorState.""" - log.msg("We've completely booted up a Tor version %s at PID %d" - % (state.protocol.version, state.tor_pid)) - - log.msg("This Tor has the following %d Circuits:" - % len(state.circuits)) - for circ in state.circuits.values(): - log.msg("%s" % circ) - - if bridge_list is not None and relay_list is None: - return state, bridge_list - elif bridge_list is None and relay_list is not None: - raise NotImplemented - else: - return state, None + #def state_complete(state, bridge_list=None, relay_list=None): + # """Called when we've got a TorState.""" + # log.msg("We've completely booted up a Tor version %s at PID %d" + # % (state.protocol.version, state.tor_pid)) + # + # log.msg("This Tor has the following %d Circuits:" + # % len(state.circuits)) + # for circ in state.circuits.values(): + # log.msg("%s" % circ) + # + # if bridge_list is not None and relay_list is None: + # return state, bridge_list + # elif bridge_list is None and relay_list is not None: + # raise NotImplemented + # else: + # return state, None

def state_attach(state, path): log.msg("Setting up custom circuit builder...") @@ -481,7 +484,7 @@ class BridgetTest(OONITest): return d

def state_attach_fail(state): - log.err("Attaching custom circuit builder failed: %s", state) + log.err("Attaching custom circuit builder failed: %s" % state)

log.msg("Bridget: initiating test ... ") @@ -491,16 +494,13 @@ class BridgetTest(OONITest): #self.current_bridge = bridge

if not self.config.config.has_key('Bridge'): - self.config.Bridge = self.current_bridge - - state = start_tor(reactor, + state = start_tor(self.reactor, self.config, self.control_port, self.tor_binary, self.data_directory) state.addCallback(remove_public_relays, self.bridges) - else: log.msg("We now have %d untested bridges..." % self.bridges_remaining()) @@ -511,7 +511,7 @@ class BridgetTest(OONITest): reconf.addErrback(reconfigure_fail, state, self.current_bridge, self.bridges_down) state.chainDeferred(reconf) - state.callback() + state.callback(controller_response) #all = [] #reconf = reconfigure_bridge(state, self.current_bridge, # self.use_pt, self.pt_type) @@ -570,17 +570,14 @@ class BridgetTest(OONITest): # % self.current_bridge) # self.bridges_up.append(self.current_bridge)

- reactor.run() - + #reactor.run()

## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None)

+ ## ISIS' NOTES ## ----------- -## nickm points out that the format operator '%', when used in log.LEVEL(), -## forces string generation even when LEVEL is not used, increasing overhead; -## we should do 'log.err("string with stuff %s", stuff)' instead. ## ## TODO: ## o cleanup documentation diff --git a/ooni/utils/log.py b/ooni/utils/log.py index dd5cf13..54c59ea 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -88,7 +88,9 @@ def start(logstdout, logfile=None, verbosity=None): log.msg("Starting OONI...")

def debug(message, level="debug", **kw): - log.msg(message, logLevel=level, **kw) + print "[%s] %s" % (level, message) + ## If we want debug messages in the logfile: + #log.msg(message, logLevel=level, **kw)

def msg(message, level="info", **kw): log.msg(message, logLevel=level, **kw) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 76ec909..46373b4 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -24,13 +24,31 @@ from zope.interface import implements def __setup_done__(proto): log.msg("Setup Complete: %s" % proto) state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallback(state_complete).addErrback(__setup_fail__) + state.post_bootstrap.addCallback(__state_complete__) + state.post_bootstrap.addErrback(__setup_fail__)

def __setup_fail__(proto): log.err("Setup Failed: %s" % proto) report.update({'setup_fail': proto}) reactor.stop()

+def __state_complete__(state, bridge_list=None, relay_list=None): + """Called when we've got a TorState.""" + log.msg("We've completely booted up a Tor version %s at PID %d" + % (state.protocol.version, state.tor_pid)) + + log.msg("This Tor has the following %d Circuits:" + % len(state.circuits)) + for circ in state.circuits.values(): + log.msg("%s" % circ) + + if bridge_list is not None and relay_list is None: + return state, bridge_list + elif bridge_list is None and relay_list is not None: + raise NotImplemented + else: + return state, None + def __updates__(_progress, _tag, _summary): log.msg("%d%%: %s", _progress, _summary)

@@ -134,7 +152,7 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, (torrc, data_dir, to_delete) = write_torrc(config, data_dir)

log.msg("Starting Tor ...") - log.msg("Using the following as our torrc:\n%s", config.create_torrc()) + log.msg("Using the following as our torrc:\n%s" % config.create_torrc()) if report is None: report = {'torrc': config.create_torrc()} else: @@ -144,6 +162,8 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, connection_creator = partial(end_point.connect, TorProtocolFactory()) process_protocol = TorProcessProtocol(connection_creator, progress) process_protocol.to_delete = to_delete + process_protocol.addCallback(process_cb) + process_protocol.addErrback(process_eb)

reactor.addSystemEventTrigger('before', 'shutdown', partial(delete_files_or_dirs, to_delete)) @@ -154,18 +174,15 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, env={'HOME': data_dir}, path=data_dir) transport.closeStdin() - except RuntimeError, e: - log.err("Starting Tor failed: %s", e) + except RuntimeError as e: + log.err("Starting Tor failed: %s" % e) process_protocol.connected_cb.errback(e) except NotImplementedError, e: url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" - log.err("Running bridget on Windows requires pywin32: %s", url) + log.err("Running bridget on Windows requires pywin32: %s" % url) process_protocol.connected_cb.errback(e)

- proto = process_protocol.connected_cb ## new defer.Deferred() - proto.addCallback(process_cb) - proto.addErrback(process_eb) - return proto + return process_protocol.connected_cb ## new defer.Deferred()

class CustomCircuit(CircuitListenerMixin): @@ -187,14 +204,14 @@ class CustomCircuit(CircuitListenerMixin): if circuit.purpose != 'GENERAL': return if self.waiting_on(circuit): - log.msg("Circuit %d (%s)", circuit.id, router.id_hex) + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex))

def circuit_built(self, circuit): "ICircuitListener" if circuit.purpose != 'GENERAL': return - log.msg("Circuit %s built ...", circuit.id) - log.msg("Full path of %s: %s", circuit.id, circuit.path) + log.msg("Circuit %s built ..." % circuit.id) + log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) for (circid, d) in self.waiting_circuits: if circid == circuit.id: self.waiting_circuits.remove((circid, d)) @@ -202,7 +219,7 @@ class CustomCircuit(CircuitListenerMixin):

def circuit_failed(self, circuit, reason): if self.waiting_on(circuit): - log.msg("Circuit %s failed for reason %s", circuit.id, reason) + log.msg("Circuit %s failed for reason %s" % (circuit.id, reason)) circid, d = None, None for c in self.waiting_circuits: if c[0] == circuit.id: @@ -211,7 +228,7 @@ class CustomCircuit(CircuitListenerMixin): raise Exception("Expected to find circuit.")

self.waiting_circuits.remove((circid, d)) - log.msg("Trying to build a circuit for %s", circid) + log.msg("Trying to build a circuit for %s" % circid) self.request_circuit_build(d)

def check_circuit_route(self, router): @@ -252,7 +269,7 @@ class CustomCircuit(CircuitListenerMixin): issue an attach on it and callback to the Deferred we issue here. """ - log.msg("Circuit %s is in progress ...", circ.id) + log.msg("Circuit %s is in progress ..." % circ.id) self.attacher.waiting_circuits.append((circ.id, self.d))

return self.state.build_circuit(path).addCallback(