- tor-commits - lists.torproject.org

[ooni-probe/master] Write some more docstrings and fix some code.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 26f7afc073cb86276a04d1faeec65dde59dd858f Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 21 12:05:17 2012 +0000 Write some more docstrings and fix some code. --- ooni/protocols/http.py | 4 ++-- ooni/reporter.py | 6 ++++-- ooni/runner.py | 23 +++++++++++++++++++++-- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py index 09bb9b9..569c382 100644 --- a/ooni/protocols/http.py +++ b/ooni/protocols/http.py @@ -56,7 +56,7 @@ class HTTPTest(OONITest): def _processResponseBody(self, data): self.response['body'] = data - #self.result['response'] = self.response + self.result['response'] = self.response self.processResponseBody(data) def processResponseBody(self, data): @@ -127,7 +127,7 @@ class HTTPTest(OONITest): if self.randomize_ua: self.randomize_useragent() - #self.result['request'] = self.request + self.result['request'] = self.request self.result['url'] = url return self.agent.request(self.request['method'], self.request['url'], Headers(self.request['headers']), diff --git a/ooni/reporter.py b/ooni/reporter.py index 4b3ed6f..b145e87 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -138,12 +138,14 @@ class OONIReporter(OReporter): # This is here for allowing reporting of legacy tests. # XXX In the future this should be removed. try: - self._tests[idx]['report'] = list(test.legacy_report) + report = list(test.legacy_report) except: # XXX I put a dict() here so that the object is re-instantiated and I # actually end up with the report I want. This could either be a # python bug or a yaml bug. - self._tests[idx]['report'] = dict(test.report) + report = dict(test.report) + + self._tests[idx]['report'] = report def done(self): diff --git a/ooni/runner.py b/ooni/runner.py index 0bcafb0..ff6b47e 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -85,13 +85,18 @@ def adaptLegacyTest(obj, config): args = {} args[self.key] = self.input result = yield my_test.startTest(args) - print "Finished!" - print result + self.report['result'] = result return LegacyOONITest def findTestClassesFromConfig(config): + """ + Takes as input the command line config parameters and returns the test + case classes. + If it detects that a certain test class is using the old OONIProbe format, + then it will adapt it to the new testing system. + """ filename = config['test'] classes = [] @@ -105,12 +110,20 @@ def findTestClassesFromConfig(config): return classes def makeTestCases(klass, tests, methodPrefix): + """ + Takes a class some tests and returns the test cases. methodPrefix is how + the test case functions should be prefixed with. + """ cases = [] for test in tests: cases.append(klass(methodPrefix+test)) return cases def loadTestsAndOptions(classes): + """ + Takes a list of classes and returnes their testcases and options. + Legacy tests will be adapted. + """ methodPrefix = 'test' suiteFactory = InputTestSuite options = [] @@ -132,6 +145,12 @@ def loadTestsAndOptions(classes): return testCases, options class ORunner(object): + """ + This is a specialized runner used by the ooniprobe command line tool. + I am responsible for reading the inputs from the test files and splitting + them in input units. I also create all the report instances required to run + the tests. + """ def __init__(self, cases, options=None, config=None): self.baseSuite = InputTestSuite self.cases = cases

1 0

[ooni-probe/master] Document TestCase
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 839564d3620c24e0057c1dfd9d39ef15f8b202ac Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 21 13:07:15 2012 +0000 Document TestCase --- ooni/nettest.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 53 insertions(+), 1 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index f3476cf..2a02a43 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -24,11 +24,63 @@ class InputTestSuite(pyunit.TestSuite): self._idx += 1 return result +def lineByLine(fp): + for x in fp.readlines(): + yield x.strip() + fp.close() + class TestCase(unittest.TestCase): - name = "DefaultOONITestCase" + """ + This is the monad of the OONI nettest universe. When you write a nettest + you will subclass this object. + + _inputs_ can be set to a static set of inputs. All the tests (the methods + starting with the "test_" prefix) will be run once per input. At every run + the _input_ attribute of the TestCase instance will be set to the value of + the current iteration over inputs. Any python iterable object can be set + to inputs. + + _inputFile_ attribute should be set to an array containing the command line + argument that should be used as the input file. Such array looks like this: + + ["commandlinearg", "c", "The description"] + + The second value of such arrray is the shorthand for the command line arg. + The user will then be able to specify inputs to the test via: + + ooniprobe mytest.py --commandlinearg path/to/file.txt + + or + + ooniprobe mytest.py -c path/to/file.txt + + + _inputProcessor_ should be set to a function that takes as argument an + open file descriptor and it will yield the input to be passed to the test + instance. + + _name_ should be set to the name of the test. + + _author_ should contain the name and contact details for the test author. + The format for such string is as follows: + + The Name <email(a)example.com> + + _version_ is the version string of the test. + """ + name = "IDidNotChangeTheName" + author = "John Doe <foo(a)example.com>" + version = "0" + inputs = [None] + inputFile = None + inputProcessor = lineByLine def getOptions(self): + if self.inputFile: + fp = open(self.inputFile) + self.inputs = inputProcessor(fp) + return {'inputs': self.inputs} def __repr__(self):

1 0

[ooni-probe/master] * Added documentation.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit fa731c54c4c28764a345523ca3ab632e81fad884 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Sep 23 12:58:21 2012 +0000 * Added documentation. * Moved general setup, circuit, and bootstrap, to utils/onion.py. --- ooni/plugins/bridget.py | 323 +++++++++++++++++++---------------------------- ooni/utils/circuit.py | 109 ---------------- ooni/utils/onion.py | 260 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 389 insertions(+), 303 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index d82f2e7..c4667e1 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -13,7 +13,6 @@ # :version: 0.1.0-alpha from __future__ import with_statement -from functools import wraps, partial from random import randint from twisted.python import usage from twisted.plugin import IPlugin @@ -25,11 +24,12 @@ from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset import os -import signal import sys def timer(secs, e=None): + import signal + import functools.wraps def decorator(func): def _timer(signum, frame): raise TimeoutError, e @@ -41,7 +41,7 @@ def timer(secs, e=None): finally: signal.alarm(0) return res - return wraps(func)(wrapper) + return functools.wraps(func)(wrapper) return decorator @@ -297,27 +297,60 @@ class BridgetTest(OONITest): def experiment(self, args): """ - XXX fill me in + We cannot use the Asset model, because that model calls + self.experiment() with the current Assets, which would be one relay + and one bridge, then it gives the defer.Deferred returned from + self.experiment() to self.control(), which means that, for each + (bridge, relay) pair, experiment gets called again, which instantiates + an additional Tor process that attempts to bind to the same + ports. Thus, additionally instantiated Tor processes return with + RuntimeErrors, which break the final defer.chainDeferred.callback(), + sending it into the errback chain. + + if bridges: + 1. configure first bridge line + 2a. configure data_dir, if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor } if any of these + 4. remove bridges which are public relays } fail, add current + 5. SIGHUP for each bridge } bridge to unreach- + } able bridges. + if relays: + 1a. configure the data_dir, if it doesn't exist + 1b. write torrc to a tempfile in data_dir + 2. start tor + 3. remove any of our relays which are already part of current + circuits + 4a. attach CustomCircuit() to self.state + 4b. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays + 5. + if bridges and relays: + 1. configure first bridge line + 2a. configure data_dir if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor + 4. remove bridges which are public relays + 5. remove any of our relays which are already part of current + circuits + 6a. attach CustomCircuit() to self.state + 6b. for each bridge, build three circuits, with three + relays each + 6c. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays :param args: The :class:`BridgetAsset` line currently being used. """ try: - from tempfile import mkstemp, mkdtemp - from shutil import rmtree - - from twisted.internet.endpoints import TCP4ClientEndpoint - - from ooni.utils import circuit - from ooni.lib.txtorcon import TorProcessProtocol - from ooni.lib.txtorcon import TorProtocolFactory + from ooni.utils.onion import start_tor, CustomCircuit from ooni.lib.txtorcon import TorConfig, TorState - except ImportError: raise TxtorconImportError - - except TxtorconImportError: - ## XXX is this something we should add to the reactor? + except TxtorconImportError, tie: + log.err(tie) sys.exit() def bootstrap(ctrl): @@ -329,18 +362,6 @@ class BridgetTest(OONITest): conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) log.msg("Tor process connected, bootstrapping ...") - def delete_temp(delete_list): - """ - Given a list of files or directories to delete, delete all and - suppress all errors. - """ - for temp in delete_list: - try: - os.unlink(temp) - except OSError: - rmtree(temp, ignore_errors=True) - - #(a)timer(self.circuit_timeout) @defer.inlineCallbacks def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): """ @@ -352,10 +373,11 @@ class BridgetTest(OONITest): log.msg("Current Bridge: %s" % bridge) try: if use_pt is False: - reset_tor = yield state.protocol.set_conf('Bridge', bridge) - elif use_pt and pt_type is not None: reset_tor = yield state.protocol.set_conf('Bridge', - pt_type +' '+ bridge) + bridge) + elif use_pt and pt_type is not None: + reset_tor = yield state.protocol.set_conf( + 'Bridge', pt_type +' '+ bridge) else: raise PTNotFoundException @@ -368,16 +390,17 @@ class BridgetTest(OONITest): if controller_response == 'OK': defer.returnValue(state.callback) else: - log.msg("TorControlProtocol responded with error:\n%s\n" - % controller_response) + log.msg("TorControlProtocol responded with error:\n%s", + controller_response) defer.returnValue(state.callback) except Exception, e: - log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s\n" - % (bridge, e)) + log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s", + bridge, e) def reconfigure_fail(state, bridge, bad): - log.msg("Reconfiguring TorConfig with parameters %s failed" % state) + log.msg("Reconfiguring TorConfig with parameters %s failed", + state) bad.append(bridge) @defer.inlineCallbacks @@ -392,104 +415,34 @@ class BridgetTest(OONITest): def __remove_line__(node, bridges=bridges): for line in bridges: if line.startswith(node): - log.msg("Removing %s because it is a public relay" % node) - bridges.remove(line) + try: + log.msg("Removing %s because it is a public relay", + node) + bridges.remove(line) + except ValueError, ve: + log.err(ve) if len(both) > 0: try: - updated = yield map(lambda node: __remove_line__(node), both) + updated = yield map(lambda node: + __remove_line__(node), both) if not updated: ## XXX do these need to be state.callback? defer.returnValue(state) else: defer.returnValue(state) except Exception, e: - log.msg("Unable to remove public relays from bridge list:\n%s" - % both) + log.msg("Removing public relays from bridge list failed:\n%s", + both) log.err(e) - - def request_circuit_build(state, deferred, path=None): - if path is None: - if relays_remaining() > 0: - first, middle,last = (state.relays.pop() - for i in range(3)) - else: - first = random.choice(state.entry_guards.values()) - middle, last = (random.choice(state.routers.values()) - for i in range(2)) - path = [first, middle, last] - else: - assert type(path) is list, "Circuit path must be a list of routers!" - assert len(path) >= 3, "Circuits must be at least three hops!" - - log.msg("Requesting a circuit: %s" - % '->'.join(map(lambda node: node, path))) - - class AppendWaiting: - def __init__(self, attacher, deferred): - self.attacher = attacher - self.d = deferred - def __call__(self, circ): - """ - Return from build_circuit is a Circuit, however, - we want to wait until it is built before we can - issue an attach on it and callback to the Deferred - we issue here. - """ - log.msg("Circuit %s is in progress ..." % circ.id) - self.attacher.waiting_circuits.append((circ.id, self.d)) - - return self.state.build_circuit(path).addCallback( - AppendWaiting(self, deferred)).addErrback( - log.err) + except ValueError, ve: + log.err(ve) def attacher_extend_circuit(attacher, deferred, router): - - - def setup_fail(proto, bridge_list, relay_list): - log.err("Setup Failed: %s" % proto) - log.err("We were given bridge list:\n%s\nAnd our relay list:\n%s\n" - % (bridge_list, relay_list)) - report.update({'setup_fail': 'FAILED', - 'proto': proto, - 'bridge_list': bridge_list, - 'relay_list': relay_list}) - reactor.stop() - - def setup_done(proto, bridge_list, relay_list): - log.msg("Setup Complete: %s" % proto) - state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) - if bridge_list is not None: - state.post_bootstrap.addCallback(remove_public_relays, bridge_list) - if relay_list is not None: - raise NotImplemented - #report.update({'success': args}) - - def start_tor(reactor, update, torrc, to_delete, control_port, tor_binary, - data_directory, bridge_list=None, relay_list=None): - """ - Create a TCP4ClientEndpoint at our control_port, and connect - it to our reactor and a spawned Tor process. Compare with - :meth:`txtorcon.launch_tor` for differences. - """ - end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port) - connection_creator = partial(end_point.connect, TorProtocolFactory()) - process_protocol = TorProcessProtocol(connection_creator, updates) - process_protocol.to_delete = to_delete - reactor.addSystemEventTrigger('before', 'shutdown', - partial(delete_temp, to_delete)) - try: - transport = reactor.spawnProcess(process_protocol, - tor_binary, - args=(tor_binary,'-f',torrc), - env={'HOME': data_directory}, - path=data_directory) - transport.closeStdin() - except RuntimeError, e: - process_protocol.connected_cb.errback(e) - finally: - return process_protocol.connected_cb, bridge_list, relay_list + ## XXX todo write me + ## state.attacher.extend_circuit + raise NotImplemented + #attacher.extend_circuit def state_complete(state, bridge_list=None, relay_list=None): """Called when we've got a TorState.""" @@ -508,12 +461,14 @@ class BridgetTest(OONITest): else: return state, None - def state_attach(state, relay_list): + def state_attach(state, path): log.msg("Setting up custom circuit builder...") attacher = CustomCircuit(state) state.set_attacher(attacher, reactor) state.add_circuit_listener(attacher) + return state + ## OLD #for circ in state.circuits.values(): # for relay in circ.path: # try: @@ -526,80 +481,47 @@ class BridgetTest(OONITest): return d def state_attach_fail(state): - log.msg("Attaching custom circuit builder failed.") + log.err("Attaching custom circuit builder failed: %s", state) - def updates(prog, tag, summary): - log.msg("%d%%: %s" % (prog, summary)) - - def write_torrc(conf, data_dir=None): - """ - Create a torrc in our data_directory. If we don't yet have a - data_directory, create a temporary one. Any temporary files or - folders are added to delete_list. - - :return: delete_list, data_dir, torrc - """ - delete_list = [] - - if data_dir is None: - data_dir = mkdtemp(prefix='bridget-tordata') - delete_list.append(data_dir) - conf.DataDirectory = data_dir - - (fd, torrc) = mkstemp(dir=data_dir) - delete_list.append(torrc) - os.write(fd, conf.create_torrc()) - os.close(fd) - return torrc, data_dir, delete_list - log.msg("Bridget: initiating test ... ") - while self.bridges_remaining() > 0: + if self.bridges_remaining() > 0: + for self.current_bridge in self.bridges: + #self.current_bridge = bridge + + if not self.config.config.has_key('Bridge'): - self.current_bridge = self.bridges.pop() - #self.current_bridge = args['bridge'] + self.config.Bridge = self.current_bridge - try: - self.bridges.remove(self.current_bridge) - except ValueError, ve: - log.err(ve) - - if not self.config.config.has_key('Bridge'): - self.config.Bridge = self.current_bridge - (torrc, self.data_directory, to_delete) = write_torrc( - self.config, self.data_directory) - - log.msg("Starting Tor ...") - log.msg("Using the following as our torrc:\n%s" - % self.config.create_torrc()) - report = {'tor_config': self.config.create_torrc()} - - state = start_tor(reactor, updates, torrc, to_delete, - self.control_port, - self.tor_binary, - self.data_directory, - bridge_list=self.bridges) - state.addCallback(setup_done) - state.addErrback(setup_fail) - state.addCallback(remove_public_relays, self.bridges) + state = start_tor(reactor, + self.config, + self.control_port, + self.tor_binary, + self.data_directory) + state.addCallback(remove_public_relays, self.bridges) + + else: + log.msg("We now have %d untested bridges..." + % self.bridges_remaining()) + reconf = defer.Deferred() + reconf.addCallback(reconfigure_bridge, state, + self.current_bridge, self.use_pt, + self.pt_type) + reconf.addErrback(reconfigure_fail, state, + self.current_bridge, self.bridges_down) + state.chainDeferred(reconf) + state.callback() + #all = [] + #reconf = reconfigure_bridge(state, self.current_bridge, + # self.use_pt, self.pt_type) + #reconf.addCallback(reconfigure_done) + #reconf.addErrback(reconfigure_fail) + #state.DeferredList(all) + + if self.relays_remaining() != 0: + state.chainDeferred() - else: - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - reconf = defer.Deferred() - reconf.addCallback(reconfigure_bridge, state, - self.current_bridge, self.use_pt, - self.pt_type) - reconf.addErrback(reconfigure_fail, state, - self.current_bridge, self.bridges_down) - state.chainDeferred(reconf) - #all = [] - #reconf = reconfigure_bridge(state, self.current_bridge, - # self.use_pt, self.pt_type) - #reconf.addCallback(reconfigure_done) - #reconf.addErrback(reconfigure_fail) - #state.DeferredList(all) while self.relays_remaining() >= 3: #path = list(self.relays.pop() for i in range(3)) @@ -609,10 +531,19 @@ class BridgetTest(OONITest): for node in circ.path: if node == self.current_relay: self.relays_up.append(self.current_relay) - try: - ext = attacher_extend_circuit(state.attacher, circ, - self.current_relay) - + + if len(circ.path) < 3: + try: + parameters = (state.attacher, circ, + self.current_relay) + ext = attacher_extend_circuit(parameters) + ext.addCallback(attacher_extend_circuit_done, + parameters) + except Exception, e: + log.msg("Extend circuit failed: %s %s" + % (e, parameters)) + else: + continue return state ## still need to attach attacher to state @@ -647,6 +578,10 @@ bridget = BridgetTest(None, None, None) ## ISIS' NOTES ## ----------- +## nickm points out that the format operator '%', when used in log.LEVEL(), +## forces string generation even when LEVEL is not used, increasing overhead; +## we should do 'log.err("string with stuff %s", stuff)' instead. +## ## TODO: ## o cleanup documentation ## x add DataDirectory option diff --git a/ooni/utils/circuit.py b/ooni/utils/circuit.py deleted file mode 100644 index d725879..0000000 --- a/ooni/utils/circuit.py +++ /dev/null @@ -1,109 +0,0 @@ -# -# circuit.py -# ---------- -# Utilities for working with Tor circuits. -# -# This code is largely taken from attach_streams_by_country.py in the txtorcon -# documentation, and as such any and all credit should go to meejah. Minor -# adjustments have been made to use OONI's logging system, and to build custom -# circuits without actually attaching streams. -# -# :author: Meejah, Isis Lovecruft -# :license: see included LICENSE file -# :version: 0.1.0-alpha -# - -import random - -from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher -from ooni.lib.txtorcon import TorInfo -from ooni.utils import log -from zope.interface import implements - - -class CustomCircuit(CircuitListenerMixin): - implements(IStreamAttacher) - - def __init__(self, state, relays=None): - self.state = state - self.waiting_circuits = [] - self.relays = relays - - def waiting_on(self, circuit): - for (circid, d) in self.waiting_circuits: - if circuit.id == circid: - return True - return False - - def circuit_extend(self, circuit, router): - "ICircuitListener" - if circuit.purpose != 'GENERAL': - return - if self.waiting_on(circuit): - log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) - - def circuit_built(self, circuit): - "ICircuitListener" - if circuit.purpose != 'GENERAL': - return - log.msg("Circuit %s built ..." % circuit.id) - log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) - for (circid, d) in self.waiting_circuits: - if circid == circuit.id: - self.waiting_circuits.remove((circid, d)) - d.callback(circuit) - - def circuit_failed(self, circuit, reason): - if self.waiting_on(circuit): - log.msg("Circuit %s failed for reason %s" - % (circuit.id, reason)) - circid, d = None, None - for c in self.waiting_circuits: - if c[0] == circuit.id: - circid, d = c - if d is None: - raise Exception("Expected to find circuit.") - - self.waiting_circuits.remove((circid, d)) - log.msg("Trying to build a circuit for %s" % circid) - self.request_circuit_build(d) - - def check_circuit_route(self, circuit, router): - if router in circuit.path: - #router.update() ## XXX can i use without args? no. - TorInfo.dump(self) - - def request_circuit_build(self, deferred, path=None): - if path is None: - if self.state.relays_remaining() > 0: - first, middle,last = (self.state.relays.pop() - for i in range(3)) - else: - first = random.choice(self.state.entry_guards.values()) - middle, last = (random.choice(self.state.routers.values()) - for i in range(2)) - path = [first, middle, last] - else: - assert type(path) is list, "Circuit path must be a list of routers!" - assert len(path) >= 3, "Circuits must be at least three hops!" - - log.msg("Requesting a circuit: %s" - % '->'.join(map(lambda node: node, path))) - - class AppendWaiting: - def __init__(self, attacher, deferred): - self.attacher = attacher - self.d = deferred - def __call__(self, circ): - """ - Return from build_circuit is a Circuit, however, - we want to wait until it is built before we can - issue an attach on it and callback to the Deferred - we issue here. - """ - log.msg("Circuit %s is in progress ..." % circ.id) - self.attacher.waiting_circuits.append((circ.id, self.d)) - - return self.state.build_circuit(path).addCallback( - AppendWaiting(self, deferred)).addErrback( - log.err) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py new file mode 100644 index 0000000..76ec909 --- /dev/null +++ b/ooni/utils/onion.py @@ -0,0 +1,260 @@ +# +# onion.py +# ---------- +# Utilities for working with Tor. +# +# This code is largely taken from txtorcon and its documentation, and as such +# any and all credit should go to Meejah. Minor adjustments have been made to +# use OONI's logging system, and to build custom circuits without actually +# attaching streams. +# +# :author: Meejah, Isis Lovecruft +# :license: see included LICENSE file +# :copyright: copyright (c) 2012 The Tor Project, Inc. +# :version: 0.1.0-alpha +# + +import random + +from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +from ooni.utils import log +from zope.interface import implements + + +def __setup_done__(proto): + log.msg("Setup Complete: %s" % proto) + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete).addErrback(__setup_fail__) + +def __setup_fail__(proto): + log.err("Setup Failed: %s" % proto) + report.update({'setup_fail': proto}) + reactor.stop() + +def __updates__(_progress, _tag, _summary): + log.msg("%d%%: %s", _progress, _summary) + +def write_torrc(conf, data_dir=None): + """ + Create a torrc in our data_dir. If we don't yet have a data_dir, create a + temporary one. Any temporary files or folders are added to delete_list. + + :return: torrc, data_dir, delete_list + """ + try: + from os import write, close + from tempfile import mkstemp, mkdtemp + except ImportError, ie: + log.err(ie) + + delete_list = [] + + if data_dir is None: + data_dir = mkdtemp(prefix='bridget-tordata') + delete_list.append(data_dir) + conf.DataDirectory = data_dir + + (fd, torrc) = mkstemp(dir=data_dir) + delete_list.append(torrc) + write(fd, conf.create_torrc()) + close(fd) + return torrc, data_dir, delete_list + +def delete_files_or_dirs(delete_list): + """ + Given a list of files or directories to delete, delete all and suppress + all errors. + + :param delete_list: + A list of files or directories to delete. + """ + try: + from os import unlink + from shutil import rmtree + except ImportError, ie: + log.err(ie) + + for temp in delete_list: + try: + unlink(temp) + except OSError: + rmtree(temp, ignore_errors=True) + +def start_tor(reactor, config, control_port, tor_binary, data_dir, + report=None, progress=__updates__, process_cb=__setup_done__, + process_eb=__setup_fail__): + """ + Use a txtorcon.TorConfig() instance, config, to write a torrc to a + tempfile in our DataDirectory, data_dir. If data_dir is None, a temp + directory will be created. Finally, create a TCP4ClientEndpoint at our + control_port, and connect it to our reactor and a spawned Tor + process. Compare with :meth:`txtorcon.launch_tor` for differences. + + :param reactor: + An instance of class:`twisted.internet.reactor`. + :param config: + An instance of class:`txtorcon.TorConfig` with all torrc options + already configured. ivar:`config.ControlPort`, + ivar:`config.SocksPort`, ivar:`config.CookieAuthentication`, should + already be set, as well as ivar:`config.UseBridges` and + ivar:`config.Bridge` if bridges are to be used. + ivar:`txtorcon.DataDirectory` does not need to be set. + :param control_port: + The port number to use for Tor's ControlPort. + :param tor_binary: + The full path to the Tor binary to use. + :param data_dir: + The directory to use as Tor's DataDirectory. + :param report: + The class:`ooni.plugoo.reports.Report` instance to .update(). + :param progress: + A non-blocking function to handle bootstrapping updates, which takes + three parameters: _progress, _tag, and _summary. + :param process_cb: + The function to callback to after + class:`ooni.lib.txtorcon.TorProcessProtocol` returns with the fully + bootstrapped Tor process. + :param process_eb: + The function to errback to if + class:`ooni.lib.txtorcon.TorProcessProtocol` fails. + :return: + A class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a + class:`txtorcon.TorControlProtocol` as .protocol. + """ + try: + from functools import partial + from twisted.internet.endpoints import TCP4ClientEndpoint + from ooni.lib.txtorcon import TorProtocolFactory + from ooni.lib.txtorcon import TorProcessProtocol + except ImportError, ie: + log.err(ie) + + ## TODO: add option to specify an already existing torrc, which + ## will require prior parsing to enforce necessary lines + (torrc, data_dir, to_delete) = write_torrc(config, data_dir) + + log.msg("Starting Tor ...") + log.msg("Using the following as our torrc:\n%s", config.create_torrc()) + if report is None: + report = {'torrc': config.create_torrc()} + else: + report.update({'torrc': config.create_torrc()}) + + end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port) + connection_creator = partial(end_point.connect, TorProtocolFactory()) + process_protocol = TorProcessProtocol(connection_creator, progress) + process_protocol.to_delete = to_delete + + reactor.addSystemEventTrigger('before', 'shutdown', + partial(delete_files_or_dirs, to_delete)) + try: + transport = reactor.spawnProcess(process_protocol, + tor_binary, + args=(tor_binary,'-f',torrc), + env={'HOME': data_dir}, + path=data_dir) + transport.closeStdin() + except RuntimeError, e: + log.err("Starting Tor failed: %s", e) + process_protocol.connected_cb.errback(e) + except NotImplementedError, e: + url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" + log.err("Running bridget on Windows requires pywin32: %s", url) + process_protocol.connected_cb.errback(e) + + proto = process_protocol.connected_cb ## new defer.Deferred() + proto.addCallback(process_cb) + proto.addErrback(process_eb) + return proto + + +class CustomCircuit(CircuitListenerMixin): + implements(IStreamAttacher) + + def __init__(self, state, relays=None): + self.state = state + self.waiting_circuits = [] + self.relays = relays + + def waiting_on(self, circuit): + for (circid, d) in self.waiting_circuits: + if circuit.id == circid: + return True + return False + + def circuit_extend(self, circuit, router): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + if self.waiting_on(circuit): + log.msg("Circuit %d (%s)", circuit.id, router.id_hex) + + def circuit_built(self, circuit): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + log.msg("Circuit %s built ...", circuit.id) + log.msg("Full path of %s: %s", circuit.id, circuit.path) + for (circid, d) in self.waiting_circuits: + if circid == circuit.id: + self.waiting_circuits.remove((circid, d)) + d.callback(circuit) + + def circuit_failed(self, circuit, reason): + if self.waiting_on(circuit): + log.msg("Circuit %s failed for reason %s", circuit.id, reason) + circid, d = None, None + for c in self.waiting_circuits: + if c[0] == circuit.id: + circid, d = c + if d is None: + raise Exception("Expected to find circuit.") + + self.waiting_circuits.remove((circid, d)) + log.msg("Trying to build a circuit for %s", circid) + self.request_circuit_build(d) + + def check_circuit_route(self, router): + """ + Check if a relay is a hop in one of our already built circuits. + + """ + for circ in self.state.circuits.values(): + if router in circuit.path: + #router.update() ## XXX can i use without args? no. + TorInfo.dump(self) + + def request_circuit_build(self, deferred, path=None): + if path is None: + if self.state.relays_remaining() > 0: + first, middle,last = (self.state.relays.pop() + for i in range(3)) + else: + first = random.choice(self.state.entry_guards.values()) + middle, last = (random.choice(self.state.routers.values()) + for i in range(2)) + path = [first, middle, last] + else: + assert type(path) is list, "Circuit path must be a list of relays!" + assert len(path) >= 3, "Circuits must be at least three hops!" + + log.msg("Requesting a circuit: %s" + % '->'.join(map(lambda node: node, path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + def __call__(self, circ): + """ + Return from build_circuit is a Circuit, however, + we want to wait until it is built before we can + issue an attach on it and callback to the Deferred + we issue here. + """ + log.msg("Circuit %s is in progress ...", circ.id) + self.attacher.waiting_circuits.append((circ.id, self.d)) + + return self.state.build_circuit(path).addCallback( + AppendWaiting(self, deferred)).addErrback( + log.err)

1 0

[ooni-probe/master] * Added singleton_semaphore utility function for safely adding processes to a
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 086c5d340d8abaea4d4e447690e26b0c619fcfea Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 25 13:56:01 2012 +0000 * Added singleton_semaphore utility function for safely adding processes to a reactor. * Fixes #6969 (bridget tries to start multiple tor processes). * Fixes #6970 (bridget skips testing the first bridge). --- ooni/plugins/bridget.py | 100 +++++++++++++++++++++------------------------- ooni/utils/onion.py | 67 ++++++++++++++++++++++++------- 2 files changed, 97 insertions(+), 70 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 83c9ed6..31537ec 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -370,7 +370,9 @@ class BridgetTest(OONITest): The :class:`BridgetAsset` line currently being used. """ try: - from ooni.utils.onion import start_tor, CustomCircuit + from ooni.utils.onion import start_tor, singleton_semaphore + from ooni.utils.onion import setup_done, setup_fail + from ooni.utils.onion import CustomCircuit from ooni.lib.txtorcon import TorConfig, TorState except ImportError: raise TxtorconImportError @@ -507,54 +509,49 @@ class BridgetTest(OONITest): log.msg("Bridget: initiating test ... ") - #defer.setDebugging(True) + d = defer.Deferred - if self.bridges_remaining() > 0: - if not 'Bridge' in self.config.config: - self.config.Bridge = self.bridges.pop() - - ## Necessary for avoiding starting several processes: - self.config.save() - assert self.config.config.has_key('Bridge'), "NO BRIDGE" - - state = start_tor(self.reactor, self.config, self.control_port, - self.tor_binary, self.data_directory) - #state.addCallback(remove_public_relays, self.bridges) - #state.callback - #rm_public_relays = defer.Deferred() - #rm_public_relays.addCallback(remove_public_relays, - # self.bridges) - #state.chainDeferred(rm_public_relays) - #state = defer.DeferredList([state]) - from ooni.utils.onion import __setup_done__, __setup_fail__ - state.addCallback(__setup_done__) - state.addErrback(__setup_fail__) - + if self.bridges_remaining() > 0 and not 'Bridge' in self.config.config: + self.config.Bridge = self.bridges.pop() + + ## Necessary for avoiding starting several processes: + self.config.save() + assert self.config.config.has_key('Bridge'), "NO BRIDGE" + + tor = start_tor(self.reactor, + self.config, + self.control_port, + self.tor_binary, + self.data_directory).addCallback( + setup_done).addErrback( + setup_fail) + self.tor_process_semaphore = True + + run_once = d().addCallback(singleton_semaphore, tor) + run_once.addErrback(setup_fail) - ## XXX Should do something like: if state.iscomplete - #if 'Bridge' in self.config.config: - if state and 'Bridge' in self.config.config: + only_bridges = d().addCallback(remove_public_relays, self.bridges) + + state = defer.gatherResults([run_once, only_bridges], consumeErrors=True) + log.debug("%s" % state.callbacks) + + if self.bridges_remaining() > 0: all = [] for bridge in self.bridges: self.current_bridge = bridge log.msg("We now have %d untested bridges..." % self.bridges_remaining()) - #reconf = defer.Deferred() - #reconf.addCallback(reconfigure_bridge, state, - # self.current_bridge, self.use_pt, - # self.pt_type) - #reconf.addErrback(reconfigure_fail, state, - # self.current_bridge, self.bridges_down) - #state.chainDeferred(reconf) - #state.callback - - reconf = reconfigure_bridge(state, - self.current_bridge, - self.use_pt, - self.pt_type) - all.append(reconf) - state.chainDeferred(defer.DeferredList(all)) - #state.addCallback(defer.DeferredList(all)) + reconf = d().addCallback(reconfigure_bridge, state, + self.current_bridge, + self.use_pt, + self.pt_type) + reconf.addCallback(reconfigure_done, self.current_bridge, + self.bridges_up) + reconf.addErrback(reconfigure_fail, self.current_bridge, + self.bridges_down) + all.append(reconf) + state.chainDeferred(defer.DeferredList(all)) + log.debug("%s" % state.callbacks) if self.relays_remaining() > 0: while self.relays_remaining() >= 3: @@ -568,23 +565,18 @@ class BridgetTest(OONITest): self.relays_up.append(self.current_relay) if len(circ.path) < 3: try: - parameters = (state.attacher, circ, - self.current_relay) - ext = attacher_extend_circuit(parameters) + ext = attacher_extend_circuit(state.attacher, circ, + self.current_relay) ext.addCallback(attacher_extend_circuit_done, - parameters) + state.attacher, circ, + self.current_relay) except Exception, e: - log.msg("Extend circuit failed: %s %s" - % (e, parameters)) + log.msg("Extend circuit failed: %s" % e) else: continue - #return state - ## still need to attach attacher to state - ## then build circuits - - - reactor.run() + #reactor.run() + return state def control(self, experiment_result, args): experiment_result.callback diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 08f27d4..5f15e90 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -23,18 +23,18 @@ from twisted.internet import defer from zope.interface import implements -def __setup_done__(proto): - log.msg("Setup Complete: %s" % proto) +def setup_done(proto): + log.msg("Setup Complete") state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallback(__state_complete__) - state.post_bootstrap.addErrback(__setup_fail__) + state.post_bootstrap.addCallback(state_complete) + state.post_bootstrap.addErrback(setup_fail) -def __setup_fail__(proto): +def setup_fail(proto): log.err("Setup Failed: %s" % proto) report.update({'setup_fail': proto}) reactor.stop() -def __state_complete__(state, bridge_list=None, relay_list=None): +def state_complete(state): """Called when we've got a TorState.""" log.msg("We've completely booted up a Tor version %s at PID %d" % (state.protocol.version, state.tor_pid)) @@ -44,14 +44,10 @@ def __state_complete__(state, bridge_list=None, relay_list=None): for circ in state.circuits.values(): log.msg("%s" % circ) - if bridge_list is not None and relay_list is None: - return state, bridge_list - elif bridge_list is None and relay_list is not None: - raise NotImplemented - else: - return state, None + return state -def __updates__(_progress, _tag, _summary): +def updates(_progress, _tag, _summary): + """Log updates on the Tor bootstrapping process.""" log.msg("%d%%: %s" % (_progress, _summary)) def write_torrc(conf, data_dir=None): @@ -59,6 +55,11 @@ def write_torrc(conf, data_dir=None): Create a torrc in our data_dir. If we don't yet have a data_dir, create a temporary one. Any temporary files or folders are added to delete_list. + :param conf: + A :class:`ooni.lib.txtorcon.TorConfig` object, with all configuration + values saved. + :param data_dir: + The Tor DataDirectory to use. :return: torrc, data_dir, delete_list """ try: @@ -78,6 +79,7 @@ def write_torrc(conf, data_dir=None): delete_list.append(torrc) write(fd, conf.create_torrc()) close(fd) + return torrc, data_dir, delete_list def delete_files_or_dirs(delete_list): @@ -101,9 +103,41 @@ def delete_files_or_dirs(delete_list): rmtree(temp, ignore_errors=True) @defer.inlineCallbacks +def singleton_semaphore(deferred_process_init, callbacks=[], errbacks=[]): + """ + Initialize a process only once, and do not return until + that initialization is complete. + + :param deferred_process_init: + A deferred which returns a connected process via + :meth:`twisted.internet.reactor.spawnProcess`. + :param callbacks: + A list of callback functions to add to the initialized processes' + deferred. + :param errbacks: + A list of errback functions to add to the initialized processes' + deferred. + :return: + The final state of the :param deferred_process_init: after the + callback chain has completed. This should be a fully initialized + process connected to a :class:`twisted.internet.reactor`. + """ + assert type(callbacks) is list + assert type(errbacks) is list + + for cb in callbacks: + deferred_process_init.addCallback(cb) + for eb in errbacks: + deferred_process_init.addErrback(eb) + + only_once = defer.DeferredSemaphore(1) + singleton = yield only_once.run(deferred_process_init) + defer.returnValue(singleton) + +(a)defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, - report=None, progress=__updates__, process_cb=__setup_done__, - process_eb=__setup_fail__): + report=None, progress=updates, process_cb=setup_done, + process_eb=setup_fail): """ Use a txtorcon.TorConfig() instance, config, to write a torrc to a tempfile in our DataDirectory, data_dir. If data_dir is None, a temp @@ -139,7 +173,8 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, The function to errback to if class:`ooni.lib.txtorcon.TorProcessProtocol` fails. :return: - A class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a + The result of the callback of a + class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a class:`txtorcon.TorControlProtocol` as .protocol. """ try:

1 0

[ooni-probe/master] * Still squashing bugs.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit cafc88339b058798005766cf97fabd659f0e20f5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Sep 23 19:00:33 2012 +0000 * Still squashing bugs. * Fixed the log.debug() function. * Put more general things in the onion.py utility (renamed circuit.py to onion.py). * Updated the installation instructions and documentation on dependencies. * Added my name to the license. --- INSTALL | 6 +++- LICENSE | 2 +- ooni/plugins/bridget.py | 85 ++++++++++++++++++++++------------------------ ooni/utils/log.py | 4 ++- ooni/utils/onion.py | 47 +++++++++++++++++-------- 5 files changed, 82 insertions(+), 62 deletions(-) diff --git a/INSTALL b/INSTALL index bac1a39..d972d26 100644 --- a/INSTALL +++ b/INSTALL @@ -1,5 +1,5 @@ # Dependencies -* Twisted: http://twistedmatrix.com/trac/ +* Twisted (>12.0.0): http://twistedmatrix.com/trac/ * PyYAML: http://pyyaml.org/ * Scapy: http://www.secdev.org/projects/scapy/ * pypcap: http://code.google.com/p/pypcap/ @@ -7,6 +7,10 @@ Optional: * BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +* Dnspython[1]: http://www.dnspython.org/ +* Paramiko[2]: http://www.lag.net/paramiko/ + +[1][2] These dependencies will soon be removed completely. # Installation diff --git a/LICENSE b/LICENSE index f1a9745..a0c3c18 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012, Jacob Appelbaum, Arturo Filastò +Copyright (c) 2012, Jacob Appelbaum, Arturo Filastò, Isis Lovecruft All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index c4667e1..cd7ba4e 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -189,7 +189,7 @@ class BridgetTest(OONITest): implements(IPlugin, ITest) shortName = "bridget" - description = "Use a Tor process to test connecting to bridges and relays" + description = "Use a Tor process to test connecting to bridges or relays" requirements = None options = BridgetArgs blocking = False @@ -218,7 +218,7 @@ class BridgetTest(OONITest): self.relays_down_count = lambda: len(self.relays_down) self.current_relay = None - def make_asset_list(opt, lst): + def __make_asset_list__(opt, lst): log.msg("Loading information from %s ..." % opt) with open(opt) as opt_file: for line in opt_file.readlines(): @@ -238,12 +238,12 @@ class BridgetTest(OONITest): if options['bridges']: self.config.UseBridges = 1 - make_asset_list(options['bridges'], self.bridges) + __make_asset_list__(options['bridges'], self.bridges) if options['relays']: - ## first hop must be in TorState().entry_guards to build circuits + ## first hop must be in TorState().guards to build circuits self.config.EntryNodes = ','.join(relay_list) - make_asset_list(options['relays'], self.relays) + __make_asset_list__(options['relays'], self.relays) if options['socks']: self.socks_port = options['socks'] @@ -279,6 +279,7 @@ class BridgetTest(OONITest): self.config.ControlPort = self.control_port self.config.CookieAuthentication = 1 + ''' def load_assets(self): """ Load bridges and/or relays from files given in user options. Bridges @@ -294,6 +295,7 @@ class BridgetTest(OONITest): assets.update({'relay': BridgetAsset(self.local_options['relays'])}) return assets + ''' def experiment(self, args): """ @@ -353,6 +355,7 @@ class BridgetTest(OONITest): log.err(tie) sys.exit() + ## XXX qu'est-que fuck? ou est utiliser ce fonction? def bootstrap(ctrl): """ Launch a Tor process with the TorConfig instance returned from @@ -388,19 +391,19 @@ class BridgetTest(OONITest): #else: # defer.returnValue((state.callback, controller_response)) if controller_response == 'OK': - defer.returnValue(state.callback) + defer.returnValue((state.callback, controller_response)) else: - log.msg("TorControlProtocol responded with error:\n%s", - controller_response) - defer.returnValue(state.callback) + log.msg("TorControlProtocol responded with error:\n%s" + % controller_response) + defer.returnValue((state.callback, None)) except Exception, e: - log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s", - bridge, e) + log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s" + % (bridge, e)) def reconfigure_fail(state, bridge, bad): - log.msg("Reconfiguring TorConfig with parameters %s failed", - state) + log.msg("Reconfiguring TorConfig with parameters %s failed" + % state) bad.append(bridge) @defer.inlineCallbacks @@ -416,8 +419,8 @@ class BridgetTest(OONITest): for line in bridges: if line.startswith(node): try: - log.msg("Removing %s because it is a public relay", - node) + log.msg("Removing %s because it is a public relay" + % node) bridges.remove(line) except ValueError, ve: log.err(ve) @@ -432,8 +435,8 @@ class BridgetTest(OONITest): else: defer.returnValue(state) except Exception, e: - log.msg("Removing public relays from bridge list failed:\n%s", - both) + log.msg("Removing public relays from bridge list failed:\n%s" + % both) log.err(e) except ValueError, ve: log.err(ve) @@ -444,22 +447,22 @@ class BridgetTest(OONITest): raise NotImplemented #attacher.extend_circuit - def state_complete(state, bridge_list=None, relay_list=None): - """Called when we've got a TorState.""" - log.msg("We've completely booted up a Tor version %s at PID %d" - % (state.protocol.version, state.tor_pid)) - - log.msg("This Tor has the following %d Circuits:" - % len(state.circuits)) - for circ in state.circuits.values(): - log.msg("%s" % circ) - - if bridge_list is not None and relay_list is None: - return state, bridge_list - elif bridge_list is None and relay_list is not None: - raise NotImplemented - else: - return state, None + #def state_complete(state, bridge_list=None, relay_list=None): + # """Called when we've got a TorState.""" + # log.msg("We've completely booted up a Tor version %s at PID %d" + # % (state.protocol.version, state.tor_pid)) + # + # log.msg("This Tor has the following %d Circuits:" + # % len(state.circuits)) + # for circ in state.circuits.values(): + # log.msg("%s" % circ) + # + # if bridge_list is not None and relay_list is None: + # return state, bridge_list + # elif bridge_list is None and relay_list is not None: + # raise NotImplemented + # else: + # return state, None def state_attach(state, path): log.msg("Setting up custom circuit builder...") @@ -481,7 +484,7 @@ class BridgetTest(OONITest): return d def state_attach_fail(state): - log.err("Attaching custom circuit builder failed: %s", state) + log.err("Attaching custom circuit builder failed: %s" % state) log.msg("Bridget: initiating test ... ") @@ -491,16 +494,13 @@ class BridgetTest(OONITest): #self.current_bridge = bridge if not self.config.config.has_key('Bridge'): - self.config.Bridge = self.current_bridge - - state = start_tor(reactor, + state = start_tor(self.reactor, self.config, self.control_port, self.tor_binary, self.data_directory) state.addCallback(remove_public_relays, self.bridges) - else: log.msg("We now have %d untested bridges..." % self.bridges_remaining()) @@ -511,7 +511,7 @@ class BridgetTest(OONITest): reconf.addErrback(reconfigure_fail, state, self.current_bridge, self.bridges_down) state.chainDeferred(reconf) - state.callback() + state.callback(controller_response) #all = [] #reconf = reconfigure_bridge(state, self.current_bridge, # self.use_pt, self.pt_type) @@ -570,17 +570,14 @@ class BridgetTest(OONITest): # % self.current_bridge) # self.bridges_up.append(self.current_bridge) - reactor.run() - + #reactor.run() ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) + ## ISIS' NOTES ## ----------- -## nickm points out that the format operator '%', when used in log.LEVEL(), -## forces string generation even when LEVEL is not used, increasing overhead; -## we should do 'log.err("string with stuff %s", stuff)' instead. ## ## TODO: ## o cleanup documentation diff --git a/ooni/utils/log.py b/ooni/utils/log.py index dd5cf13..54c59ea 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -88,7 +88,9 @@ def start(logstdout, logfile=None, verbosity=None): log.msg("Starting OONI...") def debug(message, level="debug", **kw): - log.msg(message, logLevel=level, **kw) + print "[%s] %s" % (level, message) + ## If we want debug messages in the logfile: + #log.msg(message, logLevel=level, **kw) def msg(message, level="info", **kw): log.msg(message, logLevel=level, **kw) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 76ec909..46373b4 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -24,13 +24,31 @@ from zope.interface import implements def __setup_done__(proto): log.msg("Setup Complete: %s" % proto) state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallback(state_complete).addErrback(__setup_fail__) + state.post_bootstrap.addCallback(__state_complete__) + state.post_bootstrap.addErrback(__setup_fail__) def __setup_fail__(proto): log.err("Setup Failed: %s" % proto) report.update({'setup_fail': proto}) reactor.stop() +def __state_complete__(state, bridge_list=None, relay_list=None): + """Called when we've got a TorState.""" + log.msg("We've completely booted up a Tor version %s at PID %d" + % (state.protocol.version, state.tor_pid)) + + log.msg("This Tor has the following %d Circuits:" + % len(state.circuits)) + for circ in state.circuits.values(): + log.msg("%s" % circ) + + if bridge_list is not None and relay_list is None: + return state, bridge_list + elif bridge_list is None and relay_list is not None: + raise NotImplemented + else: + return state, None + def __updates__(_progress, _tag, _summary): log.msg("%d%%: %s", _progress, _summary) @@ -134,7 +152,7 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, (torrc, data_dir, to_delete) = write_torrc(config, data_dir) log.msg("Starting Tor ...") - log.msg("Using the following as our torrc:\n%s", config.create_torrc()) + log.msg("Using the following as our torrc:\n%s" % config.create_torrc()) if report is None: report = {'torrc': config.create_torrc()} else: @@ -144,6 +162,8 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, connection_creator = partial(end_point.connect, TorProtocolFactory()) process_protocol = TorProcessProtocol(connection_creator, progress) process_protocol.to_delete = to_delete + process_protocol.addCallback(process_cb) + process_protocol.addErrback(process_eb) reactor.addSystemEventTrigger('before', 'shutdown', partial(delete_files_or_dirs, to_delete)) @@ -154,18 +174,15 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, env={'HOME': data_dir}, path=data_dir) transport.closeStdin() - except RuntimeError, e: - log.err("Starting Tor failed: %s", e) + except RuntimeError as e: + log.err("Starting Tor failed: %s" % e) process_protocol.connected_cb.errback(e) except NotImplementedError, e: url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" - log.err("Running bridget on Windows requires pywin32: %s", url) + log.err("Running bridget on Windows requires pywin32: %s" % url) process_protocol.connected_cb.errback(e) - proto = process_protocol.connected_cb ## new defer.Deferred() - proto.addCallback(process_cb) - proto.addErrback(process_eb) - return proto + return process_protocol.connected_cb ## new defer.Deferred() class CustomCircuit(CircuitListenerMixin): @@ -187,14 +204,14 @@ class CustomCircuit(CircuitListenerMixin): if circuit.purpose != 'GENERAL': return if self.waiting_on(circuit): - log.msg("Circuit %d (%s)", circuit.id, router.id_hex) + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) def circuit_built(self, circuit): "ICircuitListener" if circuit.purpose != 'GENERAL': return - log.msg("Circuit %s built ...", circuit.id) - log.msg("Full path of %s: %s", circuit.id, circuit.path) + log.msg("Circuit %s built ..." % circuit.id) + log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) for (circid, d) in self.waiting_circuits: if circid == circuit.id: self.waiting_circuits.remove((circid, d)) @@ -202,7 +219,7 @@ class CustomCircuit(CircuitListenerMixin): def circuit_failed(self, circuit, reason): if self.waiting_on(circuit): - log.msg("Circuit %s failed for reason %s", circuit.id, reason) + log.msg("Circuit %s failed for reason %s" % (circuit.id, reason)) circid, d = None, None for c in self.waiting_circuits: if c[0] == circuit.id: @@ -211,7 +228,7 @@ class CustomCircuit(CircuitListenerMixin): raise Exception("Expected to find circuit.") self.waiting_circuits.remove((circid, d)) - log.msg("Trying to build a circuit for %s", circid) + log.msg("Trying to build a circuit for %s" % circid) self.request_circuit_build(d) def check_circuit_route(self, router): @@ -252,7 +269,7 @@ class CustomCircuit(CircuitListenerMixin): issue an attach on it and callback to the Deferred we issue here. """ - log.msg("Circuit %s is in progress ...", circ.id) + log.msg("Circuit %s is in progress ..." % circ.id) self.attacher.waiting_circuits.append((circ.id, self.d)) return self.state.build_circuit(path).addCallback(

1 0

[ooni-probe/master] * Fixes #6968, #6971
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit fef162fe803fad2ff7eab45d321baf1086f6c980 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 25 00:07:54 2012 +0000 * Fixes #6968, #6971 * Found bug: first bridge doesn't get tested #6970 * I'm trying to use trac more to keep other people up-to-date, in case that wasn't obviously. I hate Trac like nobody's business. --- ooni/plugins/bridget.py | 201 +++++++++++++++++++++++++--------------------- ooni/utils/onion.py | 38 +++++++--- 2 files changed, 137 insertions(+), 102 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index cd7ba4e..83c9ed6 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -66,33 +66,43 @@ class PTNotFoundException(Exception): return sys.exit() class ValueChecker(object): - def port_check(self, number): + def port_check(self, port): """Check that given ports are in the allowed range.""" - number = int(number) - if number not in range(1024, 65535): + port = int(port) + if port not in range(1024, 65535): raise ValueError("Port out of range") + log.err() sock_check, ctrl_check = port_check, port_check allowed = "must be between 1024 and 65535." sock_check.coerceDoc = "Port to use for Tor's SocksPort, " +allowed ctrl_check.coerceDoc = "Port to use for Tor's ControlPort, " +allowed - def uid_check(pluggable_transport): - """Check that we're not root when trying to use pluggable transports.""" + def uid_check(self, pluggable_transport): + """ + Check that we're not root when trying to use pluggable transports. If + we are, setuid to normal user (1000) if we're running on a posix-based + system, and if we're Windows just tell the user that we can't be run + as root with the specified options and then exit. + """ uid, gid = os.getuid(), os.getgid() if uid == 0 and gid == 0: log.msg("Error: Running bridget as root with transports not allowed.") + if os.name == 'posix': log.msg("Dropping privileges to normal user...") os.setgid(1000) os.setuid(1000) + else: + sys.exit(0) - def dir_check(d): + def dir_check(self, d): """Check that the given directory exists.""" - if not os.isdir(d): + if not os.path.isdir(d): raise ValueError("%s doesn't exist, or has wrong permissions" % d) - def file_check(f): - if not os.isfile(f): + def file_check(self, f): + """Check that the given file exists.""" + if not os.path.isfile(f): raise ValueError("%s does not exist, or has wrong permissions" % f) class RandomPortException(Exception): @@ -121,8 +131,9 @@ class TxtorconImportError(ImportError): class BridgetArgs(usage.Options): """Commandline options.""" + global vc vc = ValueChecker() - + optParameters = [ ['bridges', 'b', None, 'File listing bridge IP:ORPorts to test'], @@ -227,6 +238,17 @@ class BridgetTest(OONITest): else: lst.append(line.replace('\n','')) + def __parse_data_dir__(data_dir): + if data_dir.startswith('~'): + data_dir = os.path.expanduser(data_dir) + elif data_dir.startswith('/'): + data_dir = os.path.join(os.getcwd(), data_dir) + elif data_dir.startswith('./'): + data_dir = os.path.abspath(data_dir) + else: + data_dir = os.path.join(os.getcwd(), data_dir) + return data_dir + if self.local_options: try: from ooni.lib.txtorcon import TorConfig @@ -258,17 +280,18 @@ class BridgetTest(OONITest): if options['torpath']: self.tor_binary = options['torpath'] - if self.local_options['datadir']: - self.data_directory = local_options['datadir'] + if options['datadir']: + self.data_directory = __parse_data_dir__(options['datadir']) else: self.data_directory = None if options['transport']: self.use_pt = True - log.msg("Using ClientTransportPlugin %s" % options['transport']) + log.msg("Using ClientTransportPlugin %s" + % options['transport']) [self.pt_type, pt_exec] = options['transport'].split(' ', 1) - ## ClientTransportPlugin transport exec path-to-binary [options] + ## ClientTransportPlugin transport exec pathtobinary [options] ## XXX we need a better way to deal with all PTs if self.pt_type == "obfs2": config.ClientTransportPlugin = self.pt_type + " " + pt_exec @@ -355,6 +378,7 @@ class BridgetTest(OONITest): log.err(tie) sys.exit() + ''' ## XXX qu'est-que fuck? ou est utiliser ce fonction? def bootstrap(ctrl): """ @@ -364,6 +388,7 @@ class BridgetTest(OONITest): conf = TorConfig(ctrl) conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) log.msg("Tor process connected, bootstrapping ...") + ''' @defer.inlineCallbacks def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): @@ -385,13 +410,19 @@ class BridgetTest(OONITest): raise PTNotFoundException controller_response = reset_tor.callback + controller_response.addCallback(reconfigure_done, + bridge, + reachable) + controller_response.addErrback(reconfigure_fail, + bridge, + reachable) #if not controller_response: # defer.returnValue((state.callback, None)) #else: # defer.returnValue((state.callback, controller_response)) if controller_response == 'OK': - defer.returnValue((state.callback, controller_response)) + defer.returnValue((state, controller_response)) else: log.msg("TorControlProtocol responded with error:\n%s" % controller_response) @@ -400,11 +431,16 @@ class BridgetTest(OONITest): except Exception, e: log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s" % (bridge, e)) + defer.returnValue((state.callback, e)) - def reconfigure_fail(state, bridge, bad): + def reconfigure_done(response, bridge, reachable): + log.msg("Reconfiguring with 'Bridge %s' successful" % bridge) + reachable.append(bridge) + + def reconfigure_fail(response, bridge, unreachable): log.msg("Reconfiguring TorConfig with parameters %s failed" % state) - bad.append(bridge) + unreachable.append(bridge) @defer.inlineCallbacks def remove_public_relays(state, bridges): @@ -423,7 +459,7 @@ class BridgetTest(OONITest): % node) bridges.remove(line) except ValueError, ve: - log.err(ve) + log.debug(ve) if len(both) > 0: try: @@ -447,23 +483,6 @@ class BridgetTest(OONITest): raise NotImplemented #attacher.extend_circuit - #def state_complete(state, bridge_list=None, relay_list=None): - # """Called when we've got a TorState.""" - # log.msg("We've completely booted up a Tor version %s at PID %d" - # % (state.protocol.version, state.tor_pid)) - # - # log.msg("This Tor has the following %d Circuits:" - # % len(state.circuits)) - # for circ in state.circuits.values(): - # log.msg("%s" % circ) - # - # if bridge_list is not None and relay_list is None: - # return state, bridge_list - # elif bridge_list is None and relay_list is not None: - # raise NotImplemented - # else: - # return state, None - def state_attach(state, path): log.msg("Setting up custom circuit builder...") attacher = CustomCircuit(state) @@ -488,50 +507,65 @@ class BridgetTest(OONITest): log.msg("Bridget: initiating test ... ") + #defer.setDebugging(True) if self.bridges_remaining() > 0: - for self.current_bridge in self.bridges: - #self.current_bridge = bridge + if not 'Bridge' in self.config.config: + self.config.Bridge = self.bridges.pop() + + ## Necessary for avoiding starting several processes: + self.config.save() + assert self.config.config.has_key('Bridge'), "NO BRIDGE" + + state = start_tor(self.reactor, self.config, self.control_port, + self.tor_binary, self.data_directory) + #state.addCallback(remove_public_relays, self.bridges) + #state.callback + #rm_public_relays = defer.Deferred() + #rm_public_relays.addCallback(remove_public_relays, + # self.bridges) + #state.chainDeferred(rm_public_relays) + #state = defer.DeferredList([state]) + from ooni.utils.onion import __setup_done__, __setup_fail__ + state.addCallback(__setup_done__) + state.addErrback(__setup_fail__) - if not self.config.config.has_key('Bridge'): - self.config.Bridge = self.current_bridge - state = start_tor(self.reactor, - self.config, - self.control_port, - self.tor_binary, - self.data_directory) - state.addCallback(remove_public_relays, self.bridges) - else: - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - reconf = defer.Deferred() - reconf.addCallback(reconfigure_bridge, state, - self.current_bridge, self.use_pt, - self.pt_type) - reconf.addErrback(reconfigure_fail, state, - self.current_bridge, self.bridges_down) - state.chainDeferred(reconf) - state.callback(controller_response) - #all = [] - #reconf = reconfigure_bridge(state, self.current_bridge, - # self.use_pt, self.pt_type) - #reconf.addCallback(reconfigure_done) - #reconf.addErrback(reconfigure_fail) - #state.DeferredList(all) - - if self.relays_remaining() != 0: - state.chainDeferred() - + ## XXX Should do something like: if state.iscomplete + #if 'Bridge' in self.config.config: + if state and 'Bridge' in self.config.config: + all = [] + for bridge in self.bridges: + self.current_bridge = bridge + log.msg("We now have %d untested bridges..." + % self.bridges_remaining()) + #reconf = defer.Deferred() + #reconf.addCallback(reconfigure_bridge, state, + # self.current_bridge, self.use_pt, + # self.pt_type) + #reconf.addErrback(reconfigure_fail, state, + # self.current_bridge, self.bridges_down) + #state.chainDeferred(reconf) + #state.callback + + reconf = reconfigure_bridge(state, + self.current_bridge, + self.use_pt, + self.pt_type) + all.append(reconf) + state.chainDeferred(defer.DeferredList(all)) + #state.addCallback(defer.DeferredList(all)) + + if self.relays_remaining() > 0: while self.relays_remaining() >= 3: #path = list(self.relays.pop() for i in range(3)) - #log.msg("Trying path %s" % '->'.join(map(lambda node: node, path))) + #log.msg("Trying path %s" % '->'.join(map(lambda node: + # node, path))) self.current_relay = self.relays.pop() for circ in state.circuits.values(): for node in circ.path: if node == self.current_relay: self.relays_up.append(self.current_relay) - if len(circ.path) < 3: try: parameters = (state.attacher, circ, @@ -545,32 +579,15 @@ class BridgetTest(OONITest): else: continue - return state + #return state ## still need to attach attacher to state ## then build circuits - ## XXX see txtorcon.TorControlProtocol.add_event_listener we - ## may not need full CustomCircuit class - ## o if bridges and relays, use one bridge then build a circuit - ## from three relays - ## o if bridges only, try one bridge at a time, but don't build - ## circuits, just return - ## o if relays only, build circuits from relays - #else: - # try: - # state.addCallback(reconfigure_bridge, self.current_bridge, - # self.use_pt, self.pt_type) - # state.addErrback(reconfigure_fail) - # except TimeoutError: - # log.msg("Adding %s to unreachable bridges..." - # % self.current_bridge) - # self.bridges_down.append(self.current_bridge) - # else: - # log.msg("Adding %s to reachable bridges..." - # % self.current_bridge) - # self.bridges_up.append(self.current_bridge) - - #reactor.run() + + reactor.run() + + def control(self, experiment_result, args): + experiment_result.callback ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) @@ -585,6 +602,6 @@ bridget = BridgetTest(None, None, None) ## x check if bridges are public relays ## o take bridge_desc file as input, also be able to give same ## format as output -## x Add assychronous timout for deferred, so that we don't wait +## x Add asynchronous timeout for deferred, so that we don't wait ## forever for bridges that don't work. ## o Add mechanism for testing through another host diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 46373b4..08f27d4 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -17,7 +17,9 @@ import random from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +from ooni.lib.txtorcon import TorState from ooni.utils import log +from twisted.internet import defer from zope.interface import implements @@ -50,7 +52,7 @@ def __state_complete__(state, bridge_list=None, relay_list=None): return state, None def __updates__(_progress, _tag, _summary): - log.msg("%d%%: %s", _progress, _summary) + log.msg("%d%%: %s" % (_progress, _summary)) def write_torrc(conf, data_dir=None): """ @@ -98,6 +100,7 @@ def delete_files_or_dirs(delete_list): except OSError: rmtree(temp, ignore_errors=True) +(a)defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, report=None, progress=__updates__, process_cb=__setup_done__, process_eb=__setup_fail__): @@ -162,18 +165,17 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, connection_creator = partial(end_point.connect, TorProtocolFactory()) process_protocol = TorProcessProtocol(connection_creator, progress) process_protocol.to_delete = to_delete - process_protocol.addCallback(process_cb) - process_protocol.addErrback(process_eb) reactor.addSystemEventTrigger('before', 'shutdown', partial(delete_files_or_dirs, to_delete)) try: - transport = reactor.spawnProcess(process_protocol, - tor_binary, - args=(tor_binary,'-f',torrc), - env={'HOME': data_dir}, - path=data_dir) - transport.closeStdin() + transport = yield reactor.spawnProcess(process_protocol, + tor_binary, + args=(tor_binary,'-f',torrc), + env={'HOME': data_dir}, + path=data_dir) + if transport: + transport.closeStdin() except RuntimeError as e: log.err("Starting Tor failed: %s" % e) process_protocol.connected_cb.errback(e) @@ -182,7 +184,23 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, log.err("Running bridget on Windows requires pywin32: %s" % url) process_protocol.connected_cb.errback(e) - return process_protocol.connected_cb ## new defer.Deferred() + #proc_proto = process_protocol.connected_cb + #proc_proto.addCallback(process_cb) + #proc_proto.addErrback(process_eb) + # + #d = yield process_protocol.connected_cb.addCallback( + # process_cb).addErrback( + # process_eb) + #d = yield process_protocol.connected_cb + #d.addCallback(process_cb) + #d.addErrback(process_eb) + # + #defer.returnValue(d) + + d = yield process_protocol.connected_cb + defer.returnValue(d) + + #return process_protocol.connected_cb.addCallback(process_cb).addErrback(process_eb) class CustomCircuit(CircuitListenerMixin):

1 0

[ooni-probe/master] * Modified the ValueChecker class to have static methods.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit d448f256ee5993237d9182164ba0efae808c18b6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Sep 28 04:58:00 2012 +0000 * Modified the ValueChecker class to have static methods. * Switched to dictionaries for self.relays and self.bridges so that all methods can access various values without having to constantly change parameters. * Fixed a couple assertions. --- ooni/plugins/bridget.py | 352 ++++++++++++++++++++++------------------------ ooni/utils/config.py | 9 +- ooni/utils/onion.py | 55 +++++--- 3 files changed, 210 insertions(+), 206 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index c749b90..208b4c3 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -12,25 +12,26 @@ # :licence: see included LICENSE # :version: 0.1.0-alpha -from __future__ import with_statement -from functools import partial -from random import randint -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import defer, error, reactor -from zope.interface import implements - -from ooni.utils import log, date -from ooni.utils.config import ValueChecker -from ooni.utils.onion import parse_data_dir -from ooni.utils.onion import TxtorconImportError -from ooni.utils.onion import PTNoBridgesException, PTNotFoundException -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset, MissingAssetException +from __future__ import with_statement +from functools import partial +from random import randint import os import sys +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor +from zope.interface import implements + +from ooni.utils import log, date +from ooni.utils.config import ValueChecker +from ooni.utils.onion import parse_data_dir +from ooni.utils.onion import TxtorconImportError +from ooni.utils.onion import PTNoBridgesException, PTNotFoundException +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset, MissingAssetException + class RandomPortException(Exception): """Raised when using a random port conflicts with configured ports.""" @@ -40,12 +41,9 @@ class RandomPortException(Exception): class BridgetArgs(usage.Options): """Commandline options.""" - global vc - vc = ValueChecker - allowed = "Port to use for Tor's %s, must be between 1024 and 65535." - sock_check = vc(allowed % "SocksPort").port_check - ctrl_check = vc(allowed % "ControlPort").port_check + sock_check = ValueChecker(allowed % "SocksPort").port_check + ctrl_check = ValueChecker(allowed % "ControlPort").port_check optParameters = [ ['bridges', 'b', None, @@ -69,7 +67,7 @@ class BridgetArgs(usage.Options): raise MissingAssetException( "Bridget can't run without bridges or relays to test!") if self['transport']: - vc().uid_check( + ValueChecker.uid_check( "Can't run bridget as root with pluggable transports!") if not self['bridges']: raise PTNoBridgesException @@ -77,9 +75,9 @@ class BridgetArgs(usage.Options): if self['random']: raise RandomPortException if self['datadir']: - vc().dir_check(self['datadir']) + ValueChecker.dir_check(self['datadir']) if self['torpath']: - vc().file_check(self['torpath']) + ValueChecker.file_check(self['torpath']) class BridgetAsset(Asset): """Class for parsing bridget Assets ignoring commented out lines.""" @@ -126,29 +124,11 @@ class BridgetTest(OONITest): running, so we need to deal with most of the TorConfig() only once, before the experiment runs. """ - self.d = defer.Deferred() - self.socks_port = 9049 self.control_port = 9052 self.circuit_timeout = 90 self.tor_binary = '/usr/sbin/tor' self.data_directory = None - self.use_pt = False - self.pt_type = None - - ## XXX we should do report['bridges_up'].append(self.current_bridge) - self.bridges, self.bridges_up, self.bridges_down = ([] for i in range(3)) - self.bridges_remaining = lambda: len(self.bridges) - self.bridges_down_count = lambda: len(self.bridges_down) - self.current_bridge = None - - self.relays, self.relays_up, self.relays_down = ([] for i in range(3)) - self.relays_remaining = lambda: len(self.relays) - self.relays_down_count = lambda: len(self.relays_down) - self.current_relay = None - - ## Make sure we don't use self.load_assets() for now: - self.assets = {} def __make_asset_list__(opt, lst): log.msg("Loading information from %s ..." % opt) @@ -159,52 +139,71 @@ class BridgetTest(OONITest): else: lst.append(line.replace('\n','')) + def __count_remaining__(which): + total, reach, unreach = map(lambda x: which[x], + ['all', 'reachable', 'unreachable']) + count = len(total) - reach() - unreach() + return count + + ## XXX should we do report['bridges_up'].append(self.bridges['current']) + self.bridges = {} + self.bridges['all'], self.bridges['up'], self.bridges['down'] = \ + ([] for i in range(3)) + self.bridges['reachable'] = lambda: len(self.bridges['up']) + self.bridges['unreachable'] = lambda: len(self.bridges['down']) + self.bridges['remaining'] = lambda: __count_remaining__(self.bridges) + self.bridges['current'] = None + self.bridges['pt_type'] = None + self.bridges['use_pt'] = False + + self.relays = {} + self.relays['all'], self.relays['up'], self.relays['down'] = \ + ([] for i in range(3)) + self.relays['reachable'] = lambda: len(self.relays['up']) + self.relays['unreachable'] = lambda: len(self.relays['down']) + self.relays['remaining'] = lambda: __count_remaining__(self.relays) + self.relays['current'] = None + if self.local_options: try: from ooni.lib.txtorcon import TorConfig except ImportError: raise TxtorconImportError - - options = self.local_options - config = self.config = TorConfig() + else: + self.config = TorConfig() + finally: + options = self.local_options if options['bridges']: self.config.UseBridges = 1 - __make_asset_list__(options['bridges'], self.bridges) - + __make_asset_list__(options['bridges'], self.bridges['all']) if options['relays']: - ## first hop must be in TorState().guards to build circuits + ## first hop must be in TorState().guards self.config.EntryNodes = ','.join(relay_list) - __make_asset_list__(options['relays'], self.relays) - + __make_asset_list__(options['relays'], self.relays['all']) if options['socks']: self.socks_port = options['socks'] if options['control']: self.control_port = options['control'] - if options['random']: log.msg("Using randomized ControlPort and SocksPort ...") self.socks_port = randint(1024, 2**16) self.control_port = randint(1024, 2**16) - if options['torpath']: self.tor_binary = options['torpath'] - if options['datadir']: self.data_directory = parse_data_dir(options['datadir']) - else: - self.data_directory = None - if options['transport']: - self.use_pt = True - log.msg("Using ClientTransportPlugin %s" - % options['transport']) - [self.pt_type, pt_exec] = options['transport'].split(' ', 1) - ## ClientTransportPlugin transport exec pathtobinary [options] ## XXX we need a better way to deal with all PTs - if self.pt_type == "obfs2": - self.config.ClientTransportPlugin = self.pt_type+" "+pt_exec + log.msg("Using ClientTransportPlugin %s" % options['transport']) + self.bridges['use_pt'] = True + [self.bridges['pt_type'], pt_exec] = \ + options['transport'].split(' ', 1) + + if self.bridges['pt_type'] == "obfs2": + self.config.ClientTransportPlugin = \ + self.bridges['pt_type'] + " " + pt_exec else: raise PTNotFoundException @@ -212,11 +211,21 @@ class BridgetTest(OONITest): self.config.ControlPort = self.control_port self.config.CookieAuthentication = 1 - def load_assets(self): + def __load_assets__(self): """ Load bridges and/or relays from files given in user options. Bridges should be given in the form IP:ORport. We don't want to load these as assets, because it's inefficient to start a Tor process for each one. + + We cannot use the Asset model, because that model calls + self.experiment() with the current Assets, which would be one relay + and one bridge, then it gives the defer.Deferred returned from + self.experiment() to self.control(), which means that, for each + (bridge, relay) pair, experiment gets called again, which instantiates + an additional Tor process that attempts to bind to the same + ports. Thus, additionally instantiated Tor processes return with + RuntimeErrors, which break the final defer.chainDeferred.callback(), + sending it into the errback chain. """ assets = {} if self.local_options: @@ -229,50 +238,40 @@ class BridgetTest(OONITest): return assets def experiment(self, args): - """ - We cannot use the Asset model, because that model calls - self.experiment() with the current Assets, which would be one relay - and one bridge, then it gives the defer.Deferred returned from - self.experiment() to self.control(), which means that, for each - (bridge, relay) pair, experiment gets called again, which instantiates - an additional Tor process that attempts to bind to the same - ports. Thus, additionally instantiated Tor processes return with - RuntimeErrors, which break the final defer.chainDeferred.callback(), - sending it into the errback chain. - - if bridges: - 1. configure first bridge line - 2a. configure data_dir, if it doesn't exist - 2b. write torrc to a tempfile in data_dir - 3. start tor } if any of these - 4. remove bridges which are public relays } fail, add current - 5. SIGHUP for each bridge } bridge to unreach- - } able bridges. - if relays: - 1a. configure the data_dir, if it doesn't exist - 1b. write torrc to a tempfile in data_dir - 2. start tor - 3. remove any of our relays which are already part of current - circuits - 4a. attach CustomCircuit() to self.state - 4b. RELAY_EXTEND for each relay } if this fails, add - } current relay to list - } of unreachable relays - 5. - if bridges and relays: - 1. configure first bridge line - 2a. configure data_dir if it doesn't exist - 2b. write torrc to a tempfile in data_dir - 3. start tor - 4. remove bridges which are public relays - 5. remove any of our relays which are already part of current - circuits - 6a. attach CustomCircuit() to self.state - 6b. for each bridge, build three circuits, with three - relays each - 6c. RELAY_EXTEND for each relay } if this fails, add - } current relay to list - } of unreachable relays + """ + if bridges: + 1. configure first bridge line + 2a. configure data_dir, if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor } if any of these + 4. remove bridges which are public relays } fail, add current + 5. SIGHUP for each bridge } bridge to unreach- + } able bridges. + if relays: + 1a. configure the data_dir, if it doesn't exist + 1b. write torrc to a tempfile in data_dir + 2. start tor + 3. remove any of our relays which are already part of current + circuits + 4a. attach CustomCircuit() to self.state + 4b. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays + 5. + if bridges and relays: + 1. configure first bridge line + 2a. configure data_dir if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor + 4. remove bridges which are public relays + 5. remove any of our relays which are already part of current + circuits + 6a. attach CustomCircuit() to self.state + 6b. for each bridge, build three circuits, with three + relays each + 6c. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays :param args: The :class:`BridgetAsset` line currently being used. Except that it @@ -290,59 +289,52 @@ class BridgetTest(OONITest): log.err(tie) sys.exit() + def reconfigure_done(state, bridges): + log.msg("Reconfiguring with 'Bridge %s' successful" + % bridges['current']) + bridges['up'].append(bridges['current']) + return state + + def reconfigure_fail(state, bridges): + log.msg("Reconfiguring TorConfig with parameters %s failed" + % state) + bridges['down'].append(bridges['current']) + return state + @defer.inlineCallbacks - def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): + def reconfigure_bridge(state, bridges): """ Rewrite the Bridge line in our torrc. If use of pluggable transports was specified, rewrite the line as: Bridge <transport_type> <ip>:<orport> Otherwise, rewrite in the standard form. """ - log.msg("Current Bridge: %s" % bridge) + log.msg("Current Bridge: %s" % bridges['current']) + log.msg("We now have %d bridges remaining to test..." + % bridges['remaining']()) try: - if use_pt is False: - reset_tor = yield state.protocol.set_conf('Bridge', - bridge) - elif use_pt and pt_type is not None: - reset_tor = yield state.protocol.set_conf( - 'Bridge', pt_type +' '+ bridge) + if bridges['use_pt'] is False: + controller_response = yield state.protocol.set_conf( + 'Bridge', bridges['current']) + elif bridges['use_pt'] and bridges['pt_type'] is not None: + controller_reponse = yield state.protocol.set_conf( + 'Bridge', bridges['pt_type'] +' '+ bridges['current']) else: raise PTNotFoundException - controller_response = reset_tor.callback - controller_response.addCallback(reconfigure_done, - bridge, - reachable) - controller_response.addErrback(reconfigure_fail, - bridge, - reachable) - - #if not controller_response: - # defer.returnValue((state.callback, None)) - #else: - # defer.returnValue((state.callback, controller_response)) if controller_response == 'OK': - defer.returnValue((state, controller_response)) + finish = reconfigure_done(state, bridges) else: - log.msg("TorControlProtocol responded with error:\n%s" - % controller_response) - defer.returnValue((state.callback, None)) - - except Exception, e: - log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s" - % (bridge, e)) - defer.returnValue((state.callback, e)) + log.err("SETCONF for %s responded with error:\n %s" + % (bridges['current'], controller_response)) + finish = reconfigure_fail(state, bridges) - def reconfigure_done(state, bridge, reachable): - log.msg("Reconfiguring with 'Bridge %s' successful" % bridge) - reachable.append(bridge) - return state + defer.returnValue(finish) - def reconfigure_fail(state, bridge, unreachable): - log.msg("Reconfiguring TorConfig with parameters %s failed" - % state) - unreachable.append(bridge) - return state + except Exception, e: + log.err("Reconfiguring torrc with Bridge line %s failed:\n%s" + % (bridges['current'], e)) + defer.returnValue(None) def attacher_extend_circuit(attacher, deferred, router): ## XXX todo write me @@ -373,25 +365,21 @@ class BridgetTest(OONITest): log.err("Attaching custom circuit builder failed: %s" % state) - ## Start the experiment - log.msg("Bridget: initiating test ... ") - all_of_the_bridges = self.bridges - all_of_the_relays = self.relays ## Local copy of orginal lists + log.msg("Bridget: initiating test ... ") ## Start the experiment + all_of_the_bridges = self.bridges['all'] + all_of_the_relays = self.relays['all'] ## Local copy of orginal lists - if self.bridges_remaining() >= 1 and not 'Bridge' in self.config.config: - ## XXX we should do self.bridges[0] + self.bridges[1:] - initial_bridge = all_of_the_bridges.pop() - self.config.Bridge = initial_bridge - self.config.save() ## avoid starting several processes + if self.bridges['remaining']() >= 1 and not 'Bridge' in self.config.config: + self.bridges['current'] = self.bridges['all'][0] + self.config.Bridge = self.bridges['current'] + ## avoid starting several + self.config.save() ## processes assert self.config.config.has_key('Bridge'), "NO BRIDGE" - state = start_tor(self.reactor, self.config, - self.control_port, self.tor_binary, - self.data_directory).addCallbacks( - setup_done, - errback=setup_fail) - state.addCallback(remove_public_relays, - self.bridges) + state = start_tor(reactor, self.config, self.control_port, + self.tor_binary, self.data_directory) + state.addCallbacks(setup_done, setup_fail) + state.addCallback(remove_public_relays, self.bridges) #controller = singleton_semaphore(bootstrap) #controller = x().addCallback(singleton_semaphore, tor) @@ -404,17 +392,12 @@ class BridgetTest(OONITest): log.debug("Current callbacks on TorState():\n%s" % state.callbacks) log.debug("TorState():\n%s" % state) - if self.bridges_remaining() > 0: + if self.bridges['remaining']() >= 2: all = [] - for bridge in self.bridges: - #self.current_bridge = bridge + for bridge in self.bridges['all'][1:]: + self.bridges['current'] = bridge new = defer.Deferred() - new.addCallback(reconfigure_bridge, state, bridge, - self.bridges_remaining(), - self.bridges_up, - self.bridges_down, - use_pt=self.use_pt, - pt_type=self.pt_type) + new.addCallback(reconfigure_bridge, state, self.bridges) all.append(new) #state.chainDeferred(defer.DeferredList(all)) @@ -426,38 +409,39 @@ class BridgetTest(OONITest): # % controller.callbacks) state.chainDeferred(check_remaining) - if self.relays_remaining() > 0: - while self.relays_remaining() >= 3: + if self.relays['remaining']() > 0: + while self.relays['remaining']() >= 3: #path = list(self.relays.pop() for i in range(3)) #log.msg("Trying path %s" % '->'.join(map(lambda node: # node, path))) - self.current_relay = self.relays.pop() + self.relays['current'] = self.relays['all'].pop() for circ in state.circuits.values(): for node in circ.path: - if node == self.current_relay: - self.relays_up.append(self.current_relay) + if node == self.relays['current']: + self.relays['up'].append(self.relays['current']) if len(circ.path) < 3: try: ext = attacher_extend_circuit(state.attacher, circ, - self.current_relay) + self.relays['current']) ext.addCallback(attacher_extend_circuit_done, state.attacher, circ, - self.current_relay) + self.relays['current']) except Exception, e: - log.msg("Extend circuit failed: %s" % e) + log.err("Extend circuit failed: %s" % e) else: continue - state.callback(all) + #state.callback(all) self.reactor.run() - #return state + return state def startTest(self, args): self.start_time = date.now() - log.msg("Starting %s" % self.shortName) - self.do_science = self.experiment(args) - self.do_science.addCallback(self.finished).addErrback(log.err) - return self.do_science + self.laboratory = defer.Deferred() + self.laboratory.addCallbacks(self.experiment, errback=log.err) + self.laboratory.addCallbacks(self.finished, errback=log.err) + self.laboratory.callback(args) + return self.laboratory ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) diff --git a/ooni/utils/config.py b/ooni/utils/config.py index 7c28f33..e429ef6 100644 --- a/ooni/utils/config.py +++ b/ooni/utils/config.py @@ -85,7 +85,8 @@ class ValueChecker(object): raise ValueError("Port out of range") log.err() - def uid_check(self, error_message): + @staticmethod + def uid_check(error_message): """ Check that we're not root. If we are, setuid(1000) to normal user if we're running on a posix-based system, and if we're on Windows just @@ -105,13 +106,15 @@ class ValueChecker(object): else: sys.exit(0) - def dir_check(self, d): + @staticmethod + def dir_check(d): """Check that the given directory exists.""" if not os.path.isdir(d): raise ValueError("%s doesn't exist, or has wrong permissions" % d) - def file_check(self, f): + @staticmethod + def file_check(f): """Check that the given file exists.""" if not os.path.isfile(f): raise ValueError("%s does not exist, or has wrong permissions" diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 82f013f..6f6b355 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -68,17 +68,31 @@ def parse_data_dir(data_dir): :return: The absolute path of :param:data_dir. """ - import os + from os import path, getcwd + import sys + + try: + assert isinstance(data_dir, str), \ + "Parameter type(data_dir) must be str" + except AssertionError, ae: + log.err(ae) if data_dir.startswith('~'): - data_dir = os.path.expanduser(data_dir) + data_dir = path.expanduser(data_dir) elif data_dir.startswith('/'): - data_dir = os.path.join(os.getcwd(), data_dir) + data_dir = path.join(getcwd(), data_dir) elif data_dir.startswith('./'): - data_dir = os.path.abspath(data_dir) + data_dir = path.abspath(data_dir) + else: + data_dir = path.join(getcwd(), data_dir) + + try: + assert path.isdir(data_dir), "Could not find %s" % data_dir + except AssertionError, ae: + log.err(ae) + sys.exit(1) else: - data_dir = os.path.join(os.getcwd(), data_dir) - return data_dir + return data_dir def write_torrc(conf, data_dir=None): """ @@ -137,11 +151,10 @@ def remove_node_from_list(node, list): if item.startswith(node): ## due to the :<port>. try: log.msg("Removing %s because it is a public relay" % node) - list.remove(line) + list.remove(item) except ValueError, ve: log.err(ve) -(a)defer.inlineCallbacks def remove_public_relays(state, bridges): """ Remove bridges from our bridge list which are also listed as public @@ -151,25 +164,26 @@ def remove_public_relays(state, bridges): XXX Does state.router.values() have all of the relays in the consensus, or just the ones we know about so far? + + XXX FIXME: There is a problem in that Tor needs a Bridge line to already be + configured in order to bootstrap. However, after bootstrapping, we grab the + microdescriptors of all the relays and check if any of our bridges are + listed as public relays. Because of this, the first bridge does not get + checked for being a relay. """ - IPs = map(lambda addr: addr.split(':',1)[0], bridges) + IPs = map(lambda addr: addr.split(':',1)[0], bridges['all']) both = set(state.routers.values()).intersection(IPs) if len(both) > 0: try: - updated = yield map(lambda node: remove_node_from_list(node), - both) + updated = map(lambda node: remove_node_from_list(node), both) if not updated: - ## XXX do these need to be state.callback? defer.returnValue(state) else: defer.returnValue(state) except Exception, e: - log.msg("Removing public relays from bridge list failed:\n%s" - % both) - log.err(e) - except ValueError, ve: - log.err(ve) + log.err("Removing public relays %s from bridge list failed:\n%s" + % (both, e)) #(a)defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, @@ -259,6 +273,7 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, transport = reactor.spawnProcess(process_protocol, tor_binary, args=(tor_binary,'-f',torrc), + env={'HOME':data_dir}, path=data_dir) transport.closeStdin() except RuntimeError, e: @@ -337,8 +352,10 @@ class CustomCircuit(CircuitListenerMixin): for i in range(2)) path = [first, middle, last] else: - assert type(path) is list, "Circuit path must be a list of relays!" - assert len(path) >= 3, "Circuits must be at least three hops!" + assert isinstance(path, list), \ + "Circuit path must be a list of relays!" + assert len(path) >= 3, \ + "Circuit path must be at least three hops!" log.msg("Requesting a circuit: %s" % '->'.join(map(lambda node: node, path)))

1 0

[ooni-probe/master] * Moved bootstrap and remove_public_relay functions to Tor utilities file.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit efc1206e8a099e699996c3a45742d4f7500a714f Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 27 01:19:49 2012 +0000 * Moved bootstrap and remove_public_relay functions to Tor utilities file. * Still sorting out the chainDeferred(defer.DeferredList(list)) problems. --- ooni/plugins/bridget.py | 136 +++++++++++++++++------------------------------ ooni/utils/onion.py | 107 ++++++++++++++++++++++++++----------- 2 files changed, 123 insertions(+), 120 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 99a7d05..c749b90 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -279,8 +279,9 @@ class BridgetTest(OONITest): in Bridget it doesn't, so it should be ignored and avoided. """ try: - from ooni.utils.process import singleton_semaphore - from ooni.utils.onion import start_tor, setup_done, setup_fail + from ooni.utils import process + from ooni.utils.onion import start_tor, remove_public_relays + from ooni.utils.onion import setup_done, setup_fail from ooni.utils.onion import CustomCircuit from ooni.lib.txtorcon import TorConfig, TorState except ImportError: @@ -289,18 +290,6 @@ class BridgetTest(OONITest): log.err(tie) sys.exit() - ''' - ## XXX qu'est-que fuck? ou est utiliser ce fonction? - def bootstrap(ctrl): - """ - Launch a Tor process with the TorConfig instance returned from - initialize() and write_torrc(). - """ - conf = TorConfig(ctrl) - conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) - log.msg("Tor process connected, bootstrapping ...") - ''' - @defer.inlineCallbacks def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None): """ @@ -312,7 +301,7 @@ class BridgetTest(OONITest): log.msg("Current Bridge: %s" % bridge) try: if use_pt is False: - reset_tor = yield state.protocol.set_conf('Bridge', + reset_tor = yield state.protocol.set_conf('Bridge', bridge) elif use_pt and pt_type is not None: reset_tor = yield state.protocol.set_conf( @@ -331,62 +320,29 @@ class BridgetTest(OONITest): #if not controller_response: # defer.returnValue((state.callback, None)) #else: - # defer.returnValue((state.callback, controller_response)) + # defer.returnValue((state.callback, controller_response)) if controller_response == 'OK': defer.returnValue((state, controller_response)) else: log.msg("TorControlProtocol responded with error:\n%s" % controller_response) defer.returnValue((state.callback, None)) - + except Exception, e: log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s" % (bridge, e)) defer.returnValue((state.callback, e)) - def reconfigure_done(response, bridge, reachable): + def reconfigure_done(state, bridge, reachable): log.msg("Reconfiguring with 'Bridge %s' successful" % bridge) reachable.append(bridge) + return state - def reconfigure_fail(response, bridge, unreachable): + def reconfigure_fail(state, bridge, unreachable): log.msg("Reconfiguring TorConfig with parameters %s failed" % state) unreachable.append(bridge) - - @defer.inlineCallbacks - def remove_public_relays(state, bridges): - """ - Remove bridges from our bridge list which are also listed as - public relays. - """ - IPs = map(lambda addr: addr.split(':',1)[0], bridges) - both = set(state.routers.values()).intersection(IPs) - - def __remove_line__(node, bridges=bridges): - for line in bridges: - if line.startswith(node): - try: - log.msg("Removing %s because it is a public relay" - % node) - bridges.remove(line) - except ValueError, ve: - log.debug(ve) - - if len(both) > 0: - try: - updated = yield map(lambda node: - __remove_line__(node), both) - if not updated: - ## XXX do these need to be state.callback? - defer.returnValue(state) - else: - defer.returnValue(state) - except Exception, e: - log.msg("Removing public relays from bridge list failed:\n%s" - % both) - log.err(e) - except ValueError, ve: - log.err(ve) + return state def attacher_extend_circuit(attacher, deferred, router): ## XXX todo write me @@ -417,54 +373,58 @@ class BridgetTest(OONITest): log.err("Attaching custom circuit builder failed: %s" % state) + ## Start the experiment log.msg("Bridget: initiating test ... ") - x = defer.Deferred - - if self.bridges_remaining() > 0 and not 'Bridge' in self.config.config: - self.config.Bridge = self.bridges.pop() - - ## Necessary for avoiding starting several processes: - self.config.save() + all_of_the_bridges = self.bridges + all_of_the_relays = self.relays ## Local copy of orginal lists + + if self.bridges_remaining() >= 1 and not 'Bridge' in self.config.config: + ## XXX we should do self.bridges[0] + self.bridges[1:] + initial_bridge = all_of_the_bridges.pop() + self.config.Bridge = initial_bridge + self.config.save() ## avoid starting several processes assert self.config.config.has_key('Bridge'), "NO BRIDGE" - tor = start_tor(self.reactor, - self.config, - self.control_port, - self.tor_binary, - self.data_directory).addCallback( - setup_done).addErrback( - setup_fail) - self.tor_process_semaphore = True + state = start_tor(self.reactor, self.config, + self.control_port, self.tor_binary, + self.data_directory).addCallbacks( + setup_done, + errback=setup_fail) + state.addCallback(remove_public_relays, + self.bridges) - run_once = x().addCallback(singleton_semaphore, tor) - run_once.addErrback(setup_fail) + #controller = singleton_semaphore(bootstrap) + #controller = x().addCallback(singleton_semaphore, tor) + #controller.addErrback(setup_fail) - filter_bridges = x().addCallback(remove_public_relays, self.bridges) + #filter_bridges = remove_public_relays(self.bridges) - state = defer.gatherResults([run_once, filter_bridges], consumeErrors=True) + #bootstrap = defer.gatherResults([controller, filter_bridges], + # consumeErrors=True) log.debug("Current callbacks on TorState():\n%s" % state.callbacks) + log.debug("TorState():\n%s" % state) if self.bridges_remaining() > 0: all = [] for bridge in self.bridges: - self.current_bridge = bridge - log.msg("We now have %d untested bridges..." - % self.bridges_remaining()) - reconf = x().addCallback(reconfigure_bridge, state, - self.current_bridge, - self.use_pt, - self.pt_type) - reconf.addCallback(reconfigure_done, self.current_bridge, - self.bridges_up) - reconf.addErrback(reconfigure_fail, self.current_bridge, - self.bridges_down) - all.append(reconf) + #self.current_bridge = bridge + new = defer.Deferred() + new.addCallback(reconfigure_bridge, state, bridge, + self.bridges_remaining(), + self.bridges_up, + self.bridges_down, + use_pt=self.use_pt, + pt_type=self.pt_type) + all.append(new) #state.chainDeferred(defer.DeferredList(all)) #state.chainDeferred(defer.gatherResults(all, consumeErrors=True)) - n_plus_one_bridges = defer.gatherResults(all, consumeErrors=True) - state.chainDeferred(n_plus_one_bridges) - log.debug("Current callbacks on TorState():\n%s" % state.callbacks) + check_remaining = defer.DeferredList(all, consumeErrors=True) + + #controller.chainDeferred(check_remaining) + #log.debug("Current callbacks on TorState():\n%s" + # % controller.callbacks) + state.chainDeferred(check_remaining) if self.relays_remaining() > 0: while self.relays_remaining() >= 3: diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 81e4ea3..82f013f 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -13,6 +13,7 @@ # :copyright: copyright (c) 2012 The Tor Project, Inc. # :version: 0.1.0-alpha # +# XXX TODO add report keys for onion methods import random @@ -31,25 +32,32 @@ def setup_done(proto): def setup_fail(proto): log.err("Setup Failed: %s" % proto) - report.update({'setup_fail': proto}) + #report.update({'setup_fail': proto}) reactor.stop() def state_complete(state): """Called when we've got a TorState.""" log.msg("We've completely booted up a Tor version %s at PID %d" % (state.protocol.version, state.tor_pid)) - log.msg("This Tor has the following %d Circuits:" % len(state.circuits)) for circ in state.circuits.values(): log.msg("%s" % circ) - - #return state + return state def updates(_progress, _tag, _summary): """Log updates on the Tor bootstrapping process.""" log.msg("%d%%: %s" % (_progress, _summary)) +def bootstrap(ctrl): + """ + Bootstrap Tor from an instance of + :class:`ooni.lib.txtorcon.TorControlProtocol`. + """ + conf = TorConfig(ctrl) + conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) + log.msg("Tor process connected, bootstrapping ...") + def parse_data_dir(data_dir): """ Parse a string that a has been given as a DataDirectory and determine @@ -124,7 +132,46 @@ def delete_files_or_dirs(delete_list): except OSError: rmtree(temp, ignore_errors=True) +def remove_node_from_list(node, list): + for item in list: ## bridges don't match completely + if item.startswith(node): ## due to the :<port>. + try: + log.msg("Removing %s because it is a public relay" % node) + list.remove(line) + except ValueError, ve: + log.err(ve) + @defer.inlineCallbacks +def remove_public_relays(state, bridges): + """ + Remove bridges from our bridge list which are also listed as public + relays. This must be called after Tor has fully bootstrapped and we have a + :class:`ooni.lib.txtorcon.TorState` with the + :attr:`ooni.lib.txtorcon.TorState.routers` attribute assigned. + + XXX Does state.router.values() have all of the relays in the consensus, or + just the ones we know about so far? + """ + IPs = map(lambda addr: addr.split(':',1)[0], bridges) + both = set(state.routers.values()).intersection(IPs) + + if len(both) > 0: + try: + updated = yield map(lambda node: remove_node_from_list(node), + both) + if not updated: + ## XXX do these need to be state.callback? + defer.returnValue(state) + else: + defer.returnValue(state) + except Exception, e: + log.msg("Removing public relays from bridge list failed:\n%s" + % both) + log.err(e) + except ValueError, ve: + log.err(ve) + +#(a)defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, report=None, progress=updates, process_cb=setup_done, process_eb=setup_fail): @@ -193,39 +240,35 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, reactor.addSystemEventTrigger('before', 'shutdown', partial(delete_files_or_dirs, to_delete)) + #try: + # transport = yield reactor.spawnProcess(process_protocol, + # tor_binary, + # args=(tor_binary,'-f',torrc), + # env={'HOME': data_dir}, + # path=data_dir) + # if transport: + # transport.closeStdin() + #except RuntimeError as e: + # log.err("Starting Tor failed: %s" % e) + # process_protocol.connected_cb.errback(e) + #except NotImplementedError, e: + # url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" + # log.err("Running bridget on Windows requires pywin32: %s" % url) + # process_protocol.connected_cb.errback(e) try: - transport = yield reactor.spawnProcess(process_protocol, - tor_binary, - args=(tor_binary,'-f',torrc), - env={'HOME': data_dir}, - path=data_dir) - if transport: - transport.closeStdin() - except RuntimeError as e: - log.err("Starting Tor failed: %s" % e) - process_protocol.connected_cb.errback(e) - except NotImplementedError, e: - url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" - log.err("Running bridget on Windows requires pywin32: %s" % url) + transport = reactor.spawnProcess(process_protocol, + tor_binary, + args=(tor_binary,'-f',torrc), + path=data_dir) + transport.closeStdin() + except RuntimeError, e: + log.err(e) process_protocol.connected_cb.errback(e) - #proc_proto = process_protocol.connected_cb - #proc_proto.addCallback(process_cb) - #proc_proto.addErrback(process_eb) - # - #d = yield process_protocol.connected_cb.addCallback( - # process_cb).addErrback( - # process_eb) + return process_protocol.connected_cb + #d = yield process_protocol.connected_cb - #d.addCallback(process_cb) - #d.addErrback(process_eb) - # #defer.returnValue(d) - - d = yield process_protocol.connected_cb - defer.returnValue(d) - - #return process_protocol.connected_cb.addCallback(process_cb).addErrback(process_eb) class CustomCircuit(CircuitListenerMixin): implements(IStreamAttacher)

1 0

[ooni-probe/master] Add some documentation on writing OONI Tests using
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit ffc44ce1cbc595ce8cbcd6522e8962f37b7e6869 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 12:00:07 2012 +0000 Add some documentation on writing OONI Tests using the new framework. --- docs/source/conf.py | 7 ++- docs/source/index.rst | 100 +++++++++++------------------------------ docs/source/install.rst | 55 ++++++++++++++++++++++ docs/source/tutorial.rst | 1 - docs/source/writing_tests.rst | 80 ++++++++++++++++++++++++++------ docs/writing_tests.md | 17 ------- ooni/nettest.py | 43 +++++++++--------- 7 files changed, 174 insertions(+), 129 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 49b96d4..e066613 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,7 +16,9 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, + os.path.join(os.path.dirname(__file__), '..', '..')) + # -- General configuration ----------------------------------------------------- @@ -25,7 +27,8 @@ import sys, os # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', +'sphinx.ext.viewcode', 'sphinx.ext.autodoc'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/source/index.rst b/docs/source/index.rst index 6e95ee7..34ca87c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,96 +1,50 @@ .. OONI documentation master file. -========== -About OONI -========== +Welcome to the OONI documentation! +================================== + The Net interprets censorship as damage and routes around it. + John Gilmore; TIME magazine (6 December 1993) -Dependencies -************ +OONI, the Open Observatory of Network Interference, is a global observation +network which aims is to collect high quality data using open methodologies, +using Free and Open Source Software (FL/OSS) to share observations and data +about the various types, methods, and amounts of network tampering in the world. -* Twisted: http://twistedmatrix.com/trac/ -* PyYAML: http://pyyaml.org/ -* Scapy: http://www.secdev.org/projects/scapy/ - * pypcap: http://code.google.com/p/pypcap/ - * libdnet: http://code.google.com/p/libdnet/ -*Optional* +Getting started +*************** -* BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +If you choose to use virtualenv to setup your development environment you will +need to do the following:: -Installation -************ + virtualenv ENV + source ENV/bin/activate + pip install twisted Scapy pyyaml -On debian you can install all the dependecies with apt-get with this command: +To get the latest version of scapy you will need mercurial. You can then install +it with:: - apt-get install python-twisted python-twisted-names python-yaml python-scapy python-beautifulsoup - -*The "hard" way* - -This involves installing the dependencies installable via easy_install/pip and -the ones that are not by building them from source. - -"simple" dependencies via easy_install: - - sudo easy_install pyyaml - sudo easy_install twisted - sudo easy_install beautifulsoup - -"simple" dependencies via pip: - - sudo pip install pyyaml - sudo pip install twisted - sudo pip install beautifulsoup + pip install hg+http://hg.secdev.org/scapy -libdnet: +On debian you can install all the dependecies with apt-get with this command:: - wget http://libdnet.googlecode.com/files/libdnet-1.12.tgz - tar xzf libdnet-1.12.tgz - cd libdnet-1.12 - ./configure && make - cd python/ - sudo python setup.py install - cd ../../ && rm -rf libdnet-1.12* - -pypcap: - - svn checkout http://pypcap.googlecode.com/svn/trunk/ pypcap-read-only - cd pypcap-read-only/ - sudo pip install pyrex - make - sudo python setup.py install - cd ../ && rm -rf pypcap-read-only - -scapy: - - wget http://www.secdev.org/projects/scapy/files/scapy-latest.zip - unzip scapy-latest.zip - cd scapy-2.2.0/ - sudo python setup.py install - cd ../ && rm -rf scapy-* - -Running -******* + apt-get install python-twisted python-twisted-names python-yaml python-scapy python-beautifulsoup -To run ooni probe do +Once you have installed all the dependencies OONI tests can be run like so:: -$ export PYTHONPATH=`pwd` + bin/ooniprobe path/to/test.py --cmd1 foo --cmd2 bar -$ python ooni/ooniprobe.py +Contents +******** .. toctree:: - :maxdepth: 2 + :maxdepth: 2 + :glob: - intro + install tutorial writing_tests - ... - -Indices and tables -================== -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/source/install.rst b/docs/source/install.rst new file mode 100644 index 0000000..a2352d0 --- /dev/null +++ b/docs/source/install.rst @@ -0,0 +1,55 @@ + +Installing OONI +=============== + +Currently no installation documentation is present, since OONI is not meant to +be installed and should be handled with care. + +Dependencies +************ + +OONI depends on the following pieces of software. + +* Twisted: http://twistedmatrix.com/trac/ +* PyYAML: http://pyyaml.org/ +* Scapy: http://www.secdev.org/projects/scapy/ + * pypcap: http://code.google.com/p/pypcap/ + * libdnet: http://code.google.com/p/libdnet/ + +*Optional* + +* BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ + +Manual installation of scapy +---------------------------- + +It is optimal to install scapy, libdnet and pypcap from source. This can be +done with the following code snippets. + +libdnet:: + + wget http://libdnet.googlecode.com/files/libdnet-1.12.tgz + tar xzf libdnet-1.12.tgz + cd libdnet-1.12 + ./configure && make + cd python/ + sudo python setup.py install + cd ../../ && rm -rf libdnet-1.12* + +pypcap:: + + svn checkout http://pypcap.googlecode.com/svn/trunk/ pypcap-read-only + cd pypcap-read-only/ + sudo pip install pyrex + make + sudo python setup.py install + cd ../ && rm -rf pypcap-read-only + +scapy:: + + wget http://www.secdev.org/projects/scapy/files/scapy-latest.zip + unzip scapy-latest.zip + cd scapy-2.2.0/ + sudo python setup.py install + cd ../ && rm -rf scapy-* + diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index b13b687..15afad7 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -1,4 +1,3 @@ -======== Tutorial ======== diff --git a/docs/source/writing_tests.rst b/docs/source/writing_tests.rst index 1aa4b7c..a18b096 100644 --- a/docs/source/writing_tests.rst +++ b/docs/source/writing_tests.rst @@ -1,26 +1,76 @@ -.. OONI documentation master file. - -================== Writing OONI tests ================== -OONIProbe tests can be written in two modes: blocking or non-blocking. -Going the blocking route is not advised and all tests in the end should end up -being written in the non-blocking way. +The OONI testing API is heavily influenced and partially based on the python +:class:`unittest` module and :class:`twsted.trial`. + + +Test Cases +---------- + +The atom of OONI Testing is called a Test Case. A test case class may contain +multiple Test Functions. + +.. autoclass:: ooni.nettest.TestCase + +:class:`ooni.nettest.TestCase` is a subclass of :class:`unittest.TestCase` so +the assert methods that apply to :class:`unittest.TestCase` will also apply to +:class:`ooni.nettest.TestCase`. + +If the test you plan to write is not listed on the `Tor OONI trac page +<https://trac.torproject.org/projects/tor/wiki/doc/OONI/Tests>`_, you should +add it to the list and following the `test template +<https://trac.torproject.org/projects/tor/wiki/doc/OONI/Tests/TestTemplate>`_ +write up a description about it. + + +Inputs +------ + +Inputs are what is given as input to every iteration of the Test Case. You have +100 inputs, then every test case will be run 100 times. + +To configure a static set of inputs you should define the +:class:`ooni.nettest.TestCase` attribute ``inputs``. The test will be run ``len(inputs)`` times. Any iterable object is a valid ``inputs`` attribute. + +If you would like to have inputs be determined from a user specified input +file, then you must set the ``inputFile`` attribute. This is an array that +specifies what command line option may be used to control this value. + +By default the ``inputProcessor`` is set to read the file line by line and +strip newline characters. To change this behavior you must set the +``inputProcessor`` attribute to a function that takes as arugment a file +descriptor and yield the next item. The default ``inputProcessor`` looks like +this:: + + + def lineByLine(fp): + for x in fp.readlines(): + yield x.strip() + fp.close() + + +Test Functions +-------------- + +These shall be defined inside of your :class:`ooni.nettest.TestCase` subclass. +These will be class methods. -A good way to understand how to write a test is also to take a look at the OONI -Test Interface in the following files: +To add data to the test report you may write directly to the report object like +so:: -* ooni/plugoo/interface.py + def my_test_function(): + result = do_something() + self.report['something'] = result -* ooni/plugoo/tests.py +OONI will then handle the writing of the data to the final test report. -Writing non-blocking tests --------------------------- +To access the current input you can use the ``input`` attribute, for example:: -To bootstrap the process of creating a new tests you can run the scaffolding -script in ooni/scaffolding.py. + def my_test_with_input(): + do_something_with_input(self.input) -This will create a new plugin with the specified name inside of ooni/plugins/. +This will at each iteration over the list of inputs do something with the +input. diff --git a/docs/writing_tests.md b/docs/writing_tests.md deleted file mode 100644 index 460620a..0000000 --- a/docs/writing_tests.md +++ /dev/null @@ -1,17 +0,0 @@ -= Writing OONI Tests = - -There are two modes of writing tests. The first way is by making the Test -blocking and have each instance of a test run in a separate thread. The other -is by making them non-blocking using the twisted deferred non blocking pattern. - -The first kind of mechanism relied of writing a test that uses blocking code -and each instance of it is run inside of a separate thread. Beware that your -blocking code must be thread safe to run properly (as is obvious :P) - -The other menthod invloves having some knowledge about twisted. The test you -will write will be written in twisted and should implement async style non -blocking architecture. - -It is recommended that tests are written using the second pattern and the first -should only be used for runnign tests that have been previously written. - diff --git a/ooni/nettest.py b/ooni/nettest.py index 2a02a43..c0c7406 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -34,39 +34,40 @@ class TestCase(unittest.TestCase): This is the monad of the OONI nettest universe. When you write a nettest you will subclass this object. - _inputs_ can be set to a static set of inputs. All the tests (the methods - starting with the "test_" prefix) will be run once per input. At every run - the _input_ attribute of the TestCase instance will be set to the value of - the current iteration over inputs. Any python iterable object can be set - to inputs. + * inputs: can be set to a static set of inputs. All the tests (the methods + starting with the "test_" prefix) will be run once per input. At every run + the _input_ attribute of the TestCase instance will be set to the value of + the current iteration over inputs. Any python iterable object can be set + to inputs. - _inputFile_ attribute should be set to an array containing the command line - argument that should be used as the input file. Such array looks like this: + * inputFile: attribute should be set to an array containing the command line + argument that should be used as the input file. Such array looks like + this: - ["commandlinearg", "c", "The description"] + ``["commandlinearg", "c", "The description"]`` - The second value of such arrray is the shorthand for the command line arg. - The user will then be able to specify inputs to the test via: + The second value of such arrray is the shorthand for the command line arg. + The user will then be able to specify inputs to the test via: - ooniprobe mytest.py --commandlinearg path/to/file.txt + ``ooniprobe mytest.py --commandlinearg path/to/file.txt`` - or + or - ooniprobe mytest.py -c path/to/file.txt + ``ooniprobe mytest.py -c path/to/file.txt`` - _inputProcessor_ should be set to a function that takes as argument an - open file descriptor and it will yield the input to be passed to the test - instance. + * inputProcessor: should be set to a function that takes as argument an + open file descriptor and it will yield the input to be passed to the test + instance. - _name_ should be set to the name of the test. + * name: should be set to the name of the test. - _author_ should contain the name and contact details for the test author. - The format for such string is as follows: + * author: should contain the name and contact details for the test author. + The format for such string is as follows: - The Name <email(a)example.com> + ``The Name <email(a)example.com>`` - _version_ is the version string of the test. + * version: is the version string of the test. """ name = "IDidNotChangeTheName" author = "John Doe <foo(a)example.com>"

1 0

[ooni-probe/master] * Added more general Tor utility functions to ooni.utils.onion.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit f88900672897e858ec85da4f5da343cf99ba28d6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Sep 26 12:43:56 2012 +0000 * Added more general Tor utility functions to ooni.utils.onion. * Added decorator-based timeout function for blocking methods to ooni.utils.timer. * Moved general Exception classes to appropriate files. * Fixed another bug in the Options type enforcer. --- ooni/plugins/bridget.py | 212 ++++++++++++++-------------------------------- ooni/plugoo/assets.py | 5 + ooni/plugoo/tests.py | 1 - ooni/utils/config.py | 65 ++++++++++++++ ooni/utils/onion.py | 90 ++++++++++++-------- ooni/utils/process.py | 54 ++++++++++++ ooni/utils/timer.py | 36 ++++++++ 7 files changed, 281 insertions(+), 182 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 31537ec..99a7d05 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -12,135 +12,48 @@ # :licence: see included LICENSE # :version: 0.1.0-alpha -from __future__ import with_statement -from random import randint -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import defer, error, reactor -from zope.interface import implements - -from ooni.utils import log -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset +from __future__ import with_statement +from functools import partial +from random import randint +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor +from zope.interface import implements + +from ooni.utils import log, date +from ooni.utils.config import ValueChecker +from ooni.utils.onion import parse_data_dir +from ooni.utils.onion import TxtorconImportError +from ooni.utils.onion import PTNoBridgesException, PTNotFoundException +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset, MissingAssetException import os import sys -def timer(secs, e=None): - import signal - import functools.wraps - def decorator(func): - def _timer(signum, frame): - raise TimeoutError, e - def wrapper(*args, **kwargs): - signal.signal(signal.SIGALRM, _timer) - signal.alarm(secs) - try: - res = func(*args, **kwargs) - finally: - signal.alarm(0) - return res - return functools.wraps(func)(wrapper) - return decorator - - -class MissingAssetException(Exception): - """Raised when neither there are neither bridges nor relays to test.""" - def __init__(self): - log.msg("Bridget can't run without bridges or relays to test!") - return sys.exit() - -class PTNoBridgesException(Exception): - """Raised when a pluggable transport is specified, but no bridges.""" - def __init__(self): - log.msg("Pluggable transport requires the bridges option") - return sys.exit() - -class PTNotFoundException(Exception): - def __init__(self, transport_type): - m = "Pluggable Transport type %s was unaccounted " % transport_type - m += "for, please contact isis(at)torproject(dot)org and it will " - m += "get included." - log.msg("%s" % m) - return sys.exit() - -class ValueChecker(object): - def port_check(self, port): - """Check that given ports are in the allowed range.""" - port = int(port) - if port not in range(1024, 65535): - raise ValueError("Port out of range") - log.err() - - sock_check, ctrl_check = port_check, port_check - allowed = "must be between 1024 and 65535." - sock_check.coerceDoc = "Port to use for Tor's SocksPort, " +allowed - ctrl_check.coerceDoc = "Port to use for Tor's ControlPort, " +allowed - - def uid_check(self, pluggable_transport): - """ - Check that we're not root when trying to use pluggable transports. If - we are, setuid to normal user (1000) if we're running on a posix-based - system, and if we're Windows just tell the user that we can't be run - as root with the specified options and then exit. - """ - uid, gid = os.getuid(), os.getgid() - if uid == 0 and gid == 0: - log.msg("Error: Running bridget as root with transports not allowed.") - if os.name == 'posix': - log.msg("Dropping privileges to normal user...") - os.setgid(1000) - os.setuid(1000) - else: - sys.exit(0) - - def dir_check(self, d): - """Check that the given directory exists.""" - if not os.path.isdir(d): - raise ValueError("%s doesn't exist, or has wrong permissions" % d) - - def file_check(self, f): - """Check that the given file exists.""" - if not os.path.isfile(f): - raise ValueError("%s does not exist, or has wrong permissions" % f) - class RandomPortException(Exception): """Raised when using a random port conflicts with configured ports.""" def __init__(self): log.msg("Unable to use random and specific ports simultaneously") return sys.exit() -class TimeoutError(Exception): - """Raised when a timer runs out.""" - pass - -class TxtorconImportError(ImportError): - """Raised when /ooni/lib/txtorcon cannot be imported from.""" - cwd, tx = os.getcwd(), 'lib/txtorcon/torconfig.py' - try: - log.msg("Unable to import from ooni.lib.txtorcon") - if cwd.endswith('ooni'): - check = os.path.join(cwd, tx) - else: - check = os.path.join(cwd, 'ooni/'+tx) - assert isfile(check) - except: - log.msg("Error: Some OONI libraries are missing!") - log.msg("Please go to /ooni/lib/ and do \"make all\"") - class BridgetArgs(usage.Options): """Commandline options.""" global vc - vc = ValueChecker() + vc = ValueChecker + allowed = "Port to use for Tor's %s, must be between 1024 and 65535." + sock_check = vc(allowed % "SocksPort").port_check + ctrl_check = vc(allowed % "ControlPort").port_check + optParameters = [ ['bridges', 'b', None, 'File listing bridge IP:ORPorts to test'], ['relays', 'f', None, 'File listing relay IPs to test'], - ['socks', 's', 9049, None, vc.sock_check], - ['control', 'c', 9052, None, vc.ctrl_check], + ['socks', 's', 9049, None, sock_check], + ['control', 'c', 9052, None, ctrl_check], ['torpath', 'p', None, 'Path to the Tor binary to use'], ['datadir', 'd', None, @@ -153,18 +66,20 @@ class BridgetArgs(usage.Options): def postOptions(self): if not self['bridges'] and not self['relays']: - raise MissingAssetException + raise MissingAssetException( + "Bridget can't run without bridges or relays to test!") if self['transport']: - vc.uid_check(self['transport']) + vc().uid_check( + "Can't run bridget as root with pluggable transports!") if not self['bridges']: raise PTNoBridgesException if self['socks'] or self['control']: if self['random']: raise RandomPortException if self['datadir']: - vc.dir_check(self['datadir']) + vc().dir_check(self['datadir']) if self['torpath']: - vc.file_check(self['torpath']) + vc().file_check(self['torpath']) class BridgetAsset(Asset): """Class for parsing bridget Assets ignoring commented out lines.""" @@ -211,6 +126,8 @@ class BridgetTest(OONITest): running, so we need to deal with most of the TorConfig() only once, before the experiment runs. """ + self.d = defer.Deferred() + self.socks_port = 9049 self.control_port = 9052 self.circuit_timeout = 90 @@ -219,6 +136,7 @@ class BridgetTest(OONITest): self.use_pt = False self.pt_type = None + ## XXX we should do report['bridges_up'].append(self.current_bridge) self.bridges, self.bridges_up, self.bridges_down = ([] for i in range(3)) self.bridges_remaining = lambda: len(self.bridges) self.bridges_down_count = lambda: len(self.bridges_down) @@ -229,6 +147,9 @@ class BridgetTest(OONITest): self.relays_down_count = lambda: len(self.relays_down) self.current_relay = None + ## Make sure we don't use self.load_assets() for now: + self.assets = {} + def __make_asset_list__(opt, lst): log.msg("Loading information from %s ..." % opt) with open(opt) as opt_file: @@ -238,17 +159,6 @@ class BridgetTest(OONITest): else: lst.append(line.replace('\n','')) - def __parse_data_dir__(data_dir): - if data_dir.startswith('~'): - data_dir = os.path.expanduser(data_dir) - elif data_dir.startswith('/'): - data_dir = os.path.join(os.getcwd(), data_dir) - elif data_dir.startswith('./'): - data_dir = os.path.abspath(data_dir) - else: - data_dir = os.path.join(os.getcwd(), data_dir) - return data_dir - if self.local_options: try: from ooni.lib.txtorcon import TorConfig @@ -281,7 +191,7 @@ class BridgetTest(OONITest): self.tor_binary = options['torpath'] if options['datadir']: - self.data_directory = __parse_data_dir__(options['datadir']) + self.data_directory = parse_data_dir(options['datadir']) else: self.data_directory = None @@ -294,7 +204,7 @@ class BridgetTest(OONITest): ## ClientTransportPlugin transport exec pathtobinary [options] ## XXX we need a better way to deal with all PTs if self.pt_type == "obfs2": - config.ClientTransportPlugin = self.pt_type + " " + pt_exec + self.config.ClientTransportPlugin = self.pt_type+" "+pt_exec else: raise PTNotFoundException @@ -302,7 +212,6 @@ class BridgetTest(OONITest): self.config.ControlPort = self.control_port self.config.CookieAuthentication = 1 - ''' def load_assets(self): """ Load bridges and/or relays from files given in user options. Bridges @@ -318,7 +227,6 @@ class BridgetTest(OONITest): assets.update({'relay': BridgetAsset(self.local_options['relays'])}) return assets - ''' def experiment(self, args): """ @@ -367,13 +275,14 @@ class BridgetTest(OONITest): } of unreachable relays :param args: - The :class:`BridgetAsset` line currently being used. + The :class:`BridgetAsset` line currently being used. Except that it + in Bridget it doesn't, so it should be ignored and avoided. """ try: - from ooni.utils.onion import start_tor, singleton_semaphore - from ooni.utils.onion import setup_done, setup_fail - from ooni.utils.onion import CustomCircuit - from ooni.lib.txtorcon import TorConfig, TorState + from ooni.utils.process import singleton_semaphore + from ooni.utils.onion import start_tor, setup_done, setup_fail + from ooni.utils.onion import CustomCircuit + from ooni.lib.txtorcon import TorConfig, TorState except ImportError: raise TxtorconImportError except TxtorconImportError, tie: @@ -509,7 +418,7 @@ class BridgetTest(OONITest): log.msg("Bridget: initiating test ... ") - d = defer.Deferred + x = defer.Deferred if self.bridges_remaining() > 0 and not 'Bridge' in self.config.config: self.config.Bridge = self.bridges.pop() @@ -527,13 +436,13 @@ class BridgetTest(OONITest): setup_fail) self.tor_process_semaphore = True - run_once = d().addCallback(singleton_semaphore, tor) + run_once = x().addCallback(singleton_semaphore, tor) run_once.addErrback(setup_fail) - only_bridges = d().addCallback(remove_public_relays, self.bridges) + filter_bridges = x().addCallback(remove_public_relays, self.bridges) - state = defer.gatherResults([run_once, only_bridges], consumeErrors=True) - log.debug("%s" % state.callbacks) + state = defer.gatherResults([run_once, filter_bridges], consumeErrors=True) + log.debug("Current callbacks on TorState():\n%s" % state.callbacks) if self.bridges_remaining() > 0: all = [] @@ -541,7 +450,7 @@ class BridgetTest(OONITest): self.current_bridge = bridge log.msg("We now have %d untested bridges..." % self.bridges_remaining()) - reconf = d().addCallback(reconfigure_bridge, state, + reconf = x().addCallback(reconfigure_bridge, state, self.current_bridge, self.use_pt, self.pt_type) @@ -549,10 +458,14 @@ class BridgetTest(OONITest): self.bridges_up) reconf.addErrback(reconfigure_fail, self.current_bridge, self.bridges_down) - all.append(reconf) - state.chainDeferred(defer.DeferredList(all)) - log.debug("%s" % state.callbacks) - + all.append(reconf) + + #state.chainDeferred(defer.DeferredList(all)) + #state.chainDeferred(defer.gatherResults(all, consumeErrors=True)) + n_plus_one_bridges = defer.gatherResults(all, consumeErrors=True) + state.chainDeferred(n_plus_one_bridges) + log.debug("Current callbacks on TorState():\n%s" % state.callbacks) + if self.relays_remaining() > 0: while self.relays_remaining() >= 3: #path = list(self.relays.pop() for i in range(3)) @@ -575,11 +488,16 @@ class BridgetTest(OONITest): else: continue - #reactor.run() - return state + state.callback(all) + self.reactor.run() + #return state - def control(self, experiment_result, args): - experiment_result.callback + def startTest(self, args): + self.start_time = date.now() + log.msg("Starting %s" % self.shortName) + self.do_science = self.experiment(args) + self.do_science.addCallback(self.finished).addErrback(log.err) + return self.do_science ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) diff --git a/ooni/plugoo/assets.py b/ooni/plugoo/assets.py index eded997..3974230 100644 --- a/ooni/plugoo/assets.py +++ b/ooni/plugoo/assets.py @@ -54,3 +54,8 @@ class Asset: except: raise StopIteration +class MissingAssetException(Exception): + """Raised when an Asset necessary for running the Test is missing.""" + def __init__(self, error_message): + log.msg(error_message) + return sys.exit() diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 42f9542..92bf88f 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -38,7 +38,6 @@ class OONITest(object): #self.ooninet = ooninet self.reactor = reactor self.result = {} - self.initialize() self.assets = self.load_assets() diff --git a/ooni/utils/config.py b/ooni/utils/config.py index ab43e66..7c28f33 100644 --- a/ooni/utils/config.py +++ b/ooni/utils/config.py @@ -1,4 +1,6 @@ import ConfigParser +import os + from ooni.utils import Storage class Config(Storage): @@ -51,3 +53,66 @@ class Config(Storage): self._cfgparser.write(cfgfile) finally: cfgfile.close() + +class ValueChecker(object): + """ + A class for general purpose value checks on commandline parameters + passed to subclasses of :class:`twisted.python.usage.Options`. + """ + def __init__(self, coerce_doc=None): + self.coerce_doc = coerce_doc + + def port_check(self, port, range_min=1024, range_max=65535): + """ + Check that given ports are in the allowed range for an unprivileged + user. + + :param port: + The port to check. + :param range_min: + The minimum allowable port number. + :param range_max: + The minimum allowable port number. + :param coerce_doc: + The documentation string to show in the optParameters menu, see + :class:`twisted.python.usage.Options`. + """ + if self.coerce_doc is not None: + coerceDoc = self.coerce_doc + + assert type(port) is int + if port not in range(range_min, range_max): + raise ValueError("Port out of range") + log.err() + + def uid_check(self, error_message): + """ + Check that we're not root. If we are, setuid(1000) to normal user if + we're running on a posix-based system, and if we're on Windows just + tell the user that we can't be run as root with the specified options + and then exit. + + :param error_message: + The string to log as an error message when the uid check fails. + """ + uid, gid = os.getuid(), os.getgid() + if uid == 0 and gid == 0: + log.msg(error_message) + if os.name == 'posix': + log.msg("Dropping privileges to normal user...") + os.setgid(1000) + os.setuid(1000) + else: + sys.exit(0) + + def dir_check(self, d): + """Check that the given directory exists.""" + if not os.path.isdir(d): + raise ValueError("%s doesn't exist, or has wrong permissions" + % d) + + def file_check(self, f): + """Check that the given file exists.""" + if not os.path.isfile(f): + raise ValueError("%s does not exist, or has wrong permissions" + % f) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 5f15e90..81e4ea3 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -44,12 +44,34 @@ def state_complete(state): for circ in state.circuits.values(): log.msg("%s" % circ) - return state + #return state def updates(_progress, _tag, _summary): """Log updates on the Tor bootstrapping process.""" log.msg("%d%%: %s" % (_progress, _summary)) +def parse_data_dir(data_dir): + """ + Parse a string that a has been given as a DataDirectory and determine + its absolute path on the filesystem. + + :param data_dir: + A directory for Tor's DataDirectory, to be parsed. + :return: + The absolute path of :param:data_dir. + """ + import os + + if data_dir.startswith('~'): + data_dir = os.path.expanduser(data_dir) + elif data_dir.startswith('/'): + data_dir = os.path.join(os.getcwd(), data_dir) + elif data_dir.startswith('./'): + data_dir = os.path.abspath(data_dir) + else: + data_dir = os.path.join(os.getcwd(), data_dir) + return data_dir + def write_torrc(conf, data_dir=None): """ Create a torrc in our data_dir. If we don't yet have a data_dir, create a @@ -103,38 +125,6 @@ def delete_files_or_dirs(delete_list): rmtree(temp, ignore_errors=True) @defer.inlineCallbacks -def singleton_semaphore(deferred_process_init, callbacks=[], errbacks=[]): - """ - Initialize a process only once, and do not return until - that initialization is complete. - - :param deferred_process_init: - A deferred which returns a connected process via - :meth:`twisted.internet.reactor.spawnProcess`. - :param callbacks: - A list of callback functions to add to the initialized processes' - deferred. - :param errbacks: - A list of errback functions to add to the initialized processes' - deferred. - :return: - The final state of the :param deferred_process_init: after the - callback chain has completed. This should be a fully initialized - process connected to a :class:`twisted.internet.reactor`. - """ - assert type(callbacks) is list - assert type(errbacks) is list - - for cb in callbacks: - deferred_process_init.addCallback(cb) - for eb in errbacks: - deferred_process_init.addErrback(eb) - - only_once = defer.DeferredSemaphore(1) - singleton = yield only_once.run(deferred_process_init) - defer.returnValue(singleton) - -(a)defer.inlineCallbacks def start_tor(reactor, config, control_port, tor_binary, data_dir, report=None, progress=updates, process_cb=setup_done, process_eb=setup_fail): @@ -237,7 +227,6 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir, #return process_protocol.connected_cb.addCallback(process_cb).addErrback(process_eb) - class CustomCircuit(CircuitListenerMixin): implements(IStreamAttacher) @@ -328,3 +317,36 @@ class CustomCircuit(CircuitListenerMixin): return self.state.build_circuit(path).addCallback( AppendWaiting(self, deferred)).addErrback( log.err) + +class TxtorconImportError(ImportError): + """Raised when /ooni/lib/txtorcon cannot be imported from.""" + from os import getcwd, path + + cwd, tx = getcwd(), 'lib/txtorcon/torconfig.py' + try: + log.msg("Unable to import from ooni.lib.txtorcon") + if cwd.endswith('ooni'): + check = path.join(cwd, tx) + elif cwd.endswith('utils'): + check = path.join(cwd, '../'+tx) + else: + check = path.join(cwd, 'ooni/'+tx) + assert path.isfile(check) + except: + log.msg("Error: Some OONI libraries are missing!") + log.msg("Please go to /ooni/lib/ and do \"make all\"") + +class PTNoBridgesException(Exception): + """Raised when a pluggable transport is specified, but not bridges.""" + def __init__(self): + log.msg("Pluggable transport requires the bridges option") + return sys.exit() + +class PTNotFoundException(Exception): + def __init__(self, transport_type): + m = "Pluggable Transport type %s was unaccounted " % transport_type + m += "for, please contact isis(at)torproject(dot)org and it will " + m += "get included." + log.msg("%s" % m) + return sys.exit() + diff --git a/ooni/utils/process.py b/ooni/utils/process.py new file mode 100644 index 0000000..e9f56a0 --- /dev/null +++ b/ooni/utils/process.py @@ -0,0 +1,54 @@ +# +# process.py +# ---------- +# OONI utilities for dealing with starting and stopping processes. +# +# :author: Isis Lovecruft +# :version: 0.1.0-pre-alpha +# :license: see include LICENSE file +# :copyright: copyright (c) 2012, Isis Lovecruft, The Tor Project Inc. +# + +from twisted.internet import defer + +(a)defer.inlineCallbacks +def singleton_semaphore(deferred_process_init, + callbacks=[], errbacks=[], + max_inits=1): + """ + Initialize a process only once, and do not return until + that initialization is complete. If the keyword parameter max_inits= + is given, run the process a maximum of that number of times. + + :param deferred_process_init: + A deferred which returns a connected process via + :meth:`twisted.internet.reactor.spawnProcess`. + :param callbacks: + A list of callback functions to add to the initialized processes' + deferred. + :param errbacks: + A list of errback functions to add to the initialized processes' + deferred. + :param max_inits: + An integer specifying the maximum number of allowed + initializations for :param:deferred_process_init. If no maximum + is given, only one instance (a singleton) of the process is + initialized. + :return: + The final state of the :param deferred_process_init: after the + callback chain has completed. This should be a fully initialized + process connected to a :class:`twisted.internet.reactor`. + """ + assert type(callbacks) is list + assert type(errbacks) is list + assert type(max_inits) is int + + for cb in callbacks: + deferred_process_init.addCallback(cb) + for eb in errbacks: + deferred_process_init.addErrback(eb) + + only_this_many = defer.DeferredSemaphore(max_inits) + singleton = yield only_this_many.run(deferred_process_init) + defer.returnValue(singleton) + diff --git a/ooni/utils/timer.py b/ooni/utils/timer.py new file mode 100644 index 0000000..8ad9b79 --- /dev/null +++ b/ooni/utils/timer.py @@ -0,0 +1,36 @@ +# +# timer.py +# ---------- +# OONI utilities for adding timeouts to functions and to Deferreds. +# +# :author: Isis Lovecruft +# :version: 0.1.0-pre-alpha +# :license: see include LICENSE file +# :copyright: copyright (c) 2012, Isis Lovecruft, The Tor Project Inc. +# + + +class TimeoutError(Exception): + """Raised when a timer runs out.""" + pass + +def timeout(secs, e=None): + """ + A decorator for blocking methods to cause them to timeout. + """ + import signal + import functools.wraps + def decorator(func): + def _timeout(signum, frame): + raise TimeoutError, e + + def wrapper(*args, **kwargs): + signal.signal(signal.SIGALRM, _timeout) + signal.alarm(secs) + try: + res = func(*args, **kwargs) + finally: + signal.alarm(0) + return res + return functools.wraps(func)(wrapper) + return decorator

1 0