commit 7e4f2834a3a2de48ccb77b0de42a24c78b3fc51a Merge: f9f97ae 05dd861 Author: Isis Lovecruft isis@torproject.org Date: Thu Oct 4 13:47:47 2012 +0000

Merge branch 'trial-refactor' into bridget-pt-merge-trial

Conflicts: ooni/plugins/new_bridget.py ooni/utils/log.py

.gitignore | 3 +- README.md | 18 + bin/ooniprobe | 28 +- docs/design.dia | Bin 1706 -> 1706 bytes docs/source/api/ooni.rst | 60 + docs/source/api/ooni.templates.rst | 11 + docs/source/api/ooni.utils.rst | 51 + docs/source/conf.py | 14 +- docs/source/index.rst | 101 +- docs/source/install.rst | 55 + docs/source/tutorial.rst | 1 - docs/source/writing_tests.rst | 88 +- docs/writing_tests.md | 17 - nettests/example_httpt.py | 32 + nettests/example_scapyt.py | 15 + nettests/myip.py | 18 + ooni/assets/captive_portal_tests.txt | 4 - ooni/assets/example.txt | 2 - ooni/assets/greatfire.lst | 2100 -------------------- ooni/assets/redirects.yaml | 17 - ooni/assets/tcpscan.txt | 2 - ooni/inputunit.py | 65 + ooni/lib/txscapy.py | 40 +- ooni/nettest.py | 90 + ooni/oonicli.py | 105 + ooni/ooniprobe.py | 14 +- ooni/oonitests/bridget.py | 373 ---- ooni/plugins/bridget.py | 6 +- ooni/plugins/domclass.py | 35 +- ooni/plugins/httpt.py | 9 +- ooni/plugoo/interface.py | 1 + ooni/plugoo/reports.py | 51 +- ooni/plugoo/tests.py | 16 +- ooni/plugoo/work.py | 2 - ooni/protocols/http.py | 17 +- ooni/reporter.py | 212 ++ ooni/runner.py | 193 ++ ooni/scaffolding.py | 10 +- ooni/templates/httpt.py | 165 ++ ooni/templates/scapyt.py | 69 + ooni/utils/__init__.py | 1 - ooni/utils/hacks.py | 53 + ooni/utils/log.py | 10 +- 43 files changed, 1507 insertions(+), 2667 deletions(-)

diff --cc ooni/plugins/bridget.py index 92748c1,0000000..188009c mode 100644,000000..100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@@ -1,496 -1,0 +1,496 @@@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- - # ++# +# +-----------+ +# | BRIDGET | +# | +--------------------------------------------+ +# +--------| Use a Tor process to test making a Tor | +# | connection to a list of bridges or relays. | +# +--------------------------------------------+ +# +# :authors: Isis Lovecruft, Arturo Filasto +# :licence: see included LICENSE +# :version: 0.1.0-alpha + +from __future__ import with_statement +from functools import partial +from random import randint + +import os +import sys + +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor +from zope.interface import implements + +from ooni.utils import log, date +from ooni.utils.config import ValueChecker +from ooni.utils.onion import parse_data_dir +from ooni.utils.onion import TxtorconImportError +from ooni.utils.onion import PTNoBridgesException, PTNotFoundException +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset, MissingAssetException + + +class RandomPortException(Exception): + """Raised when using a random port conflicts with configured ports.""" + def __init__(self): + log.msg("Unable to use random and specific ports simultaneously") + return sys.exit() + +class BridgetArgs(usage.Options): + """Commandline options.""" + allowed = "Port to use for Tor's %s, must be between 1024 and 65535." + sock_check = ValueChecker(allowed % "SocksPort").port_check + ctrl_check = ValueChecker(allowed % "ControlPort").port_check + + optParameters = [ + ['bridges', 'b', None, + 'File listing bridge IP:ORPorts to test'], + ['relays', 'f', None, + 'File listing relay IPs to test'], + ['socks', 's', 9049, None, sock_check], + ['control', 'c', 9052, None, ctrl_check], + ['torpath', 'p', None, + 'Path to the Tor binary to use'], + ['datadir', 'd', None, + 'Tor DataDirectory to use'], + ['transport', 't', None, + 'Tor ClientTransportPlugin'], + ['resume', 'r', 0, + 'Resume at this index']] + optFlags = [['random', 'x', 'Use random ControlPort and SocksPort']] + + def postOptions(self): + if not self['bridges'] and not self['relays']: + raise MissingAssetException( + "Bridget can't run without bridges or relays to test!") + if self['transport']: + ValueChecker.uid_check( + "Can't run bridget as root with pluggable transports!") + if not self['bridges']: + raise PTNoBridgesException + if self['socks'] or self['control']: + if self['random']: + raise RandomPortException + if self['datadir']: + ValueChecker.dir_check(self['datadir']) + if self['torpath']: + ValueChecker.file_check(self['torpath']) + +class BridgetAsset(Asset): + """Class for parsing bridget Assets ignoring commented out lines.""" + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + if line.startswith('#'): + return + else: + return line.replace('\n','') + +class BridgetTest(OONITest): + """ + XXX fill me in + - :ivar config: ++ :ivar config: + An :class:`ooni.lib.txtorcon.TorConfig` instance. + :ivar relays: + A list of all provided relays to test. + :ivar bridges: + A list of all provided bridges to test. + :ivar socks_port: + Integer for Tor's SocksPort. + :ivar control_port: + Integer for Tor's ControlPort. + :ivar transport: + String defining the Tor's ClientTransportPlugin, for testing + a bridge's pluggable transport functionality. + :ivar tor_binary: + Path to the Tor binary to use, e.g. '/usr/sbin/tor' + """ + implements(IPlugin, ITest) + + shortName = "bridget" + description = "Use a Tor process to test connecting to bridges or relays" + requirements = None + options = BridgetArgs + blocking = False + + def initialize(self): + """ + Extra initialization steps. We only want one child Tor process + running, so we need to deal with most of the TorConfig() only once, + before the experiment runs. + """ + self.socks_port = 9049 + self.control_port = 9052 + self.circuit_timeout = 90 + self.tor_binary = '/usr/sbin/tor' + self.data_directory = None + + def __make_asset_list__(opt, lst): + log.msg("Loading information from %s ..." % opt) + with open(opt) as opt_file: + for line in opt_file.readlines(): + if line.startswith('#'): + continue + else: + lst.append(line.replace('\n','')) + + def __count_remaining__(which): + total, reach, unreach = map(lambda x: which[x], + ['all', 'reachable', 'unreachable']) + count = len(total) - reach() - unreach() + return count + + ## XXX should we do report['bridges_up'].append(self.bridges['current']) + self.bridges = {} + self.bridges['all'], self.bridges['up'], self.bridges['down'] = \ + ([] for i in range(3)) + self.bridges['reachable'] = lambda: len(self.bridges['up']) + self.bridges['unreachable'] = lambda: len(self.bridges['down']) + self.bridges['remaining'] = lambda: __count_remaining__(self.bridges) + self.bridges['current'] = None + self.bridges['pt_type'] = None + self.bridges['use_pt'] = False + + self.relays = {} + self.relays['all'], self.relays['up'], self.relays['down'] = \ + ([] for i in range(3)) + self.relays['reachable'] = lambda: len(self.relays['up']) + self.relays['unreachable'] = lambda: len(self.relays['down']) + self.relays['remaining'] = lambda: __count_remaining__(self.relays) + self.relays['current'] = None + + if self.local_options: + try: + from ooni.lib.txtorcon import TorConfig + except ImportError: + raise TxtorconImportError + else: + self.config = TorConfig() + finally: + options = self.local_options + + if options['bridges']: + self.config.UseBridges = 1 + __make_asset_list__(options['bridges'], self.bridges['all']) + if options['relays']: + ## first hop must be in TorState().guards + self.config.EntryNodes = ','.join(relay_list) + __make_asset_list__(options['relays'], self.relays['all']) + if options['socks']: + self.socks_port = options['socks'] + if options['control']: + self.control_port = options['control'] + if options['random']: + log.msg("Using randomized ControlPort and SocksPort ...") + self.socks_port = randint(1024, 2**16) + self.control_port = randint(1024, 2**16) + if options['torpath']: + self.tor_binary = options['torpath'] + if options['datadir']: + self.data_directory = parse_data_dir(options['datadir']) + if options['transport']: + ## ClientTransportPlugin transport exec pathtobinary [options] + ## XXX we need a better way to deal with all PTs + log.msg("Using ClientTransportPlugin %s" % options['transport']) + self.bridges['use_pt'] = True + [self.bridges['pt_type'], pt_exec] = \ + options['transport'].split(' ', 1) + + if self.bridges['pt_type'] == "obfs2": + self.config.ClientTransportPlugin = \ + self.bridges['pt_type'] + " " + pt_exec + else: + raise PTNotFoundException + + self.config.SocksPort = self.socks_port + self.config.ControlPort = self.control_port + self.config.CookieAuthentication = 1 + + def __load_assets__(self): + """ + Load bridges and/or relays from files given in user options. Bridges + should be given in the form IP:ORport. We don't want to load these as + assets, because it's inefficient to start a Tor process for each one. + + We cannot use the Asset model, because that model calls + self.experiment() with the current Assets, which would be one relay + and one bridge, then it gives the defer.Deferred returned from + self.experiment() to self.control(), which means that, for each + (bridge, relay) pair, experiment gets called again, which instantiates + an additional Tor process that attempts to bind to the same + ports. Thus, additionally instantiated Tor processes return with + RuntimeErrors, which break the final defer.chainDeferred.callback(), + sending it into the errback chain. + """ + assets = {} + if self.local_options: + if self.local_options['bridges']: + assets.update({'bridge': + BridgetAsset(self.local_options['bridges'])}) + if self.local_options['relays']: + assets.update({'relay': + BridgetAsset(self.local_options['relays'])}) + return assets + + def experiment(self, args): + """ + if bridges: + 1. configure first bridge line + 2a. configure data_dir, if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor } if any of these + 4. remove bridges which are public relays } fail, add current + 5. SIGHUP for each bridge } bridge to unreach- + } able bridges. + if relays: + 1a. configure the data_dir, if it doesn't exist + 1b. write torrc to a tempfile in data_dir + 2. start tor + 3. remove any of our relays which are already part of current + circuits + 4a. attach CustomCircuit() to self.state + 4b. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays + 5. + if bridges and relays: + 1. configure first bridge line + 2a. configure data_dir if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor + 4. remove bridges which are public relays + 5. remove any of our relays which are already part of current + circuits + 6a. attach CustomCircuit() to self.state + 6b. for each bridge, build three circuits, with three + relays each + 6c. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays + + :param args: + The :class:`BridgetAsset` line currently being used. Except that it + in Bridget it doesn't, so it should be ignored and avoided. + """ + try: + from ooni.utils import process + from ooni.utils.onion import remove_public_relays, start_tor + from ooni.utils.onion import start_tor_filter_nodes + from ooni.utils.onion import setup_fail, setup_done + from ooni.utils.onion import CustomCircuit + from ooni.utils.timer import deferred_timeout, TimeoutError + from ooni.lib.txtorcon import TorConfig, TorState + except ImportError: + raise TxtorconImportError + except TxtorconImportError, tie: + log.err(tie) + sys.exit() + + def reconfigure_done(state, bridges): + """ + Append :ivar:`bridges['current']` to the list + :ivar:`bridges['up']. + """ + log.msg("Reconfiguring with 'Bridge %s' successful" + % bridges['current']) + bridges['up'].append(bridges['current']) + return state + + def reconfigure_fail(state, bridges): + """ + Append :ivar:`bridges['current']` to the list + :ivar:`bridges['down']. + """ + log.msg("Reconfiguring TorConfig with parameters %s failed" + % state) + bridges['down'].append(bridges['current']) + return state + + @defer.inlineCallbacks + def reconfigure_bridge(state, bridges): + """ + Rewrite the Bridge line in our torrc. If use of pluggable + transports was specified, rewrite the line as: + Bridge <transport_type> <IP>:<ORPort> + Otherwise, rewrite in the standard form: + Bridge <IP>:<ORPort> + + :param state: + A fully bootstrapped instance of + :class:`ooni.lib.txtorcon.TorState`. + :param bridges: + A dictionary of bridges containing the following keys: + + bridges['remaining'] :: A function returning and int for the + number of remaining bridges to test. + bridges['current'] :: A string containing the <IP>:<ORPort> + of the current bridge. + bridges['use_pt'] :: A boolean, True if we're testing + bridges with a pluggable transport; + False otherwise. + bridges['pt_type'] :: If :ivar:`bridges['use_pt'] is True, + this is a string containing the type + of pluggable transport to test. + :return: + :param:`state` + """ + log.msg("Current Bridge: %s" % bridges['current']) + log.msg("We now have %d bridges remaining to test..." + % bridges['remaining']()) + try: + if bridges['use_pt'] is False: + controller_response = yield state.protocol.set_conf( + 'Bridge', bridges['current']) + elif bridges['use_pt'] and bridges['pt_type'] is not None: + controller_reponse = yield state.protocol.set_conf( + 'Bridge', bridges['pt_type'] +' '+ bridges['current']) + else: + raise PTNotFoundException + + if controller_response == 'OK': + finish = yield reconfigure_done(state, bridges) + else: + log.err("SETCONF for %s responded with error:\n %s" + % (bridges['current'], controller_response)) + finish = yield reconfigure_fail(state, bridges) + + defer.returnValue(finish) + + except Exception, e: + log.err("Reconfiguring torrc with Bridge line %s failed:\n%s" + % (bridges['current'], e)) + defer.returnValue(None) + + def attacher_extend_circuit(attacher, deferred, router): + ## XXX todo write me + ## state.attacher.extend_circuit + raise NotImplemented + #attacher.extend_circuit + + def state_attach(state, path): + log.msg("Setting up custom circuit builder...") + attacher = CustomCircuit(state) + state.set_attacher(attacher, reactor) + state.add_circuit_listener(attacher) + return state + + ## OLD + #for circ in state.circuits.values(): + # for relay in circ.path: + # try: + # relay_list.remove(relay) + # except KeyError: + # continue + ## XXX how do we attach to circuits with bridges? + d = defer.Deferred() + attacher.request_circuit_build(d) + return d + + def state_attach_fail(state): + log.err("Attaching custom circuit builder failed: %s" % state) + + log.msg("Bridget: initiating test ... ") ## Start the experiment + + ## if we've at least one bridge, and our config has no 'Bridge' line + if self.bridges['remaining']() >= 1 \ + and not 'Bridge' in self.config.config: + + ## configure our first bridge line + self.bridges['current'] = self.bridges['all'][0] + self.config.Bridge = self.bridges['current'] + ## avoid starting several + self.config.save() ## processes + assert self.config.config.has_key('Bridge'), "No Bridge Line" + + ## start tor and remove bridges which are public relays + from ooni.utils.onion import start_tor_filter_nodes + state = start_tor_filter_nodes(reactor, self.config, + self.control_port, self.tor_binary, + self.data_directory, self.bridges) + #controller = defer.Deferred() + #controller.addCallback(singleton_semaphore, tor) + #controller.addErrback(setup_fail) + #bootstrap = defer.gatherResults([controller, filter_bridges], + # consumeErrors=True) + + if state is not None: + log.debug("state:\n%s" % state) + log.debug("Current callbacks on TorState():\n%s" + % state.callbacks) + + ## if we've got more bridges + if self.bridges['remaining']() >= 2: + #all = [] + for bridge in self.bridges['all'][1:]: + self.bridges['current'] = bridge + #new = defer.Deferred() + #new.addCallback(reconfigure_bridge, state, self.bridges) + #all.append(new) + #check_remaining = defer.DeferredList(all, consumeErrors=True) + #state.chainDeferred(check_remaining) + state.addCallback(reconfigure_bridge, self.bridges) + + if self.relays['remaining']() > 0: + while self.relays['remaining']() >= 3: + #path = list(self.relays.pop() for i in range(3)) + #log.msg("Trying path %s" % '->'.join(map(lambda node: + # node, path))) + self.relays['current'] = self.relays['all'].pop() + for circ in state.circuits.values(): + for node in circ.path: + if node == self.relays['current']: + self.relays['up'].append(self.relays['current']) + if len(circ.path) < 3: + try: + ext = attacher_extend_circuit(state.attacher, circ, + self.relays['current']) + ext.addCallback(attacher_extend_circuit_done, + state.attacher, circ, + self.relays['current']) + except Exception, e: + log.err("Extend circuit failed: %s" % e) + else: + continue + + #state.callback(all) + #self.reactor.run() + return state + + def startTest(self, args): + """ + Local override of :meth:`OONITest.startTest` to bypass calling + self.control. + + :param args: + The current line of :class:`Asset`, not used but kept for + compatibility reasons. + :return: + A fired deferred which callbacks :meth:`experiment` and + :meth:`OONITest.finished`. + """ + self.start_time = date.now() + self.d = self.experiment(args) + self.d.addErrback(log.err) + self.d.addCallbacks(self.finished, log.err) + return self.d + +## So that getPlugins() can register the Test: +bridget = BridgetTest(None, None, None) + + +## ISIS' NOTES +## ----------- - ## +## TODO: +## x cleanup documentation +## x add DataDirectory option +## x check if bridges are public relays +## o take bridge_desc file as input, also be able to give same +## format as output +## x Add asynchronous timeout for deferred, so that we don't wait ++## o Add assychronous timout for deferred, so that we don't wait +## forever for bridges that don't work. diff --cc ooni/utils/log.py index fe737f5,6ff6bf9..63b7c27 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@@ -36,11 -36,11 +36,11 @@@ class OONITestFailure(Failure) Can be given an Exception as an argument, else will use the most recent Exception from the current stack frame. """ - def __init__(self, _type=None, + def __init__(self, exception=None, _type=None, _traceback=None, _capture=False): - Failure.__init__(self, exc_value=exception, exc_type=_type, + Failure.__init__(self, exc_type=_type, exc_tb=_traceback, captureVars=_capture) - + class OONILogObserver(log.FileLogObserver): """ Supports logging level verbosity.