commit 2e49b7330b155ca4746896939ebe48ce67f147b3 Merge: 0080bf6 62956eb Author: Arturo Filastò arturo@filasto.net Date: Thu Sep 13 23:16:03 2012 +0000
Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe
Conflicts: ooni/assets/bridgetests.txt ooni/ooniprobe.py ooni/plugins/domclass.py ooni/plugins/new_bridget.py
.gitmodules | 9 + README.md | 22 ++- TODO | 26 +-- ooni/__init__.py | 2 +- ooni/assets/bridgetests.txt | 8 + ooni/example_plugins/skel.py | 4 +- ooni/lib/Makefile | 30 --- ooni/lib/__init__.py | 41 +++- ooni/lib/txscapy | 1 + ooni/lib/txscapy.py | 363 ------------------------------- ooni/lib/txtorcon | 1 + ooni/lib/txtraceroute | 1 + ooni/ooniprobe.log | 1 - ooni/ooniprobe.py | 19 ++- ooni/oonitests/dnstamper.py | 142 ++++++++----- ooni/plugins/blocking.py | 4 +- ooni/plugins/dnstamper.py | 343 ++++++++++++++++++++++++++++++ ooni/plugins/domclass.py | 3 +- ooni/plugins/httphost.py | 4 +- ooni/plugins/new_bridget.py | 484 ++++++++++++++++++++++++++++++++++-------- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 25 ++- ooni/plugoo/work.py | 2 + ooni/utils/log.py | 116 ++++++++--- 24 files changed, 1045 insertions(+), 608 deletions(-)
diff --cc ooni/assets/bridgetests.txt index 5519eea,7bba841..b69fbca --- a/ooni/assets/bridgetests.txt +++ b/ooni/assets/bridgetests.txt @@@ -1,3 -1,5 +1,11 @@@ ++<<<<<<< HEAD +88.130.86.191:443 +195.74.237.236:9001 +127.0.0.1:9050 ++======= + #213.151.89.102:9001 + #108.166.106.156:443 + #217.150.224.213:443 + 85.193.252.111:443 + #68.98.41.14:9001 ++>>>>>>> 62956ebab7779c1b61ce3d6e8ac750552fd1c988 diff --cc ooni/plugins/new_bridget.py index 3e4db56,0000000..736b084 mode 100644,000000..100644 --- a/ooni/plugins/new_bridget.py +++ b/ooni/plugins/new_bridget.py @@@ -1,105 -1,0 +1,421 @@@ - """ - This is a self genrated test created by scaffolding.py. - you will need to fill it up with all your necessities. - Safe hacking :). - """ - from exceptions import Exception - from datetime import datetime - from zope.interface import implements - from twisted.python import usage - from twisted.plugin import IPlugin - from twisted.internet import reactor, task - - from ooni.utils import log - from ooni.plugoo.tests import ITest, OONITest - from ooni.plugoo.assets import Asset - - from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState - from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor - - class bridgetArgs(usage.Options): - optParameters = [['bridges', 'b', None, 'List of bridges to scan'], - ['relays', 'f', None, 'List of relays to scan'], - ['resume', 'r', 0, 'Resume at this index'], - ['timeout', 't', 5, 'Timeout in seconds after which to consider a bridge not working'] - ] - - class bridgetTest(OONITest): ++#!/usr/bin/env python ++# -*- encoding: utf-8 -*- ++# ++# +-----------+ ++# | BRIDGET | ++# | +----------------------------------------------+ ++# +--------| Use a slave Tor process to test making a Tor | ++# | connection to a list of bridges or relays. | ++# +----------------------------------------------+ ++# ++# :authors: Arturo Filasto, Isis Lovecruft ++# :licence: see included LICENSE ++# :version: 0.1.0-alpha ++ ++from __future__ import with_statement ++from zope.interface import implements ++from twisted.python import usage ++from twisted.plugin import IPlugin ++from twisted.internet import defer, error, reactor ++ ++import random ++import sys ++ ++try: ++ from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher ++except: ++ print "BridgeT requires txtorcon: https://github.com/meejah/txtorcon.git" ++ print "Your copy of OONI should have it included, if you're seeing this" ++ print "message, please file a bug report." ++ log.msg ("Bridget: Unable to import from ooni.lib.txtorcon") ++ ++from ooni.utils import log ++from ooni.plugoo.tests import ITest, OONITest ++from ooni.plugoo.assets import Asset ++ ++ ++class BridgetArgs(usage.Options): ++ ++ def portCheck(number): ++ number = int(number) ++ if number not in range(1024, 65535): ++ raise ValueError("Port out of range") ++ portCheck.coerceDoc = "Ports must be between 1024 and 65535" ++ ++ optParameters = [ ++ ['bridges', 'b', None, ++ 'List of bridges to scan <IP>:<ORport>'], ++ ['relays', 'f', None, ++ 'List of relays to scan <IP>'], ++ ['socks', 's', 9049, portCheck, ++ 'Tor SocksPort to use'], ++ ['control', 'c', 9052, portCheck, ++ 'Tor ControlPort to use'], ++ ['tor-path', 'p', None, ++ 'Path to the Tor binary to use'], ++ ['data-dir', 'd', None, ++ 'Tor DataDirectory to use'], ++ ['transport', 't', None, ++ 'Tor ClientTransportPlugin'], ++ ['resume', 'r', 0, ++ 'Resume at this index']] ++ optFlags = [['random', 'x', 'Randomize control and socks ports']] ++ ++ def postOptions(self): ++ ## We can't test pluggable transports without bridges ++ if self['transport'] and not self['bridges']: ++ e = "Pluggable transport requires the bridges option" ++ raise usage.UsageError, e ++ ## We can't use random and static port simultaneously ++ if self['socks'] and self['control']: ++ if self['random']: ++ e = "Unable to use random and specific ports simultaneously" ++ raise usageError, e ++ ++class CustomCircuit(CircuitListenerMixin): ++ implements(IStreamAttacher) ++ ++ from txtorcon.interface import IRouterContainer, ICircuitContainer ++ ++ def __init__(self, state): ++ self.state = state ++ self.waiting_circuits = [] ++ ++ def waiting_on(self, circuit): ++ for (circid, d) in self.waiting_circuits: ++ if circuit.id == circid: ++ return true ++ return False ++ ++ def circuit_extend(self, circuit, router): ++ "ICircuitListener" ++ if circuit.purpose != 'GENERAL': ++ return ++ if self.waiting_on(circuit): ++ log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) ++ ++ def circuit_built(self, circuit): ++ "ICircuitListener" ++ if circuit.purpose != 'GENERAL': ++ return ++ ++ log.msg("Circuit %s built ..." % circuit.id) ++ log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) ++ ++ for (circid, d) in self.waiting_circuits: ++ if circid == circuit.id: ++ self.waiting_circuits.remove(circid, d) ++ d.callback(circuit) ++ ++ def circuit_failed(self, circuit, reason): ++ if self.waiting_on(circuit): ++ log.msg("A circuit we requested %s failed for reason %s" % ++ (circuit.id, reason)) ++ circid, d = None, None ++ for x in self.waiting_circuits: ++ if x[0] == circuit.id: ++ circid, d, stream_cc = x ++ if d is None: ++ raise Exception("Expected to find circuit.") ++ ++ self.waiting_circuits.remove((circid, d)) ++ log.msg("Trying to build a circuit for %s" % circid) ++ self.request_circuit_build(d) ++ ++ def check_circuit_route(self, circuit, router): ++ if router in circuit.path: ++ #router.update() ## XXX can i use without args? no. ++ TorInfo.dump(self) ++ ++ def request_circuit_build(self, deferred): ++ entries = self.state.entry_guards.value() ++ relays = self.state.routers.values() ++ ++ log.msg("We have these nodes listed as entry guards:") ++ log.msg("%s" % entries) ++ log.msg("We have these nodes listed as relays:") ++ log.msg("%s" % relays) ++ ++ path = [random.choice(entries), ++ random.choice(relays), ++ random.choice(relays)] ++ ++ log.msg("Requesting a circuit: %s" ++ % '-->'.join(map(lambda x: x.location.countrycode, path))) ++ ++ class AppendWaiting: ++ def __init__(self, attacher, deferred): ++ self.attacher = attacher ++ self.d = deferred ++ ++ def __call__(self, circuit): ++ """ ++ Return from build_circuit is a Circuit, however, we want to ++ wait until it is built before we can issue an attach on it and ++ callback to the Deferred we issue here. ++ """ ++ log.msg("Circuit %s is in progress ..." % circuit.id) ++ self.attacher.waiting_circuits.append((circuit.id, self.d)) ++ ++ return self.state.build_circuit(path).addCallback(AppendWaiting(self, deferred_to_callback)).addErrback(log.err) ++ ++class BridgetAsset(Asset): ++ """ ++ Class for parsing bridge assets so that they can be commented out. ++ """ ++ def __init__(self, file=None): ++ self = Asset.__init__(self, file) ++ ++ def parse_line(self, line): ++ if line.startswith('#'): ++ return ++ else: ++ return line.replace('\n','') ++ ++class BridgetTest(OONITest): ++ """ ++ XXX fill me in ++ ++ :ivar config: ++ An :class:`ooni.lib.txtorcon.TorConfig` instance. ++ :ivar relay_list: ++ A list of all provided relays to test. We have to do this because ++ txtorcon.TorState().entry_guards won't build a custom circuit if the ++ first hop isn't in the torrc's EntryNodes. ++ :ivar bridge_list: ++ A list of all provided bridges to test. ++ :ivar socks_port: ++ Integer for Tor's SocksPort. ++ :ivar control_port: ++ Integer for Tor's ControlPort. ++ :ivar plug_transport: ++ String defining the Tor's ClientTransportPlugin, for testing ++ a bridge's pluggable transport functionality. ++ :ivar tor_binary: ++ Path to the Tor binary to use, e.g. '/usr/sbin/tor' ++ """ + implements(IPlugin, ITest) + + shortName = "bridget" - description = "bridget" ++ description = "Use a Tor process to test connecting to bridges and relays" + requirements = None - options = bridgetArgs ++ options = BridgetArgs + blocking = False + - def experiment(self, args): - log.msg("Doing test") - last_update = datetime.now() - tor_log = [] - - def check_timeout(): - log.msg("Checking for timeout") - time_since_update = datetime.now() - last_update - if time_since_update.seconds > self.local_options['timeout']: - log.msg("Timed out when connecting to %s" % args) - l.stop() - self.result['reason'] = 'timeout' - d.errback(args) - return ++ def initialize(self): ++ """ ++ Extra initialization steps. We only want one child Tor process ++ running, so we need to deal with the creation of TorConfig() only ++ once, before the experiment runs. ++ """ ++ self.socks_port = 9049 ++ self.control_port = 9052 ++ self.tor_binary = '/usr/sbin/tor' ++ self.data_directory = None + - def updates(prog, tag, summary): - tor_log.append((prog, tag, summary)) - last_update = datetime.now() - log.msg("%d%%: %s" % (prog, summary)) ++ if self.local_options: ++ try: ++ from ooni.lib.txtorcon import TorConfig ++ except: ++ e = "Could not import TorConfig class from txtorcon!" ++ raise ImportError, e + - def setup_failed(failure): - log.msg("Setup Failed.") - if not self.result['reason']: - self.result['reason'] = 'unknown' - self.result['input'] = args - self.result['result'] = 'failed' - self.result['tor_log'] = tor_log - return ++ options = self.local_options ++ self.config = TorConfig() + - def setup_complete(proto): - log.msg("Setup Complete.") - self.result['input'] = args - self.result['result'] = 'success' - return ++ ## Don't run the experiment if we don't have anything to test ++ if not options['bridges'] and not options['relays']: ++ self.suicide = True ++ ++ if options['bridges']: ++ self.config.UseBridges = 1 + - config = TorConfig() - import random - config.SocksPort = random.randint(1024, 2**16) - config.ControlPort = random.randint(1024, 2**16) ++ if options['relays']: ++ ## Stupid hack for testing only relays: ++ ## Tor doesn't use EntryNodes when UseBridges is enabled, but ++ ## config.state.entry_guards needs to include the first hop to ++ ## build a custom circuit. ++ self.config.EntryNodes = ','.join(relay_list) + - if 'bridge' in args: - config.UseBridges = 1 - config.Bridge = args['bridge'] ++ if options['socks']: ++ self.socks_port = options['socks'] + - config.save() ++ if options['control']: ++ self.control_port = options['control'] + - print config.config - self.result['tor_config'] = config.config - log.msg("Starting Tor connecting to %s" % args['bridge']) ++ if options['random']: ++ log.msg("Using randomized ControlPort and SocksPort ...") ++ self.socks_port = random.randint(1024, 2**16) ++ self.control_port = random.randint(1024, 2**16) + - l = task.LoopingCall(check_timeout) - l.start(1.0) ++ if options['tor-path']: ++ self.tor_binary = options['tor-path'] + - d = launch_tor(config, self.reactor, control_port=config.ControlPort, progress_updates=updates) - d.addCallback(setup_complete) - d.addErrback(setup_failed) - return d ++ if options['data-dir']: ++ self.config.DataDirectory = options['data-dir'] ++ ++ if options['transport']: ++ ## ClientTransportPlugin transport socks4|socks5 IP:PORT ++ ## ClientTransportPlugin transport exec path-to-binary [options] ++ if not options['bridges']: ++ e = "You must use the bridge option to test a transport." ++ raise usage.UsageError("%s" % e) ++ ++ log.msg("Using pluggable transport ...") ++ ## XXX fixme there's got to be a better way to check the exec ++ assert type(options['transport']) is str ++ self.config.ClientTransportPlugin = options['transport'] ++ ++ self.config.SocksPort = self.socks_port ++ self.config.ControlPort = self.control_port ++ self.config.save() + + def load_assets(self): - assets = {} ++ """ ++ Load bridges and/or relays from files given in user options. Bridges ++ should be given in the form IP:ORport. We don't want to load these as ++ assets, because it's inefficient to start a Tor process for each one. ++ """ ++ assets = {} ++ self.bridge_list = [] ++ self.relay_list = [] ++ ++ ## XXX fix me ++ ## we should probably find a more memory nice way to load addresses, ++ ## in case the files are really large + if self.local_options: + if self.local_options['bridges']: - assets.update({'bridge': Asset(self.local_options['bridges'])}) - elif self.local_options['relays']: - assets.update({'relay': Asset(self.local_options['relay'])}) ++ log.msg("Loading bridge information from %s ..." ++ % self.local_options['bridges']) ++ with open(self.local_options['bridges']) as bridge_file: ++ for line in bridge_file.readlines(): ++ if line.startswith('#'): ++ continue ++ else: ++ self.bridge_list.append(line.replace('\n','')) ++ assets.update({'bridges': self.bridge_list}) ++ ++ if self.local_options['relays']: ++ log.msg("Loading relay information from %s ..." ++ % self.local_options['relays']) ++ with open(options['relays']) as relay_file: ++ for line in relay_file.readlines(): ++ if line.startswith('#'): ++ continue ++ else: ++ self.relay_list.append(line.replace('\n','')) ++ assets.update({'relays': self.relay_list}) + return assets + - # We need to instantiate it otherwise getPlugins does not detect it - # XXX Find a way to load plugins without instantiating them. - bridget = bridgetTest(None, None, None) ++ def experiment(self, args): ++ """ ++ XXX fill me in ++ ++ :param args: ++ The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being ++ used. ++ :meth launch_tor: ++ Returns a Deferred which callbacks with a ++ :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol ++ <TorProcessProtocol>` connected to the fully-bootstrapped Tor; ++ this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol ++ <TorControlProtocol>` instance as .protocol. ++ """ ++ from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState ++ from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor ++ ++ def bootstrap(ctrl): ++ """ ++ Launch a Tor process with the TorConfig instance returned from ++ initialize(). ++ """ ++ conf = TorConfig(ctrl) ++ conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) ++ log.msg("Tor process connected, bootstrapping ...") ++ ++ def reconf_controller(conf, bridge): ++ ## if bridges and relays, use one bridge then build a circuit ++ ## from three relays ++ conf.Bridge = bridge ++ ## XXX do we need a SIGHUP to restart? ++ ++ ## XXX see txtorcon.TorControlProtocol.add_event_listener we ++ ## may not need full CustomCircuit class ++ ++ ## if bridges only, try one bridge at a time, but don't build ++ ## circuits, just return ++ ## if relays only, build circuits from relays ++ ++ def reconf_fail(args): ++ log.msg("Reconfiguring Tor config with args %s failed" % args) ++ reactor.stop() ++ ++ def setup_fail(args): ++ log.msg("Setup Failed.") ++ report.update({'failed': args}) ++ reactor.stop() ++ ++ def setup_done(proto): ++ log.msg("Setup Complete: %s" % proto) ++ state = TorState(proto.tor_protocol) ++ state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) ++ report.update({'success': args}) ++ ++ def updates(prog, tag, summary): ++ log.msg("%d%%: %s" % (prog, summary)) ++ ++ if len(args) == 0: ++ log.msg("Bridget needs lists of bridges and/or relays to test!") ++ log.msg("Exiting ...") ++ d = sys.exit() ++ return d ++ ++ else: ++ if len(self.bridge_list) >= 1: ++ for bridge in self.bridge_list: ++ try: ++ print "BRIDGE IS %s" % bridge ++ reconf_controller(self.config, bridge) ++ except: ++ reconf_fail(bridge) ++ ++ log.msg("Bridget: initiating test ... ") ++ log.msg("Using the following as our torrc:\n%s" ++ % self.config.create_torrc()) ++ report = {'tor_config': self.config.config} ++ log.msg("Starting Tor ...") ++ ++ ## :return: a Deferred which callbacks with a TorProcessProtocol ++ ## connected to the fully-bootstrapped Tor; this has a ++ ## txtorcon.TorControlProtocol instance as .protocol. ++ d = launch_tor(self.config, ++ reactor, ++ progress_updates=updates, ++ tor_binary=self.tor_binary) ++ d.addCallback(bootstrap, self.config) ++ d.addErrback(setup_fail) ++ ## now build circuits ++ ++ #print "Tor process ID: %s" % d.transport.pid ++ return d ++ ++## So that getPlugins() can register the Test: ++bridget = BridgetTest(None, None, None) ++ ++## ISIS' NOTES ++## ----------- ++## self.config.save() only needs to be called if Tor is already running. ++## ++## need to add transport type to torrc Bridge line: ++## Bridge <transport> IP:ORPort <fingerprint> ++## ++## TODO: ++## o add option for any kwarg=arg self.config setting ++## o cleanup documentation ++## o check if bridges are public relays ++## o take bridge_desc file as input, also be able to give same ++## format as output ++## o change the stupid name ++## ++## FIX: ++## data directory is not found, or permissions aren't right