[tor-commits] [ooni-probe/master] Merge branch 'nettest'
isis at torproject.org
isis at torproject.org
Thu Oct 4 14:41:15 UTC 2012
commit 16c3cfedbd80d23337579d6d5fc3f9bff367e714
Merge: 2e49b73 d621155
Author: Arturo Filastò <arturo at filasto.net>
Date: Fri Sep 28 12:06:54 2012 +0000
Merge branch 'nettest'
Conflicts:
ooni/plugins/new_bridget.py
.gitignore | 1 +
README.md | 10 +
bin/ooniprobe | 16 +-
docs/design.dia | Bin 1706 -> 1706 bytes
docs/source/conf.py | 7 +-
docs/source/index.rst | 100 ++----
docs/source/install.rst | 55 +++
docs/source/tutorial.rst | 1 -
docs/source/writing_tests.rst | 88 ++++-
docs/writing_tests.md | 17 -
ooni/input.py | 65 ++++
ooni/nettest.py | 90 +++++
ooni/oonicli.py | 105 ++++++
ooni/oonitests/bridget.py | 373 --------------------
ooni/plugins/httpt.py | 9 +-
ooni/plugins/new_bridget.py | 4 +-
ooni/plugoo/tests.py | 5 +-
ooni/protocols/http.py | 17 +-
ooni/reporter.py | 199 +++++++++++
ooni/runner.py | 191 ++++++++++
20 files changed, 852 insertions(+), 501 deletions(-)
diff --cc README.md
index 7d6e37a,123d837..23deb32
--- a/README.md
+++ b/README.md
@@@ -32,8 -26,18 +32,18 @@@ If you see some errors see INSTALL to i
To list the help for a specific test:
- python ooniprobe.py httpt --help
+ $ python ooniprobe.py httpt --help
+ ## Virtualenv way (Recommended)
+
+ virtualenv2 ENV/
+ source ENV/bin/activate
+ pip install twisted Scapy
+
+ To install the most up to date scapy version (requires mercurial):
+
+ pip install hg+http://hg.secdev.org/scapy
+
# More details
diff --cc ooni/ooniprobe.py
index 86f4e2d,f80669e..86f4e2d
mode 100755,100644..100644
--- a/ooni/ooniprobe.py
+++ b/ooni/ooniprobe.py
diff --cc ooni/plugins/new_bridget.py
index 736b084,b26455e..673cacc
--- a/ooni/plugins/new_bridget.py
+++ b/ooni/plugins/new_bridget.py
@@@ -1,421 -1,105 +1,421 @@@
-"""
-This is a self genrated test created by scaffolding.py.
-you will need to fill it up with all your necessities.
-Safe hacking :).
-"""
-from exceptions import Exception
-from datetime import datetime
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from twisted.internet import reactor, task
-
-from ooni.utils import log
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.plugoo.assets import Asset
-
-from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState
-from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor
-
-class bridgetArgs(usage.Options):
- optParameters = [['bridges', 'b', None, 'List of bridges to scan'],
- ['relays', 'f', None, 'List of relays to scan'],
- ['resume', 'r', 0, 'Resume at this index'],
- ['timeout', 't', 5, 'Timeout in seconds after which to consider a bridge not working']
- ]
-
-class bridgetTest(OONITest):
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+#
+# +-----------+
+# | BRIDGET |
+# | +----------------------------------------------+
+# +--------| Use a slave Tor process to test making a Tor |
+# | connection to a list of bridges or relays. |
+# +----------------------------------------------+
+#
+# :authors: Arturo Filasto, Isis Lovecruft
+# :licence: see included LICENSE
+# :version: 0.1.0-alpha
+
+from __future__ import with_statement
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+from twisted.internet import defer, error, reactor
+
+import random
+import sys
+
+try:
+ from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher
+except:
+ print "BridgeT requires txtorcon: https://github.com/meejah/txtorcon.git"
+ print "Your copy of OONI should have it included, if you're seeing this"
+ print "message, please file a bug report."
+ log.msg ("Bridget: Unable to import from ooni.lib.txtorcon")
+
+from ooni.utils import log
+from ooni.plugoo.tests import ITest, OONITest
+from ooni.plugoo.assets import Asset
+
+
+class BridgetArgs(usage.Options):
+
+ def portCheck(number):
+ number = int(number)
+ if number not in range(1024, 65535):
+ raise ValueError("Port out of range")
+ portCheck.coerceDoc = "Ports must be between 1024 and 65535"
+
+ optParameters = [
+ ['bridges', 'b', None,
+ 'List of bridges to scan <IP>:<ORport>'],
+ ['relays', 'f', None,
+ 'List of relays to scan <IP>'],
+ ['socks', 's', 9049, portCheck,
+ 'Tor SocksPort to use'],
+ ['control', 'c', 9052, portCheck,
+ 'Tor ControlPort to use'],
+ ['tor-path', 'p', None,
+ 'Path to the Tor binary to use'],
+ ['data-dir', 'd', None,
+ 'Tor DataDirectory to use'],
+ ['transport', 't', None,
+ 'Tor ClientTransportPlugin'],
+ ['resume', 'r', 0,
+ 'Resume at this index']]
+ optFlags = [['random', 'x', 'Randomize control and socks ports']]
+
+ def postOptions(self):
+ ## We can't test pluggable transports without bridges
+ if self['transport'] and not self['bridges']:
+ e = "Pluggable transport requires the bridges option"
+ raise usage.UsageError, e
+ ## We can't use random and static port simultaneously
+ if self['socks'] and self['control']:
+ if self['random']:
+ e = "Unable to use random and specific ports simultaneously"
+ raise usageError, e
+
+class CustomCircuit(CircuitListenerMixin):
+ implements(IStreamAttacher)
+
+ from txtorcon.interface import IRouterContainer, ICircuitContainer
+
+ def __init__(self, state):
+ self.state = state
+ self.waiting_circuits = []
+
+ def waiting_on(self, circuit):
+ for (circid, d) in self.waiting_circuits:
+ if circuit.id == circid:
+ return true
+ return False
+
+ def circuit_extend(self, circuit, router):
+ "ICircuitListener"
+ if circuit.purpose != 'GENERAL':
+ return
+ if self.waiting_on(circuit):
+ log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex))
+
+ def circuit_built(self, circuit):
+ "ICircuitListener"
+ if circuit.purpose != 'GENERAL':
+ return
+
+ log.msg("Circuit %s built ..." % circuit.id)
+ log.msg("Full path of %s: %s" % (circuit.id, circuit.path))
+
+ for (circid, d) in self.waiting_circuits:
+ if circid == circuit.id:
+ self.waiting_circuits.remove(circid, d)
+ d.callback(circuit)
+
+ def circuit_failed(self, circuit, reason):
+ if self.waiting_on(circuit):
+ log.msg("A circuit we requested %s failed for reason %s" %
+ (circuit.id, reason))
+ circid, d = None, None
+ for x in self.waiting_circuits:
+ if x[0] == circuit.id:
+ circid, d, stream_cc = x
+ if d is None:
+ raise Exception("Expected to find circuit.")
+
+ self.waiting_circuits.remove((circid, d))
+ log.msg("Trying to build a circuit for %s" % circid)
+ self.request_circuit_build(d)
+
+ def check_circuit_route(self, circuit, router):
+ if router in circuit.path:
+ #router.update() ## XXX can i use without args? no.
+ TorInfo.dump(self)
+
+ def request_circuit_build(self, deferred):
+ entries = self.state.entry_guards.value()
+ relays = self.state.routers.values()
+
+ log.msg("We have these nodes listed as entry guards:")
+ log.msg("%s" % entries)
+ log.msg("We have these nodes listed as relays:")
+ log.msg("%s" % relays)
+
+ path = [random.choice(entries),
+ random.choice(relays),
+ random.choice(relays)]
+
+ log.msg("Requesting a circuit: %s"
+ % '-->'.join(map(lambda x: x.location.countrycode, path)))
+
+ class AppendWaiting:
+ def __init__(self, attacher, deferred):
+ self.attacher = attacher
+ self.d = deferred
+
+ def __call__(self, circuit):
+ """
+ Return from build_circuit is a Circuit, however, we want to
+ wait until it is built before we can issue an attach on it and
+ callback to the Deferred we issue here.
+ """
+ log.msg("Circuit %s is in progress ..." % circuit.id)
+ self.attacher.waiting_circuits.append((circuit.id, self.d))
+
+ return self.state.build_circuit(path).addCallback(AppendWaiting(self, deferred_to_callback)).addErrback(log.err)
+
+class BridgetAsset(Asset):
+ """
+ Class for parsing bridge assets so that they can be commented out.
+ """
+ def __init__(self, file=None):
+ self = Asset.__init__(self, file)
+
+ def parse_line(self, line):
+ if line.startswith('#'):
+ return
+ else:
+ return line.replace('\n','')
+
+class BridgetTest(OONITest):
+ """
+ XXX fill me in
+
+ :ivar config:
+ An :class:`ooni.lib.txtorcon.TorConfig` instance.
+ :ivar relay_list:
+ A list of all provided relays to test. We have to do this because
+ txtorcon.TorState().entry_guards won't build a custom circuit if the
+ first hop isn't in the torrc's EntryNodes.
+ :ivar bridge_list:
+ A list of all provided bridges to test.
+ :ivar socks_port:
+ Integer for Tor's SocksPort.
+ :ivar control_port:
+ Integer for Tor's ControlPort.
+ :ivar plug_transport:
+ String defining the Tor's ClientTransportPlugin, for testing
+ a bridge's pluggable transport functionality.
+ :ivar tor_binary:
+ Path to the Tor binary to use, e.g. \'/usr/sbin/tor\'
+ """
implements(IPlugin, ITest)
- shortName = "bridget"
- description = "Use a Tor process to test connecting to bridges and relays"
+ shortName = "newbridget"
+ description = "bridget"
requirements = None
- options = bridgetArgs
+ options = BridgetArgs
blocking = False
- def experiment(self, args):
- log.msg("Doing test")
- last_update = datetime.now()
- tor_log = []
-
- def check_timeout():
- log.msg("Checking for timeout")
- time_since_update = datetime.now() - last_update
- if time_since_update.seconds > self.local_options['timeout']:
- log.msg("Timed out when connecting to %s" % args)
- l.stop()
- self.result['reason'] = 'timeout'
- d.errback(args)
- return
+ def initialize(self):
+ """
+ Extra initialization steps. We only want one child Tor process
+ running, so we need to deal with the creation of TorConfig() only
+ once, before the experiment runs.
+ """
+ self.socks_port = 9049
+ self.control_port = 9052
+ self.tor_binary = '/usr/sbin/tor'
+ self.data_directory = None
- def updates(prog, tag, summary):
- tor_log.append((prog, tag, summary))
- last_update = datetime.now()
- log.msg("%d%%: %s" % (prog, summary))
+ if self.local_options:
+ try:
+ from ooni.lib.txtorcon import TorConfig
+ except:
+ e = "Could not import TorConfig class from txtorcon!"
+ raise ImportError, e
- def setup_failed(failure):
- log.msg("Setup Failed.")
- if not self.result['reason']:
- self.result['reason'] = 'unknown'
- self.result['input'] = args
- self.result['result'] = 'failed'
- self.result['tor_log'] = tor_log
- return
+ options = self.local_options
+ self.config = TorConfig()
- def setup_complete(proto):
- log.msg("Setup Complete.")
- self.result['input'] = args
- self.result['result'] = 'success'
- return
+ ## Don't run the experiment if we don't have anything to test
+ if not options['bridges'] and not options['relays']:
+ self.suicide = True
+
+ if options['bridges']:
+ self.config.UseBridges = 1
- config = TorConfig()
- import random
- config.SocksPort = random.randint(1024, 2**16)
- config.ControlPort = random.randint(1024, 2**16)
+ if options['relays']:
+ ## Stupid hack for testing only relays:
+ ## Tor doesn't use EntryNodes when UseBridges is enabled, but
+ ## config.state.entry_guards needs to include the first hop to
+ ## build a custom circuit.
+ self.config.EntryNodes = ','.join(relay_list)
- if 'bridge' in args:
- config.UseBridges = 1
- config.Bridge = args['bridge']
+ if options['socks']:
+ self.socks_port = options['socks']
- config.save()
+ if options['control']:
+ self.control_port = options['control']
- print config.config
- self.result['tor_config'] = config.config
- log.msg("Starting Tor connecting to %s" % args['bridge'])
+ if options['random']:
+ log.msg("Using randomized ControlPort and SocksPort ...")
+ self.socks_port = random.randint(1024, 2**16)
+ self.control_port = random.randint(1024, 2**16)
- l = task.LoopingCall(check_timeout)
- l.start(1.0)
+ if options['tor-path']:
+ self.tor_binary = options['tor-path']
- d = launch_tor(config, self.reactor, control_port=config.ControlPort, progress_updates=updates)
- d.addCallback(setup_complete)
- d.addErrback(setup_failed)
- return d
+ if options['data-dir']:
+ self.config.DataDirectory = options['data-dir']
+
+ if options['transport']:
+ ## ClientTransportPlugin transport socks4|socks5 IP:PORT
+ ## ClientTransportPlugin transport exec path-to-binary [options]
+ if not options['bridges']:
+ e = "You must use the bridge option to test a transport."
+ raise usage.UsageError("%s" % e)
+
+ log.msg("Using pluggable transport ...")
+ ## XXX fixme there's got to be a better way to check the exec
+ assert type(options['transport']) is str
+ self.config.ClientTransportPlugin = options['transport']
+
+ self.config.SocksPort = self.socks_port
+ self.config.ControlPort = self.control_port
+ self.config.save()
def load_assets(self):
- assets = {}
+ """
+ Load bridges and/or relays from files given in user options. Bridges
+ should be given in the form IP:ORport. We don't want to load these as
+ assets, because it's inefficient to start a Tor process for each one.
+ """
+ assets = {}
+ self.bridge_list = []
+ self.relay_list = []
+
+ ## XXX fix me
+ ## we should probably find a more memory nice way to load addresses,
+ ## in case the files are really large
if self.local_options:
if self.local_options['bridges']:
- assets.update({'bridge': Asset(self.local_options['bridges'])})
- elif self.local_options['relays']:
- assets.update({'relay': Asset(self.local_options['relay'])})
+ log.msg("Loading bridge information from %s ..."
+ % self.local_options['bridges'])
+ with open(self.local_options['bridges']) as bridge_file:
+ for line in bridge_file.readlines():
+ if line.startswith('#'):
+ continue
+ else:
+ self.bridge_list.append(line.replace('\n',''))
+ assets.update({'bridges': self.bridge_list})
+
+ if self.local_options['relays']:
+ log.msg("Loading relay information from %s ..."
+ % self.local_options['relays'])
+ with open(options['relays']) as relay_file:
+ for line in relay_file.readlines():
+ if line.startswith('#'):
+ continue
+ else:
+ self.relay_list.append(line.replace('\n',''))
+ assets.update({'relays': self.relay_list})
return assets
-# We need to instantiate it otherwise getPlugins does not detect it
-# XXX Find a way to load plugins without instantiating them.
-bridget = bridgetTest(None, None, None)
+ def experiment(self, args):
+ """
+ XXX fill me in
+
+ :param args:
+ The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being
+ used.
+ :meth launch_tor:
+ Returns a Deferred which callbacks with a
+ :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol
+ <TorProcessProtocol>` connected to the fully-bootstrapped Tor;
+ this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol
+ <TorControlProtocol>` instance as .protocol.
+ """
+ from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState
+ from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor
+
+ def bootstrap(ctrl):
+ """
+ Launch a Tor process with the TorConfig instance returned from
+ initialize().
+ """
+ conf = TorConfig(ctrl)
+ conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail)
+ log.msg("Tor process connected, bootstrapping ...")
+
+ def reconf_controller(conf, bridge):
+ ## if bridges and relays, use one bridge then build a circuit
+ ## from three relays
+ conf.Bridge = bridge
+ ## XXX do we need a SIGHUP to restart?
+
+ ## XXX see txtorcon.TorControlProtocol.add_event_listener we
+ ## may not need full CustomCircuit class
+
+ ## if bridges only, try one bridge at a time, but don't build
+ ## circuits, just return
+ ## if relays only, build circuits from relays
+
+ def reconf_fail(args):
+ log.msg("Reconfiguring Tor config with args %s failed" % args)
+ reactor.stop()
+
+ def setup_fail(args):
+ log.msg("Setup Failed.")
+ report.update({'failed': args})
+ reactor.stop()
+
+ def setup_done(proto):
+ log.msg("Setup Complete: %s" % proto)
+ state = TorState(proto.tor_protocol)
+ state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail)
+ report.update({'success': args})
+
+ def updates(prog, tag, summary):
+ log.msg("%d%%: %s" % (prog, summary))
+
+ if len(args) == 0:
+ log.msg("Bridget needs lists of bridges and/or relays to test!")
+ log.msg("Exiting ...")
+ d = sys.exit()
+ return d
+
+ else:
+ if len(self.bridge_list) >= 1:
+ for bridge in self.bridge_list:
+ try:
+ print "BRIDGE IS %s" % bridge
+ reconf_controller(self.config, bridge)
+ except:
+ reconf_fail(bridge)
+
+ log.msg("Bridget: initiating test ... ")
+ log.msg("Using the following as our torrc:\n%s"
+ % self.config.create_torrc())
+ report = {'tor_config': self.config.config}
+ log.msg("Starting Tor ...")
+
+ ## :return: a Deferred which callbacks with a TorProcessProtocol
+ ## connected to the fully-bootstrapped Tor; this has a
+ ## txtorcon.TorControlProtocol instance as .protocol.
+ d = launch_tor(self.config,
+ reactor,
+ progress_updates=updates,
+ tor_binary=self.tor_binary)
+ d.addCallback(bootstrap, self.config)
+ d.addErrback(setup_fail)
+ ## now build circuits
+
+ #print "Tor process ID: %s" % d.transport.pid
+ return d
+
+## So that getPlugins() can register the Test:
+bridget = BridgetTest(None, None, None)
+
+## ISIS' NOTES
+## -----------
+## self.config.save() only needs to be called if Tor is already running.
+##
+## need to add transport type to torrc Bridge line:
+## Bridge <transport> IP:ORPort <fingerprint>
+##
+## TODO:
+## o add option for any kwarg=arg self.config setting
+## o cleanup documentation
+## o check if bridges are public relays
+## o take bridge_desc file as input, also be able to give same
+## format as output
+## o change the stupid name
+##
+## FIX:
+## data directory is not found, or permissions aren't right
More information about the tor-commits
mailing list