[tor-commits] [ooni-probe/master] Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe

isis at torproject.org isis at torproject.org
Thu Oct 4 14:41:15 UTC 2012


commit 2e49b7330b155ca4746896939ebe48ce67f147b3
Merge: 0080bf6 62956eb
Author: Arturo Filastò <arturo at filasto.net>
Date:   Thu Sep 13 23:16:03 2012 +0000

    Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe
    
    Conflicts:
    	ooni/assets/bridgetests.txt
    	ooni/ooniprobe.py
    	ooni/plugins/domclass.py
    	ooni/plugins/new_bridget.py

 .gitmodules                  |    9 +
 README.md                    |   22 ++-
 TODO                         |   26 +--
 ooni/__init__.py             |    2 +-
 ooni/assets/bridgetests.txt  |    8 +
 ooni/example_plugins/skel.py |    4 +-
 ooni/lib/Makefile            |   30 ---
 ooni/lib/__init__.py         |   41 +++-
 ooni/lib/txscapy             |    1 +
 ooni/lib/txscapy.py          |  363 -------------------------------
 ooni/lib/txtorcon            |    1 +
 ooni/lib/txtraceroute        |    1 +
 ooni/ooniprobe.log           |    1 -
 ooni/ooniprobe.py            |   19 ++-
 ooni/oonitests/dnstamper.py  |  142 ++++++++-----
 ooni/plugins/blocking.py     |    4 +-
 ooni/plugins/dnstamper.py    |  343 ++++++++++++++++++++++++++++++
 ooni/plugins/domclass.py     |    3 +-
 ooni/plugins/httphost.py     |    4 +-
 ooni/plugins/new_bridget.py  |  484 ++++++++++++++++++++++++++++++++++--------
 ooni/plugoo/reports.py       |    2 +-
 ooni/plugoo/tests.py         |   25 ++-
 ooni/plugoo/work.py          |    2 +
 ooni/utils/log.py            |  116 ++++++++---
 24 files changed, 1045 insertions(+), 608 deletions(-)

diff --cc ooni/assets/bridgetests.txt
index 5519eea,7bba841..b69fbca
--- a/ooni/assets/bridgetests.txt
+++ b/ooni/assets/bridgetests.txt
@@@ -1,3 -1,5 +1,11 @@@
++<<<<<<< HEAD
 +88.130.86.191:443
 +195.74.237.236:9001
 +127.0.0.1:9050
++=======
+ #213.151.89.102:9001
+ #108.166.106.156:443
+ #217.150.224.213:443
+ 85.193.252.111:443
+ #68.98.41.14:9001
++>>>>>>> 62956ebab7779c1b61ce3d6e8ac750552fd1c988
diff --cc ooni/plugins/new_bridget.py
index 3e4db56,0000000..736b084
mode 100644,000000..100644
--- a/ooni/plugins/new_bridget.py
+++ b/ooni/plugins/new_bridget.py
@@@ -1,105 -1,0 +1,421 @@@
- """
- This is a self genrated test created by scaffolding.py.
- you will need to fill it up with all your necessities.
- Safe hacking :).
- """
- from exceptions import Exception
- from datetime import datetime
- from zope.interface import implements
- from twisted.python import usage
- from twisted.plugin import IPlugin
- from twisted.internet import reactor, task
- 
- from ooni.utils import log
- from ooni.plugoo.tests import ITest, OONITest
- from ooni.plugoo.assets import Asset
- 
- from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState
- from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor
- 
- class bridgetArgs(usage.Options):
-     optParameters = [['bridges', 'b', None, 'List of bridges to scan'],
-                      ['relays', 'f', None, 'List of relays to scan'],
-                      ['resume', 'r', 0, 'Resume at this index'],
-                      ['timeout', 't', 5, 'Timeout in seconds after which to consider a bridge not working']
-                     ]
- 
- class bridgetTest(OONITest):
++#!/usr/bin/env python
++# -*- encoding: utf-8 -*-
++#
++#  +-----------+
++#  |  BRIDGET  |
++#  |        +----------------------------------------------+
++#  +--------| Use a slave Tor process to test making a Tor |
++#           | connection to a list of bridges or relays.   |
++#           +----------------------------------------------+
++#
++# :authors: Arturo Filasto, Isis Lovecruft
++# :licence: see included LICENSE
++# :version: 0.1.0-alpha
++
++from __future__             import with_statement
++from zope.interface         import implements
++from twisted.python         import usage
++from twisted.plugin         import IPlugin
++from twisted.internet       import defer, error, reactor
++
++import random
++import sys
++
++try:
++    from ooni.lib.txtorcon  import CircuitListenerMixin, IStreamAttacher
++except:
++    print "BridgeT requires txtorcon: https://github.com/meejah/txtorcon.git"
++    print "Your copy of OONI should have it included, if you're seeing this"
++    print "message, please file a bug report."
++    log.msg ("Bridget: Unable to import from ooni.lib.txtorcon")
++
++from ooni.utils             import log
++from ooni.plugoo.tests      import ITest, OONITest
++from ooni.plugoo.assets     import Asset
++
++
++class BridgetArgs(usage.Options):
++
++    def portCheck(number):
++        number = int(number)
++        if number not in range(1024, 65535):
++            raise ValueError("Port out of range")
++    portCheck.coerceDoc = "Ports must be between 1024 and 65535"
++
++    optParameters = [
++        ['bridges', 'b', None,
++         'List of bridges to scan <IP>:<ORport>'],
++        ['relays', 'f', None,
++         'List of relays to scan <IP>'],
++        ['socks', 's', 9049, portCheck,
++         'Tor SocksPort to use'],
++        ['control', 'c', 9052, portCheck,
++         'Tor ControlPort to use'],
++        ['tor-path', 'p', None,
++         'Path to the Tor binary to use'],
++        ['data-dir', 'd', None,
++         'Tor DataDirectory to use'],
++        ['transport', 't', None,
++         'Tor ClientTransportPlugin'],
++        ['resume', 'r', 0,
++         'Resume at this index']]
++    optFlags = [['random', 'x', 'Randomize control and socks ports']]
++
++    def postOptions(self):
++        ## We can't test pluggable transports without bridges
++        if self['transport'] and not self['bridges']:
++            e = "Pluggable transport requires the bridges option"
++            raise usage.UsageError, e
++        ## We can't use random and static port simultaneously
++        if self['socks'] and self['control']:
++            if self['random']:
++                e = "Unable to use random and specific ports simultaneously"
++                raise usageError, e
++
++class CustomCircuit(CircuitListenerMixin):
++    implements(IStreamAttacher)
++
++    from txtorcon.interface import IRouterContainer, ICircuitContainer
++
++    def __init__(self, state):
++        self.state = state
++        self.waiting_circuits = []
++
++    def waiting_on(self, circuit):
++        for (circid, d) in self.waiting_circuits:
++            if circuit.id == circid:
++                return true
++        return False
++
++    def circuit_extend(self, circuit, router):
++        "ICircuitListener"
++        if circuit.purpose != 'GENERAL':
++            return
++        if self.waiting_on(circuit):
++            log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex))
++
++    def circuit_built(self, circuit):
++        "ICircuitListener"
++        if circuit.purpose != 'GENERAL':
++            return
++
++        log.msg("Circuit %s built ..." % circuit.id)
++        log.msg("Full path of %s: %s" % (circuit.id, circuit.path))
++
++        for (circid, d) in self.waiting_circuits:
++            if circid == circuit.id:
++                self.waiting_circuits.remove(circid, d)
++                d.callback(circuit)
++
++    def circuit_failed(self, circuit, reason):
++        if self.waiting_on(circuit):
++            log.msg("A circuit we requested %s failed for reason %s" %
++                    (circuit.id, reason))
++            circid, d = None, None
++            for x in self.waiting_circuits:
++                if x[0] == circuit.id:
++                    circid, d, stream_cc = x
++            if d is None:
++                raise Exception("Expected to find circuit.")
++
++            self.waiting_circuits.remove((circid, d))
++            log.msg("Trying to build a circuit for %s" % circid)
++            self.request_circuit_build(d)
++
++    def check_circuit_route(self, circuit, router):
++        if router in circuit.path:
++            #router.update() ## XXX can i use without args? no.
++            TorInfo.dump(self)
++
++    def request_circuit_build(self, deferred):
++        entries = self.state.entry_guards.value()
++        relays  = self.state.routers.values()
++
++        log.msg("We have these nodes listed as entry guards:")
++        log.msg("%s" % entries)
++        log.msg("We have these nodes listed as relays:")
++        log.msg("%s" % relays)
++
++        path = [random.choice(entries),
++                random.choice(relays),
++                random.choice(relays)]
++
++        log.msg("Requesting a circuit: %s"
++                % '-->'.join(map(lambda x: x.location.countrycode, path)))
++
++        class AppendWaiting:
++            def __init__(self, attacher, deferred):
++                self.attacher = attacher
++                self.d        = deferred
++
++            def __call__(self, circuit):
++                """
++                Return from build_circuit is a Circuit, however, we want to
++                wait until it is built before we can issue an attach on it and
++                callback to the Deferred we issue here.
++                """
++                log.msg("Circuit %s is in progress ..." % circuit.id)
++                self.attacher.waiting_circuits.append((circuit.id, self.d))
++
++        return self.state.build_circuit(path).addCallback(AppendWaiting(self, deferred_to_callback)).addErrback(log.err)
++
++class BridgetAsset(Asset):
++    """
++    Class for parsing bridge assets so that they can be commented out.
++    """
++    def __init__(self, file=None):
++        self = Asset.__init__(self, file)
++
++    def parse_line(self, line):
++        if line.startswith('#'):
++            return
++        else:
++            return line.replace('\n','')
++
++class BridgetTest(OONITest):
++    """
++    XXX fill me in
++
++    :ivar config:
++        An :class:`ooni.lib.txtorcon.TorConfig` instance.
++    :ivar relay_list:
++        A list of all provided relays to test. We have to do this because
++        txtorcon.TorState().entry_guards won't build a custom circuit if the
++        first hop isn't in the torrc's EntryNodes.
++    :ivar bridge_list:
++        A list of all provided bridges to test.
++    :ivar socks_port:
++        Integer for Tor's SocksPort.
++    :ivar control_port:
++        Integer for Tor's ControlPort.
++    :ivar plug_transport:
++        String defining the Tor's ClientTransportPlugin, for testing
++        a bridge's pluggable transport functionality.
++    :ivar tor_binary:
++        Path to the Tor binary to use, e.g. \'/usr/sbin/tor\'
++    """
 +    implements(IPlugin, ITest)
 +
 +    shortName = "bridget"
-     description = "bridget"
++    description = "Use a Tor process to test connecting to bridges and relays"
 +    requirements = None
-     options = bridgetArgs
++    options = BridgetArgs
 +    blocking = False
 +
-     def experiment(self, args):
-         log.msg("Doing test")
-         last_update = datetime.now()
-         tor_log = []
- 
-         def check_timeout():
-             log.msg("Checking for timeout")
-             time_since_update = datetime.now() - last_update
-             if time_since_update.seconds > self.local_options['timeout']:
-                 log.msg("Timed out when connecting to %s" % args)
-                 l.stop()
-                 self.result['reason'] = 'timeout'
-                 d.errback(args)
-             return
++    def initialize(self):
++        """
++        Extra initialization steps. We only want one child Tor process
++        running, so we need to deal with the creation of TorConfig() only
++        once, before the experiment runs.
++        """
++        self.socks_port     = 9049
++        self.control_port   = 9052
++        self.tor_binary     = '/usr/sbin/tor'
++        self.data_directory = None
 +
-         def updates(prog, tag, summary):
-             tor_log.append((prog, tag, summary))
-             last_update = datetime.now()
-             log.msg("%d%%: %s" % (prog, summary))
++        if self.local_options:
++            try:
++                from ooni.lib.txtorcon import TorConfig
++            except:
++                e = "Could not import TorConfig class from txtorcon!"
++                raise ImportError, e
 +
-         def setup_failed(failure):
-             log.msg("Setup Failed.")
-             if not self.result['reason']:
-                 self.result['reason'] = 'unknown'
-             self.result['input'] = args
-             self.result['result'] = 'failed'
-             self.result['tor_log'] = tor_log
-             return
++            options             = self.local_options
++            self.config         = TorConfig()
 +
-         def setup_complete(proto):
-             log.msg("Setup Complete.")
-             self.result['input'] = args
-             self.result['result'] = 'success'
-             return
++            ## Don't run the experiment if we don't have anything to test
++            if not options['bridges'] and not options['relays']:
++                self.suicide = True
++
++            if options['bridges']:
++                self.config.UseBridges = 1
 +
-         config = TorConfig()
-         import random
-         config.SocksPort = random.randint(1024, 2**16)
-         config.ControlPort = random.randint(1024, 2**16)
++            if options['relays']:
++                ## Stupid hack for testing only relays:
++                ## Tor doesn't use EntryNodes when UseBridges is enabled, but
++                ## config.state.entry_guards needs to include the first hop to
++                ## build a custom circuit.
++                self.config.EntryNodes = ','.join(relay_list)
 +
-         if 'bridge' in args:
-             config.UseBridges = 1
-             config.Bridge = args['bridge']
++            if options['socks']:
++                self.socks_port = options['socks']
 +
-         config.save()
++            if options['control']:
++                self.control_port = options['control']
 +
-         print config.config
-         self.result['tor_config'] = config.config
-         log.msg("Starting Tor connecting to %s" % args['bridge'])
++            if options['random']:
++                log.msg("Using randomized ControlPort and SocksPort ...")
++                self.socks_port   = random.randint(1024, 2**16)
++                self.control_port = random.randint(1024, 2**16)
 +
-         l = task.LoopingCall(check_timeout)
-         l.start(1.0)
++            if options['tor-path']:
++                self.tor_binary = options['tor-path']
 +
-         d = launch_tor(config, self.reactor, control_port=config.ControlPort, progress_updates=updates)
-         d.addCallback(setup_complete)
-         d.addErrback(setup_failed)
-         return d
++            if options['data-dir']:
++                self.config.DataDirectory = options['data-dir']
++
++            if options['transport']:
++                ## ClientTransportPlugin transport socks4|socks5 IP:PORT
++                ## ClientTransportPlugin transport exec path-to-binary [options]
++                if not options['bridges']:
++                    e = "You must use the bridge option to test a transport."
++                    raise usage.UsageError("%s" % e)
++
++                log.msg("Using pluggable transport ...")
++                ## XXX fixme there's got to be a better way to check the exec
++                assert type(options['transport']) is str
++                self.config.ClientTransportPlugin = options['transport']
++
++            self.config.SocksPort   = self.socks_port
++            self.config.ControlPort = self.control_port
++            self.config.save()
 +
 +    def load_assets(self):
-         assets = {}
++        """
++        Load bridges and/or relays from files given in user options. Bridges
++        should be given in the form IP:ORport. We don't want to load these as
++        assets, because it's inefficient to start a Tor process for each one.
++        """
++        assets           = {}
++        self.bridge_list = []
++        self.relay_list  = []
++
++        ## XXX fix me
++        ## we should probably find a more memory nice way to load addresses,
++        ## in case the files are really large
 +        if self.local_options:
 +            if self.local_options['bridges']:
-                 assets.update({'bridge': Asset(self.local_options['bridges'])})
-             elif self.local_options['relays']:
-                 assets.update({'relay': Asset(self.local_options['relay'])})
++                log.msg("Loading bridge information from %s ..."
++                        % self.local_options['bridges'])
++                with open(self.local_options['bridges']) as bridge_file:
++                    for line in bridge_file.readlines():
++                        if line.startswith('#'):
++                            continue
++                        else:
++                            self.bridge_list.append(line.replace('\n',''))
++                assets.update({'bridges': self.bridge_list})
++
++            if self.local_options['relays']:
++                log.msg("Loading relay information from %s  ..."
++                        % self.local_options['relays'])
++                with open(options['relays']) as relay_file:
++                    for line in relay_file.readlines():
++                        if line.startswith('#'):
++                            continue
++                        else:
++                            self.relay_list.append(line.replace('\n',''))
++                assets.update({'relays': self.relay_list})
 +        return assets
 +
- # We need to instantiate it otherwise getPlugins does not detect it
- # XXX Find a way to load plugins without instantiating them.
- bridget = bridgetTest(None, None, None)
++    def experiment(self, args):
++        """
++        XXX fill me in
++
++        :param args:
++            The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being
++            used.
++        :meth launch_tor:
++            Returns a Deferred which callbacks with a
++            :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol
++            <TorProcessProtocol>` connected to the fully-bootstrapped Tor;
++            this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol
++            <TorControlProtocol>` instance as .protocol.
++        """
++        from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState
++        from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor
++
++        def bootstrap(ctrl):
++            """
++            Launch a Tor process with the TorConfig instance returned from
++            initialize().
++            """
++            conf = TorConfig(ctrl)
++            conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail)
++            log.msg("Tor process connected, bootstrapping ...")
++
++        def reconf_controller(conf, bridge):
++            ## if bridges and relays, use one bridge then build a circuit
++            ## from three relays
++            conf.Bridge = bridge
++            ## XXX do we need a SIGHUP to restart?
++
++            ## XXX see txtorcon.TorControlProtocol.add_event_listener we
++            ## may not need full CustomCircuit class
++
++            ## if bridges only, try one bridge at a time, but don't build
++            ## circuits, just return
++            ## if relays only, build circuits from relays
++
++        def reconf_fail(args):
++            log.msg("Reconfiguring Tor config with args %s failed" % args)
++            reactor.stop()
++
++        def setup_fail(args):
++            log.msg("Setup Failed.")
++            report.update({'failed': args})
++            reactor.stop()
++
++        def setup_done(proto):
++            log.msg("Setup Complete: %s" % proto)
++            state = TorState(proto.tor_protocol)
++            state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail)
++            report.update({'success': args})
++
++        def updates(prog, tag, summary):
++            log.msg("%d%%: %s" % (prog, summary))
++
++        if len(args) == 0:
++            log.msg("Bridget needs lists of bridges and/or relays to test!")
++            log.msg("Exiting ...")
++            d = sys.exit()
++            return d
++
++        else:
++            if len(self.bridge_list) >= 1:
++                for bridge in self.bridge_list:
++                    try:
++                        print "BRIDGE IS %s" % bridge
++                        reconf_controller(self.config, bridge)
++                    except:
++                        reconf_fail(bridge)
++
++            log.msg("Bridget: initiating test ... ")
++            log.msg("Using the following as our torrc:\n%s"
++                    % self.config.create_torrc())
++            report = {'tor_config': self.config.config}
++            log.msg("Starting Tor ...")
++
++            ## :return: a Deferred which callbacks with a TorProcessProtocol
++            ##          connected to the fully-bootstrapped Tor; this has a
++            ##          txtorcon.TorControlProtocol instance as .protocol.
++            d = launch_tor(self.config,
++                           reactor,
++                           progress_updates=updates,
++                           tor_binary=self.tor_binary)
++            d.addCallback(bootstrap, self.config)
++            d.addErrback(setup_fail)
++            ## now build circuits
++
++            #print "Tor process ID: %s" % d.transport.pid
++            return d
++
++## So that getPlugins() can register the Test:
++bridget = BridgetTest(None, None, None)
++
++## ISIS' NOTES
++## -----------
++## self.config.save() only needs to be called if Tor is already running.
++##
++## need to add transport type to torrc Bridge line:
++##       Bridge <transport> IP:ORPort <fingerprint>
++##
++## TODO:
++##       o  add option for any kwarg=arg self.config setting
++##       o  cleanup documentation
++##       o  check if bridges are public relays
++##       o  take bridge_desc file as input, also be able to give same
++##          format as output
++##       o  change the stupid name
++##
++## FIX:
++##     data directory is not found, or permissions aren't right





More information about the tor-commits mailing list