[tor-commits] [ooni-probe/master] * Refactor all the bridget tests into a better dir structure
isis at torproject.org
isis at torproject.org
Sat Nov 3 01:24:44 UTC 2012
commit 04a61965d73cb96745e898c5bfd4cf097a090803
Author: Isis Lovecruft <isis at torproject.org>
Date: Tue Oct 23 10:44:46 2012 +0000
* Refactor all the bridget tests into a better dir structure
---
bin/canary | 27 ++
ooni/bridget/__init__.py | 14 +
ooni/bridget/custodiet.py | 421 ++++++++++++++++++++++++
ooni/bridget/tests/__init__.py | 14 +
ooni/bridget/tests/bridget.py | 499 ++++++++++++++++++++++++++++
ooni/bridget/utils/__init__.py | 1 +
ooni/bridget/utils/inputs.py | 174 ++++++++++
ooni/bridget/utils/interface.py | 54 +++
ooni/bridget/utils/log.py | 98 ++++++
ooni/bridget/utils/nodes.py | 176 ++++++++++
ooni/bridget/utils/onion.py | 686 +++++++++++++++++++++++++++++++++++++++
ooni/bridget/utils/reports.py | 144 ++++++++
ooni/bridget/utils/tests.py | 141 ++++++++
ooni/bridget/utils/work.py | 147 +++++++++
ooni/plugins/bridget.py | 500 ----------------------------
15 files changed, 2596 insertions(+), 500 deletions(-)
diff --git a/bin/canary b/bin/canary
new file mode 100755
index 0000000..1473ae4
--- /dev/null
+++ b/bin/canary
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+###############################################################################
+#
+# canary
+# -----------------
+# Test Tor bridge reachability.
+#
+# :authors: Isis Lovecruft
+# :copyright: 2012 Isis Lovecruft, The Tor Project
+# :licence: see included LICENSE file
+# :version: 0.2.0-beta
+###############################################################################
+
+import os, sys
+import copy_reg
+
+# Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain.
+sys.path[:] = map(os.path.abspath, sys.path)
+sys.path.insert(0, os.path.abspath(os.getcwd()))
+
+# This is a hack to overcome a bug in python
+from ooni.utils.hacks import patched_reduce_ex
+copy_reg._reduce_ex = patched_reduce_ex
+
+from ooni.bridget import spelunker
+spelunker.descend()
diff --git a/ooni/bridget/__init__.py b/ooni/bridget/__init__.py
new file mode 100644
index 0000000..4648d77
--- /dev/null
+++ b/ooni/bridget/__init__.py
@@ -0,0 +1,14 @@
+#-*- coding: utf-8 -*-
+
+#import os, sys
+#import copy_reg
+
+## Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain.
+#sys.path[:] = map(os.path.abspath, sys.path)
+#sys.path.insert(0, os.path.abspath(os.getcwd()))
+
+## This is a hack to overcome a bug in python
+#from ooni.utils.hacks import patched_reduce_ex
+#copy_reg._reduce_ex = patched_reduce_ex
+
+__all__ = ['custodiet']
diff --git a/ooni/bridget/custodiet.py b/ooni/bridget/custodiet.py
new file mode 100755
index 0000000..8cbcfce
--- /dev/null
+++ b/ooni/bridget/custodiet.py
@@ -0,0 +1,421 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8
+#
+# custodiet
+# *********
+#
+# "...quis custodiet ipsos custodes?"
+# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.)
+#
+# "'Hand me the Custodian,' Goodchild demands, inserting the waiflike
+# robot into Bambara's opened navel. 'Providing conscience for those who
+# have none.' Goodchild and the other Breen government agents disappear
+# into the surrounding desert in a vehicle, kicking up cloud of white dust.
+# Bambara awakens, and, patting the dust from his clothing, turns to
+# greet a one-armed child. 'Hi, my name's Bambara; I'm a
+# thirty-six-year-old Virgo and a former killer, who's hobbies include
+# performing recreational autopsies, defecating, and drinking rum. I've
+# recently been given a conscience, and would very much like to help you.'
+# Cut to Bambara and the child, now with one of Bambara's arms, leaving
+# a surgical clinic."
+# - AeonFlux, "The Purge" (sometime in the late 90s)
+#
+# :copyright: (c) 2012 Isis Lovecruft
+# :license: see LICENSE for more details.
+# :version: 0.1.0-beta
+#
+
+# ooniprobe.py imports
+import sys
+from signal import SIGTERM, signal
+from pprint import pprint
+
+from twisted.python import usage
+from twisted.internet import reactor
+from twisted.plugin import getPlugins
+
+from zope.interface.verify import verifyObject
+from zope.interface.exceptions import BrokenImplementation
+from zope.interface.exceptions import BrokenMethodImplementation
+
+from ooni.bridget.tests import bridget
+from ooni.bridget.utils import log, tests, work, reports
+from ooni.bridget.utils.interface import ITest
+from ooni.utils.logo import getlogo
+
+# runner.py imports
+import os
+import types
+import time
+import inspect
+import yaml
+
+from twisted.internet import defer, reactor
+from twisted.python import reflect, failure, usage
+from twisted.python import log as tlog
+
+from twisted.trial import unittest
+from twisted.trial.runner import TrialRunner, TestLoader
+from twisted.trial.runner import isPackage, isTestCase, ErrorHolder
+from twisted.trial.runner import filenameToModule, _importFromFile
+
+from ooni import nettest
+from ooni.inputunit import InputUnitFactory
+from ooni.nettest import InputTestSuite
+from ooni.plugoo import tests as oonitests
+from ooni.reporter import ReporterFactory
+from ooni.utils import log, geodata, date
+from ooni.utils.legacy import LegacyOONITest
+from ooni.utils.legacy import start_legacy_test, adapt_legacy_test
+
+
+__version__ = "0.1.0-beta"
+
+
+#def retrieve_plugoo():
+# """
+# Get all the plugins that implement the ITest interface and get the data
+# associated to them into a dict.
+# """
+# interface = ITest
+# d = {}
+# error = False
+# for p in getPlugins(interface, plugins):
+# try:
+# verifyObject(interface, p)
+# d[p.shortName] = p
+# except BrokenImplementation, bi:
+# print "Plugin Broken"
+# print bi
+# error = True
+# if error != False:
+# print "Plugin Loaded!"
+# return d
+#
+#plugoo = retrieve_plugoo()
+
+"""
+
+ai to watch over which tests to run - custodiet
+
+ * runTest() or getPrefixMethodNames() to run the tests in order for each
+ test (esp. the tcp and icmp parts) to be oonicompat we should use the
+ test_icmp_ping API framework for those.
+
+ * should handle calling
+
+tests to run:
+ echo
+ syn
+ fin
+ conn
+ tls
+ tor
+need fakebridge - canary
+
+"""
+
+def runTest(test, options, global_options, reactor=reactor):
+ """
+ Run an OONI probe test by name.
+
+ @param test: a string specifying the test name as specified inside of
+ shortName.
+
+ @param options: the local options to be passed to the test.
+
+ @param global_options: the global options for OONI
+ """
+ parallelism = int(global_options['parallelism'])
+ worker = work.Worker(parallelism, reactor=reactor)
+ test_class = plugoo[test].__class__
+ report = reports.Report(test, global_options['output'])
+
+ log_to_stdout = True
+ if global_options['quiet']:
+ log_to_stdout = False
+
+ log.start(log_to_stdout,
+ global_options['log'],
+ global_options['verbosity'])
+
+ resume = 0
+ if not options:
+ options = {}
+ if 'resume' in options:
+ resume = options['resume']
+
+ test = test_class(options, global_options, report, reactor=reactor)
+ if test.tool:
+ test.runTool()
+ return True
+
+ if test.ended:
+ print "Ending test"
+ return None
+
+ wgen = work.WorkGenerator(test,
+ dict(options),
+ start=resume)
+ for x in wgen:
+ worker.push(x)
+
+class MainOptions(usage.Options):
+ tests = [bridget, ]
+ subCommands = []
+ for test in tests:
+ print test
+ testopt = getattr(test, 'options')
+ subCommands.append([test, None, testopt, "Run the %s test" % test])
+
+ optFlags = [
+ ['quiet', 'q', "Don't log to stdout"]
+ ]
+
+ optParameters = [
+ ['parallelism', 'n', 10, "Specify the number of parallel tests to run"],
+ #['target-node', 't', 'localhost:31415', 'Select target node'],
+ ['output', 'o', 'bridge.log', "Specify output report file"],
+ ['reportfile', 'o', 'bridge.log', "Specify output log file"],
+ ['verbosity', 'v', 1, "Specify the logging level"],
+ ]
+
+ def opt_version(self):
+ """
+ Display OONI version and exit.
+ """
+ print "OONI version:", __version__
+ sys.exit(0)
+
+ def __str__(self):
+ """
+ Hack to get the sweet ascii art into the help output and replace the
+ strings "Commands" with "Tests".
+ """
+ return getlogo() + '\n' + self.getSynopsis() + '\n' + \
+ self.getUsage(width=None).replace("Commands:", "Tests:")
+
+
+
+def isTestCase(thing):
+ try:
+ return issubclass(thing, unittest.TestCase)
+ except TypeError:
+ return False
+
+def isLegacyTest(obj):
+ """
+ Returns True if the test in question is written using the OONITest legacy
+ class.
+ We do this for backward compatibility of the OONIProbe API.
+ """
+ try:
+ if issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest:
+ return True
+ else:
+ return False
+ except TypeError:
+ return False
+
+def processTest(obj, config):
+ """
+ Process the parameters and :class:`twisted.python.usage.Options` of a
+ :class:`ooni.nettest.Nettest`.
+
+ :param obj:
+ An uninstantiated old test, which should be a subclass of
+ :class:`ooni.plugoo.tests.OONITest`.
+ :param config:
+ A configured and instantiated :class:`twisted.python.usage.Options`
+ class.
+ """
+
+ inputFile = obj.inputFile
+
+ if obj.optParameters or inputFile:
+ if not obj.optParameters:
+ obj.optParameters = []
+
+ if inputFile:
+ obj.optParameters.append(inputFile)
+
+ class Options(usage.Options):
+ optParameters = obj.optParameters
+
+ options = Options()
+ options.parseOptions(config['subArgs'])
+ obj.localOptions = options
+
+ if inputFile:
+ obj.inputFile = options[inputFile[0]]
+ try:
+ tmp_obj = obj()
+ tmp_obj.getOptions()
+ except usage.UsageError:
+ options.opt_help()
+
+ return obj
+
+def findTestClassesFromConfig(config):
+ """
+ Takes as input the command line config parameters and returns the test
+ case classes.
+ If it detects that a certain test class is using the old OONIProbe format,
+ then it will adapt it to the new testing system.
+
+ :param config:
+ A configured and instantiated :class:`twisted.python.usage.Options`
+ class.
+ :return:
+ A list of class objects found in a file or module given on the
+ commandline.
+ """
+
+ filename = config['test']
+ classes = []
+
+ module = filenameToModule(filename)
+ for name, val in inspect.getmembers(module):
+ if isTestCase(val):
+ classes.append(processTest(val, config))
+ elif isLegacyTest(val):
+ classes.append(adapt_legacy_test(val, config))
+ return classes
+
+def makeTestCases(klass, tests, methodPrefix):
+ """
+ Takes a class some tests and returns the test cases. methodPrefix is how
+ the test case functions should be prefixed with.
+ """
+
+ cases = []
+ for test in tests:
+ cases.append(klass(methodPrefix+test))
+ return cases
+
+def loadTestsAndOptions(classes, config):
+ """
+ Takes a list of classes and returns their testcases and options.
+ Legacy tests will be adapted.
+ """
+
+ methodPrefix = 'test'
+ suiteFactory = InputTestSuite
+ options = []
+ testCases = []
+ names = []
+
+ _old_klass_type = LegacyOONITest
+
+ for klass in classes:
+ if isinstance(klass, _old_klass_type):
+ try:
+ cases = start_legacy_test(klass)
+ #cases.callback()
+ if cases:
+ print cases
+ return [], []
+ testCases.append(cases)
+ except Exception, e:
+ log.err(e)
+ else:
+ try:
+ opts = klass.local_options
+ options.append(opts)
+ except AttributeError, ae:
+ options.append([])
+ log.err(ae)
+ elif not isinstance(klass, _old_klass_type):
+ tests = reflect.prefixedMethodNames(klass, methodPrefix)
+ if tests:
+ cases = makeTestCases(klass, tests, methodPrefix)
+ testCases.append(cases)
+ try:
+ k = klass()
+ opts = k.getOptions()
+ options.append(opts)
+ except AttributeError, ae:
+ options.append([])
+ log.err(ae)
+ else:
+ try:
+ raise RuntimeError, "Class is some strange type!"
+ except RuntimeError, re:
+ log.err(re)
+
+ return testCases, options
+
+class ORunner(object):
+ """
+ This is a specialized runner used by the ooniprobe command line tool.
+ I am responsible for reading the inputs from the test files and splitting
+ them in input units. I also create all the report instances required to run
+ the tests.
+ """
+ def __init__(self, cases, options=None, config=None, *arg, **kw):
+ self.baseSuite = InputTestSuite
+ self.cases = cases
+ self.options = options
+
+ try:
+ assert len(options) != 0, "Length of options is zero!"
+ except AssertionError, ae:
+ self.inputs = []
+ log.err(ae)
+ else:
+ try:
+ first = options.pop(0)
+ except:
+ first = {}
+ if 'inputs' in first:
+ self.inputs = options['inputs']
+ else:
+ log.msg("Could not find inputs!")
+ log.msg("options[0] = %s" % first)
+ self.inputs = [None]
+
+ try:
+ reportFile = open(config['reportfile'], 'a+')
+ except:
+ filename = 'report_'+date.timestamp()+'.yaml'
+ reportFile = open(filename, 'a+')
+ self.reporterFactory = ReporterFactory(reportFile,
+ testSuite=self.baseSuite(self.cases))
+
+ def runWithInputUnit(self, inputUnit):
+ idx = 0
+ result = self.reporterFactory.create()
+
+ for inputs in inputUnit:
+ result.reporterFactory = self.reporterFactory
+
+ suite = self.baseSuite(self.cases)
+ suite.input = inputs
+ suite(result, idx)
+
+ # XXX refactor all of this index bullshit to avoid having to pass
+ # this index around. Probably what I want to do is go and make
+ # changes to report to support the concept of having multiple runs
+ # of the same test.
+ # We currently need to do this addition in order to get the number
+ # of times the test cases that have run inside of the test suite.
+ idx += (suite._idx - idx)
+
+ result.done()
+
+ def run(self):
+ self.reporterFactory.options = self.options
+ for inputUnit in InputUnitFactory(self.inputs):
+ self.runWithInputUnit(inputUnit)
+
+if __name__ == "__main__":
+ config = Options()
+ config.parseOptions()
+
+ if not config.subCommand:
+ config.opt_help()
+ signal(SIGTERM)
+ #sys.exit(1)
+
+ runTest(config.subCommand, config.subOptions, config)
+ reactor.run()
diff --git a/ooni/bridget/tests/__init__.py b/ooni/bridget/tests/__init__.py
new file mode 100644
index 0000000..9ecc88d
--- /dev/null
+++ b/ooni/bridget/tests/__init__.py
@@ -0,0 +1,14 @@
+# -*- coding: UTF-8
+#
+# bridget/tests/__init__.py
+# *************************
+#
+# "...quis custodiet ipsos custodes?"
+# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.)
+#
+# :copyright: (c) 2012 Isis Lovecruft
+# :license: see LICENSE for more details.
+# :version: 0.1.0-beta
+#
+
+all = ['bridget']
diff --git a/ooni/bridget/tests/bridget.py b/ooni/bridget/tests/bridget.py
new file mode 100644
index 0000000..a334747
--- /dev/null
+++ b/ooni/bridget/tests/bridget.py
@@ -0,0 +1,499 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+#
+# +-----------+
+# | BRIDGET |
+# | +--------------------------------------------+
+# +--------| Use a Tor process to test making a Tor |
+# | connection to a list of bridges or relays. |
+# +--------------------------------------------+
+#
+# :authors: Isis Lovecruft, Arturo Filasto
+# :licence: see included LICENSE
+# :version: 0.1.0-alpha
+
+from __future__ import with_statement
+from functools import partial
+from random import randint
+
+import os
+import sys
+
+from twisted.python import usage
+from twisted.plugin import IPlugin
+from twisted.internet import defer, error, reactor
+from zope.interface import implements
+
+from ooni.utils import log, date
+from ooni.utils.config import ValueChecker
+
+from ooni.plugoo.tests import ITest, OONITest
+from ooni.plugoo.assets import Asset, MissingAssetException
+from ooni.utils.onion import TxtorconImportError
+from ooni.utils.onion import PTNoBridgesException, PTNotFoundException
+
+try:
+ from ooni.utils.onion import parse_data_dir
+except:
+ log.msg("Please go to /ooni/lib and do 'make txtorcon' to run this test!")
+
+class RandomPortException(Exception):
+ """Raised when using a random port conflicts with configured ports."""
+ def __init__(self):
+ log.msg("Unable to use random and specific ports simultaneously")
+ return sys.exit()
+
+class BridgetArgs(usage.Options):
+ """Commandline options."""
+ allowed = "Port to use for Tor's %s, must be between 1024 and 65535."
+ sock_check = ValueChecker(allowed % "SocksPort").port_check
+ ctrl_check = ValueChecker(allowed % "ControlPort").port_check
+
+ optParameters = [
+ ['bridges', 'b', None,
+ 'File listing bridge IP:ORPorts to test'],
+ ['relays', 'f', None,
+ 'File listing relay IPs to test'],
+ ['socks', 's', 9049, None, sock_check],
+ ['control', 'c', 9052, None, ctrl_check],
+ ['torpath', 'p', None,
+ 'Path to the Tor binary to use'],
+ ['datadir', 'd', None,
+ 'Tor DataDirectory to use'],
+ ['transport', 't', None,
+ 'Tor ClientTransportPlugin'],
+ ['resume', 'r', 0,
+ 'Resume at this index']]
+ optFlags = [['random', 'x', 'Use random ControlPort and SocksPort']]
+
+ def postOptions(self):
+ if not self['bridges'] and not self['relays']:
+ raise MissingAssetException(
+ "Bridget can't run without bridges or relays to test!")
+ if self['transport']:
+ ValueChecker.uid_check(
+ "Can't run bridget as root with pluggable transports!")
+ if not self['bridges']:
+ raise PTNoBridgesException
+ if self['socks'] or self['control']:
+ if self['random']:
+ raise RandomPortException
+ if self['datadir']:
+ ValueChecker.dir_check(self['datadir'])
+ if self['torpath']:
+ ValueChecker.file_check(self['torpath'])
+
+class BridgetAsset(Asset):
+ """Class for parsing bridget Assets ignoring commented out lines."""
+ def __init__(self, file=None):
+ self = Asset.__init__(self, file)
+
+ def parse_line(self, line):
+ if line.startswith('#'):
+ return
+ else:
+ return line.replace('\n','')
+
+class BridgetTest(OONITest):
+ """
+ XXX fill me in
+
+ :ivar config:
+ An :class:`ooni.lib.txtorcon.TorConfig` instance.
+ :ivar relays:
+ A list of all provided relays to test.
+ :ivar bridges:
+ A list of all provided bridges to test.
+ :ivar socks_port:
+ Integer for Tor's SocksPort.
+ :ivar control_port:
+ Integer for Tor's ControlPort.
+ :ivar transport:
+ String defining the Tor's ClientTransportPlugin, for testing
+ a bridge's pluggable transport functionality.
+ :ivar tor_binary:
+ Path to the Tor binary to use, e.g. \'/usr/sbin/tor\'
+ """
+ implements(IPlugin, ITest)
+
+ shortName = "bridget"
+ description = "Use a Tor process to test connecting to bridges or relays"
+ requirements = None
+ options = BridgetArgs
+ blocking = False
+
+ def initialize(self):
+ """
+ Extra initialization steps. We only want one child Tor process
+ running, so we need to deal with most of the TorConfig() only once,
+ before the experiment runs.
+ """
+ self.socks_port = 9049
+ self.control_port = 9052
+ self.circuit_timeout = 90
+ self.tor_binary = '/usr/sbin/tor'
+ self.data_directory = None
+
+ def __make_asset_list__(opt, lst):
+ log.msg("Loading information from %s ..." % opt)
+ with open(opt) as opt_file:
+ for line in opt_file.readlines():
+ if line.startswith('#'):
+ continue
+ else:
+ lst.append(line.replace('\n',''))
+
+ def __count_remaining__(which):
+ total, reach, unreach = map(lambda x: which[x],
+ ['all', 'reachable', 'unreachable'])
+ count = len(total) - reach() - unreach()
+ return count
+
+ ## XXX should we do report['bridges_up'].append(self.bridges['current'])
+ self.bridges = {}
+ self.bridges['all'], self.bridges['up'], self.bridges['down'] = \
+ ([] for i in range(3))
+ self.bridges['reachable'] = lambda: len(self.bridges['up'])
+ self.bridges['unreachable'] = lambda: len(self.bridges['down'])
+ self.bridges['remaining'] = lambda: __count_remaining__(self.bridges)
+ self.bridges['current'] = None
+ self.bridges['pt_type'] = None
+ self.bridges['use_pt'] = False
+
+ self.relays = {}
+ self.relays['all'], self.relays['up'], self.relays['down'] = \
+ ([] for i in range(3))
+ self.relays['reachable'] = lambda: len(self.relays['up'])
+ self.relays['unreachable'] = lambda: len(self.relays['down'])
+ self.relays['remaining'] = lambda: __count_remaining__(self.relays)
+ self.relays['current'] = None
+
+ if self.local_options:
+ try:
+ from ooni.lib.txtorcon import TorConfig
+ except ImportError:
+ raise TxtorconImportError
+ else:
+ self.config = TorConfig()
+ finally:
+ options = self.local_options
+
+ if options['bridges']:
+ self.config.UseBridges = 1
+ __make_asset_list__(options['bridges'], self.bridges['all'])
+ if options['relays']:
+ ## first hop must be in TorState().guards
+ self.config.EntryNodes = ','.join(relay_list)
+ __make_asset_list__(options['relays'], self.relays['all'])
+ if options['socks']:
+ self.socks_port = options['socks']
+ if options['control']:
+ self.control_port = options['control']
+ if options['random']:
+ log.msg("Using randomized ControlPort and SocksPort ...")
+ self.socks_port = randint(1024, 2**16)
+ self.control_port = randint(1024, 2**16)
+ if options['torpath']:
+ self.tor_binary = options['torpath']
+ if options['datadir']:
+ self.data_directory = parse_data_dir(options['datadir'])
+ if options['transport']:
+ ## ClientTransportPlugin transport exec pathtobinary [options]
+ ## XXX we need a better way to deal with all PTs
+ log.msg("Using ClientTransportPlugin %s" % options['transport'])
+ self.bridges['use_pt'] = True
+ [self.bridges['pt_type'], pt_exec] = \
+ options['transport'].split(' ', 1)
+
+ if self.bridges['pt_type'] == "obfs2":
+ self.config.ClientTransportPlugin = \
+ self.bridges['pt_type'] + " " + pt_exec
+ else:
+ raise PTNotFoundException
+
+ self.config.SocksPort = self.socks_port
+ self.config.ControlPort = self.control_port
+ self.config.CookieAuthentication = 1
+
+ def __load_assets__(self):
+ """
+ Load bridges and/or relays from files given in user options. Bridges
+ should be given in the form IP:ORport. We don't want to load these as
+ assets, because it's inefficient to start a Tor process for each one.
+
+ We cannot use the Asset model, because that model calls
+ self.experiment() with the current Assets, which would be one relay
+ and one bridge, then it gives the defer.Deferred returned from
+ self.experiment() to self.control(), which means that, for each
+ (bridge, relay) pair, experiment gets called again, which instantiates
+ an additional Tor process that attempts to bind to the same
+ ports. Thus, additionally instantiated Tor processes return with
+ RuntimeErrors, which break the final defer.chainDeferred.callback(),
+ sending it into the errback chain.
+ """
+ assets = {}
+ if self.local_options:
+ if self.local_options['bridges']:
+ assets.update({'bridge':
+ BridgetAsset(self.local_options['bridges'])})
+ if self.local_options['relays']:
+ assets.update({'relay':
+ BridgetAsset(self.local_options['relays'])})
+ return assets
+
+ def experiment(self, args):
+ """
+ if bridges:
+ 1. configure first bridge line
+ 2a. configure data_dir, if it doesn't exist
+ 2b. write torrc to a tempfile in data_dir
+ 3. start tor } if any of these
+ 4. remove bridges which are public relays } fail, add current
+ 5. SIGHUP for each bridge } bridge to unreach-
+ } able bridges.
+ if relays:
+ 1a. configure the data_dir, if it doesn't exist
+ 1b. write torrc to a tempfile in data_dir
+ 2. start tor
+ 3. remove any of our relays which are already part of current
+ circuits
+ 4a. attach CustomCircuit() to self.state
+ 4b. RELAY_EXTEND for each relay } if this fails, add
+ } current relay to list
+ } of unreachable relays
+ 5.
+ if bridges and relays:
+ 1. configure first bridge line
+ 2a. configure data_dir if it doesn't exist
+ 2b. write torrc to a tempfile in data_dir
+ 3. start tor
+ 4. remove bridges which are public relays
+ 5. remove any of our relays which are already part of current
+ circuits
+ 6a. attach CustomCircuit() to self.state
+ 6b. for each bridge, build three circuits, with three
+ relays each
+ 6c. RELAY_EXTEND for each relay } if this fails, add
+ } current relay to list
+ } of unreachable relays
+
+ :param args:
+ The :class:`BridgetAsset` line currently being used. Except that it
+ in Bridget it doesn't, so it should be ignored and avoided.
+ """
+ try:
+ from ooni.utils import process
+ from ooni.utils.onion import remove_public_relays, start_tor
+ from ooni.utils.onion import start_tor_filter_nodes
+ from ooni.utils.onion import setup_fail, setup_done
+ from ooni.utils.onion import CustomCircuit
+ from ooni.utils.timer import deferred_timeout, TimeoutError
+ from ooni.lib.txtorcon import TorConfig, TorState
+ except ImportError:
+ raise TxtorconImportError
+ except TxtorconImportError, tie:
+ log.err(tie)
+ sys.exit()
+
+ def reconfigure_done(state, bridges):
+ """
+ Append :ivar:`bridges['current']` to the list
+ :ivar:`bridges['up'].
+ """
+ log.msg("Reconfiguring with 'Bridge %s' successful"
+ % bridges['current'])
+ bridges['up'].append(bridges['current'])
+ return state
+
+ def reconfigure_fail(state, bridges):
+ """
+ Append :ivar:`bridges['current']` to the list
+ :ivar:`bridges['down'].
+ """
+ log.msg("Reconfiguring TorConfig with parameters %s failed"
+ % state)
+ bridges['down'].append(bridges['current'])
+ return state
+
+ @defer.inlineCallbacks
+ def reconfigure_bridge(state, bridges):
+ """
+ Rewrite the Bridge line in our torrc. If use of pluggable
+ transports was specified, rewrite the line as:
+ Bridge <transport_type> <IP>:<ORPort>
+ Otherwise, rewrite in the standard form:
+ Bridge <IP>:<ORPort>
+
+ :param state:
+ A fully bootstrapped instance of
+ :class:`ooni.lib.txtorcon.TorState`.
+ :param bridges:
+ A dictionary of bridges containing the following keys:
+
+ bridges['remaining'] :: A function returning and int for the
+ number of remaining bridges to test.
+ bridges['current'] :: A string containing the <IP>:<ORPort>
+ of the current bridge.
+ bridges['use_pt'] :: A boolean, True if we're testing
+ bridges with a pluggable transport;
+ False otherwise.
+ bridges['pt_type'] :: If :ivar:`bridges['use_pt'] is True,
+ this is a string containing the type
+ of pluggable transport to test.
+ :return:
+ :param:`state`
+ """
+ log.msg("Current Bridge: %s" % bridges['current'])
+ log.msg("We now have %d bridges remaining to test..."
+ % bridges['remaining']())
+ try:
+ if bridges['use_pt'] is False:
+ controller_response = yield state.protocol.set_conf(
+ 'Bridge', bridges['current'])
+ elif bridges['use_pt'] and bridges['pt_type'] is not None:
+ controller_reponse = yield state.protocol.set_conf(
+ 'Bridge', bridges['pt_type'] +' '+ bridges['current'])
+ else:
+ raise PTNotFoundException
+
+ if controller_response == 'OK':
+ finish = yield reconfigure_done(state, bridges)
+ else:
+ log.err("SETCONF for %s responded with error:\n %s"
+ % (bridges['current'], controller_response))
+ finish = yield reconfigure_fail(state, bridges)
+
+ defer.returnValue(finish)
+
+ except Exception, e:
+ log.err("Reconfiguring torrc with Bridge line %s failed:\n%s"
+ % (bridges['current'], e))
+ defer.returnValue(None)
+
+ def attacher_extend_circuit(attacher, deferred, router):
+ ## XXX todo write me
+ ## state.attacher.extend_circuit
+ raise NotImplemented
+ #attacher.extend_circuit
+
+ def state_attach(state, path):
+ log.msg("Setting up custom circuit builder...")
+ attacher = CustomCircuit(state)
+ state.set_attacher(attacher, reactor)
+ state.add_circuit_listener(attacher)
+ return state
+
+ ## OLD
+ #for circ in state.circuits.values():
+ # for relay in circ.path:
+ # try:
+ # relay_list.remove(relay)
+ # except KeyError:
+ # continue
+ ## XXX how do we attach to circuits with bridges?
+ d = defer.Deferred()
+ attacher.request_circuit_build(d)
+ return d
+
+ def state_attach_fail(state):
+ log.err("Attaching custom circuit builder failed: %s" % state)
+
+ log.msg("Bridget: initiating test ... ") ## Start the experiment
+
+ ## if we've at least one bridge, and our config has no 'Bridge' line
+ if self.bridges['remaining']() >= 1 \
+ and not 'Bridge' in self.config.config:
+
+ ## configure our first bridge line
+ self.bridges['current'] = self.bridges['all'][0]
+ self.config.Bridge = self.bridges['current']
+ ## avoid starting several
+ self.config.save() ## processes
+ assert self.config.config.has_key('Bridge'), "No Bridge Line"
+
+ ## start tor and remove bridges which are public relays
+ from ooni.utils.onion import start_tor_filter_nodes
+ state = start_tor_filter_nodes(reactor, self.config,
+ self.control_port, self.tor_binary,
+ self.data_directory, self.bridges)
+ #controller = defer.Deferred()
+ #controller.addCallback(singleton_semaphore, tor)
+ #controller.addErrback(setup_fail)
+ #bootstrap = defer.gatherResults([controller, filter_bridges],
+ # consumeErrors=True)
+
+ if state is not None:
+ log.debug("state:\n%s" % state)
+ log.debug("Current callbacks on TorState():\n%s"
+ % state.callbacks)
+
+ ## if we've got more bridges
+ if self.bridges['remaining']() >= 2:
+ #all = []
+ for bridge in self.bridges['all'][1:]:
+ self.bridges['current'] = bridge
+ #new = defer.Deferred()
+ #new.addCallback(reconfigure_bridge, state, self.bridges)
+ #all.append(new)
+ #check_remaining = defer.DeferredList(all, consumeErrors=True)
+ #state.chainDeferred(check_remaining)
+ state.addCallback(reconfigure_bridge, self.bridges)
+
+ if self.relays['remaining']() > 0:
+ while self.relays['remaining']() >= 3:
+ #path = list(self.relays.pop() for i in range(3))
+ #log.msg("Trying path %s" % '->'.join(map(lambda node:
+ # node, path)))
+ self.relays['current'] = self.relays['all'].pop()
+ for circ in state.circuits.values():
+ for node in circ.path:
+ if node == self.relays['current']:
+ self.relays['up'].append(self.relays['current'])
+ if len(circ.path) < 3:
+ try:
+ ext = attacher_extend_circuit(state.attacher, circ,
+ self.relays['current'])
+ ext.addCallback(attacher_extend_circuit_done,
+ state.attacher, circ,
+ self.relays['current'])
+ except Exception, e:
+ log.err("Extend circuit failed: %s" % e)
+ else:
+ continue
+
+ #state.callback(all)
+ #self.reactor.run()
+ return state
+
+ def startTest(self, args):
+ """
+ Local override of :meth:`OONITest.startTest` to bypass calling
+ self.control.
+
+ :param args:
+ The current line of :class:`Asset`, not used but kept for
+ compatibility reasons.
+ :return:
+ A fired deferred which callbacks :meth:`experiment` and
+ :meth:`OONITest.finished`.
+ """
+ self.start_time = date.now()
+ self.d = self.experiment(args)
+ self.d.addErrback(log.err)
+ self.d.addCallbacks(self.finished, log.err)
+ return self.d
+
+## So that getPlugins() can register the Test:
+#bridget = BridgetTest(None, None, None)
+
+## ISIS' NOTES
+## -----------
+## TODO:
+## x cleanup documentation
+## x add DataDirectory option
+## x check if bridges are public relays
+## o take bridge_desc file as input, also be able to give same
+## format as output
+## x Add asynchronous timeout for deferred, so that we don't wait
+## o Add assychronous timout for deferred, so that we don't wait
+## forever for bridges that don't work.
diff --git a/ooni/bridget/utils/__init__.py b/ooni/bridget/utils/__init__.py
new file mode 100644
index 0000000..92893d6
--- /dev/null
+++ b/ooni/bridget/utils/__init__.py
@@ -0,0 +1 @@
+all = ['inputs', 'log', 'onion', 'tests', 'interface', 'nodes', 'reports', 'work']
diff --git a/ooni/bridget/utils/inputs.py b/ooni/bridget/utils/inputs.py
new file mode 100644
index 0000000..fe058cc
--- /dev/null
+++ b/ooni/bridget/utils/inputs.py
@@ -0,0 +1,174 @@
+#-*- coding: utf-8 -*-
+#
+# inputs.py
+# *********
+#
+# "...quis custodiet ipsos custodes?"
+# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.)
+#
+# :copyright: (c) 2012 Isis Lovecruft
+# :license: see LICENSE for more details.
+# :version: 0.1.0-beta
+#
+
+#from types import FunctionType, FileType
+import types
+
+from ooni.bridget import log
+from ooni.utils import date, Storage
+
+class InputFile:
+ """
+ This is a class describing a file used to store Tor bridge or relays
+ inputs. It is a python iterator object, allowing it to be efficiently
+ looped.
+
+ This class should not be used directly, but rather its subclasses,
+ BridgeFile and RelayFile should be used instead.
+ """
+
+ def __init__(self, file, **kw):
+ """
+ ## This is an InputAsset file, created because you tried to pass a
+ ## non-existent filename to a test.
+ ##
+ ## To use this file, place one input to be tested per line. Each
+ ## test takes different inputs. Lines which are commented out with
+ ## a '#' are not used.
+ """
+ self.file = file
+ self.eof = False
+ self.all = Storage()
+
+ for key, value in input_dict:
+ self.all[key] = value
+
+ try:
+ self.handler = open(self.file, 'r')
+ except IOError:
+ with open(self.file, 'w') as explain:
+ for line in self.__init__.__doc__:
+ explain.writeline(line)
+ self.handler = open(self.file, 'r')
+ try:
+ assert isinstance(self.handler, file), "That's not a file!"
+ except AssertionError, ae:
+ log.err(ae)
+
+ # def __handler__(self):
+ # """
+ # Attempt to open InputFile.file and check that it is actually a file.
+ # If it's not, create it and add an explaination for how InputFile files
+ # should be used.
+
+ # :return:
+ # A :type:`file` which has been opened in read-only mode.
+ # """
+ # try:
+ # handler = open(self.file, 'r')
+ # except IOError, ioerror: ## not the hacker <(A)3
+ # log.err(ioerror)
+ # explanation = (
+ # with open(self.file, 'w') as explain:
+ # for line in explanation:
+ # explain.writeline(line)
+ # handler = open(self.file, 'r')
+ # try:
+ # assert isinstance(handler, file), "That's not a file!"
+ # except AssertionError, ae:
+ # log.err(ae)
+ # else:
+ # return handler
+
+ def __iter__(next, StopIteration):
+ """
+ Returns the next input from the file.
+ """
+ #return self.next()
+ return self
+
+ def len(self):
+ """
+ Returns the number of the lines in the InputFile.
+ """
+ with open(self.file, 'r') as input_file:
+ lines = input_file.readlines()
+ for number, line in enumerate(lines):
+ self.input_dict[number] = line
+ return number + 1
+
+ def next(self):
+ try:
+ return self.next_input()
+ except:
+ raise StopIteration
+
+ def next_input(self):
+ """
+ Return the next input.
+ """
+ line = self.handler.readline()
+ if line:
+ parsed_line = self.parse_line(line)
+ if parsed_line:
+ return parsed_line
+ else:
+ self.fh.seek(0)
+ raise StopIteration
+
+ def default_parser(self, line):
+ """
+ xxx fill me in
+ """
+ if not line.startswith('#'):
+ return line.replace('\n', '')
+ else:
+ return False
+
+ def parse_line(self, line):
+ """
+ Override this method if you need line by line parsing of an Asset.
+
+ The default parsing action is to ignore lines which are commented out
+ with a '#', and to strip the newline character from the end of the
+ line.
+
+ If the line was commented out return an empty string instead.
+
+ If a subclass Foo incorporates another class Bar, when Bar is not
+ also a subclass of InputFile, and Bar.parse_line() exists, then
+ do not overwrite Bar's parse_line method.
+ """
+ assert not hasattr(super(InputFile, self), 'parse_line')
+
+ if self.parser is None:
+ if not line.startswith('#'):
+ return line.replace('\n', '')
+ else:
+ return ''
+ else:
+ try:
+ assert isinstance(self.parser, FunctionType),"Not a function!"
+ except AssertionError, ae:
+ log.err(ae)
+ else:
+ return self.parser(line)
+
+class BridgeFile(InputFile):
+ """
+ xxx fill me in
+ """
+ def __init__(self, **kw):
+ super(BridgeFile, self).__init__(**kw)
+
+class MissingInputException(Exception):
+ """
+
+ Raised when an :class:`InputFile` necessary for running the Test is
+ missing.
+
+ """
+ def __init__(self, error_message):
+ print error_message
+ import sys
+ return sys.exit()
diff --git a/ooni/bridget/utils/interface.py b/ooni/bridget/utils/interface.py
new file mode 100644
index 0000000..aa55436
--- /dev/null
+++ b/ooni/bridget/utils/interface.py
@@ -0,0 +1,54 @@
+from zope.interface import implements, Interface, Attribute
+
+class ITest(Interface):
+ """
+ This interface represents an OONI test. It fires a deferred on completion.
+ """
+
+ shortName = Attribute("""A short user facing description for this test""")
+ description = Attribute("""A string containing a longer description for the test""")
+
+ requirements = Attribute("""What is required to run this this test, for example raw socket access or UDP or TCP""")
+
+ options = Attribute("""These are the arguments to be passed to the test for it's execution""")
+
+ blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""")
+
+ def control(experiment_result, args):
+ """
+ @param experiment_result: The result returned by the experiment method.
+
+ @param args: the keys of this dict are the names of the assets passed in
+ from load_assets. The value is one item of the asset.
+
+ Must return a dict containing what should be written to the report.
+ Anything returned by control ends up inside of the YAMLOONI report.
+ """
+
+ def experiment(args):
+ """
+ Perform all the operations that are necessary to running a test.
+
+ @param args: the keys of this dict are the names of the assets passed in
+ from load_assets. The value is one item of the asset.
+
+ Must return a dict containing the values to be passed to control.
+ """
+
+ def load_assets():
+ """
+ Load the assets that should be passed to the Test. These are the inputs
+ to the OONI test.
+ Must return a dict that has as keys the asset names and values the
+ asset contents.
+ If the test does not have any assets it should return an empty dict.
+ """
+
+ def end():
+ """
+ This can be called at any time to terminate the execution of all of
+ these test instances.
+
+ What this means is that no more test instances with new parameters will
+ be created. A report will be written.
+ """
diff --git a/ooni/bridget/utils/log.py b/ooni/bridget/utils/log.py
new file mode 100644
index 0000000..eef50d8
--- /dev/null
+++ b/ooni/bridget/utils/log.py
@@ -0,0 +1,98 @@
+"""
+OONI logging facility.
+"""
+from sys import stderr, stdout
+
+from twisted.python import log, util
+from twisted.python.failure import Failure
+
+def _get_log_level(level):
+ english = ['debug', 'info', 'warn', 'err', 'crit']
+
+ levels = dict(zip(range(len(english)), english))
+ number = dict(zip(english, range(len(english))))
+
+ if not level:
+ return number['info']
+ else:
+ ve = "Unknown log level: %s\n" % level
+ ve += "Allowed levels: %s\n" % [word for word in english]
+
+ if type(level) is int:
+ if 0 <= level <= 4:
+ return level
+ elif type(level) is str:
+ if number.has_key(level.lower()):
+ return number[level]
+ else:
+ raise ValueError, ve
+ else:
+ raise ValueError, ve
+
+class OONITestFailure(Failure):
+ """
+ For handling Exceptions asynchronously.
+
+ Can be given an Exception as an argument, else will use the
+ most recent Exception from the current stack frame.
+ """
+ def __init__(self, exception=None, _type=None,
+ _traceback=None, _capture=False):
+ Failure.__init__(self, exc_type=_type,
+ exc_tb=_traceback, captureVars=_capture)
+
+class OONILogObserver(log.FileLogObserver):
+ """
+ Supports logging level verbosity.
+ """
+ def __init__(self, logfile, verb=None):
+ log.FileLogObserver.__init__(self, logfile)
+ self.level = _get_log_level(verb) if verb is not None else 1
+ assert type(self.level) is int
+
+ def emit(self, eventDict):
+ if 'logLevel' in eventDict:
+ msgLvl = _get_log_level(eventDict['logLevel'])
+ assert type(msgLvl) is int
+ ## only log our level and higher
+ if self.level <= msgLvl:
+ text = log.textFromEventDict(eventDict)
+ else:
+ text = None
+ else:
+ text = log.textFromEventDict(eventDict)
+
+ if text is None:
+ return
+
+ timeStr = self.formatTime(eventDict['time'])
+ fmtDict = {'system': eventDict['system'],
+ 'text': text.replace('\n','\n\t')}
+ msgStr = log._safeFormat("[%(system)s] %(text)s\n", fmtDict)
+
+ util.untilConcludes(self.write, timeStr + " " + msgStr)
+ util.untilConcludes(self.flush)
+
+def start(logfile=None, verbosity=None):
+ if log.defaultObserver:
+ verbosity = _get_log_level(verbosity)
+
+ ## Always log to file, keep level at info
+ file = open(logfile, 'a') if logfile else stderr
+ OONILogObserver(file, "info").start()
+
+ log.msg("Starting OONI...")
+
+def debug(message, level="debug", **kw):
+ print "[%s] %s" % (level, message)
+ ## If we want debug messages in the logfile:
+ #log.msg(message, logLevel=level, **kw)
+
+def msg(message, level="info", **kw):
+ log.msg(message, logLevel=level, **kw)
+
+def err(message, level="err", **kw):
+ log.err(logLevel=level, **kw)
+
+def fail(message, exception, level="crit", **kw):
+ log.failure(message, OONITestFailure(exception, **kw), logLevel=level)
diff --git a/ooni/bridget/utils/nodes.py b/ooni/bridget/utils/nodes.py
new file mode 100644
index 0000000..155f183
--- /dev/null
+++ b/ooni/bridget/utils/nodes.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8
+"""
+ nodes
+ *****
+
+ This contains all the code related to Nodes
+ both network and code execution.
+
+ :copyright: (c) 2012 by Arturo Filastò, Isis Lovecruft
+ :license: see LICENSE for more details.
+
+"""
+
+import os
+from binascii import hexlify
+
+try:
+ import paramiko
+except:
+ print "Error: module paramiko is not installed."
+from pprint import pprint
+import sys
+import socks
+import xmlrpclib
+
+class Node(object):
+ def __init__(self, address, port):
+ self.address = address
+ self.port = port
+
+class LocalNode(object):
+ def __init__(self):
+ pass
+
+"""
+[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5")
+[]: node_socket = node.wrap_socket()
+"""
+class NetworkNode(Node):
+ def __init__(self, address, port, node_type="SOCKS5", auth_creds=None):
+ self.node = Node(address,port)
+
+ # XXX support for multiple types
+ # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...)
+ self.node_type = node_type
+ # type-specific authentication credentials
+ self.auth_creds = auth_creds
+
+ def _get_socksipy_socket(self, proxy_type, auth_creds):
+ import socks
+ s = socks.socksocket()
+ # auth_creds[0] -> username
+ # auth_creds[1] -> password
+ s.setproxy(proxy_type, self.node.address, self.node.port,
+ self.auth_creds[0], self.auth_creds[1])
+ return s
+
+ def _get_socket_wrapper(self):
+ if (self.node_type.startswith("SOCKS")): # SOCKS proxies
+ if (self.node_type != "SOCKS5"):
+ proxy_type = socks.PROXY_TYPE_SOCKS5
+ elif (self.node_type != "SOCKS4"):
+ proxy_type = socks.PROXY_TYPE_SOCKS4
+ else:
+ print "We don't know this proxy type."
+ sys.exit(1)
+
+ return self._get_socksipy_socket(proxy_type)
+ elif (self.node_type == "HTTP"): # HTTP proxies
+ return self._get_socksipy_socket(PROXY_TYPE_HTTP)
+ else: # Unknown proxies
+ print "We don't know this proxy type."
+ sys.exit(1)
+
+ def wrap_socket(self):
+ return self._get_socket_wrapper()
+
+class CodeExecNode(Node):
+ def __init__(self, address, port, node_type, auth_creds):
+ self.node = Node(address,port)
+
+ # node type (SSH proxy, etc.)
+ self.node_type = node_type
+ # type-specific authentication credentials
+ self.auth_creds = auth_creds
+
+ def add_unit(self):
+ pass
+
+ def get_status(self):
+ pass
+
+class PlanetLab(CodeExecNode):
+ def __init__(self, address, auth_creds, ooni):
+ self.auth_creds = auth_creds
+
+ self.config = ooni.utils.config
+ self.logger = ooni.logger
+ self.name = "PlanetLab"
+
+ def _api_auth(self):
+ api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/')
+ auth = {}
+ ## should be changed to separate node.conf file
+ auth['Username'] = self.config.main.pl_username
+ auth['AuthString'] = self.config.main.pl_password
+ auth['AuthMethod'] = "password"
+ authorized = api_server.AuthCheck(auth)
+
+ if authorized:
+ print 'We are authorized!'
+ return auth
+ else:
+ print 'Authorization failed. Please check your settings for pl_username and pl_password in the ooni-probe.conf file.'
+
+ def _search_for_nodes(self, node_filter=None):
+ api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True)
+ node_filter = {'hostname': '*.cert.org.cn'}
+ return_fields = ['hostname', 'site_id']
+ all_nodes = api_server.GetNodes(self.api_auth(), node_filter, boot_state_filter)
+ pprint(all_nodes)
+ return all_nodes
+
+ def _add_nodes_to_slice(self):
+ api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True)
+ all_nodes = self.search_for_nodes()
+ for node in all_nodes:
+ api_server.AddNode(self.api_auth(), node['site_id'], all_nodes)
+ print 'Adding nodes %s' % node['hostname']
+
+ def _auth_login(slicename, machinename):
+ """Attempt to authenticate to the given PL node, slicename and
+ machinename, using any of the private keys in ~/.ssh/ """
+
+ agent = paramiko.Agent()
+ agent_keys = agent.get_keys()
+ if len(agent_keys) == 0:
+ return
+
+ for key in agent_keys:
+ print 'Trying ssh-agent key %s' % hexlify(key.get_fingerprint()),
+ try:
+ paramiko.transport.auth_publickey(machinename, slicename)
+ print 'Public key authentication to PlanetLab node %s successful.' % machinename,
+ return
+ except paramiko.SSHException:
+ print 'Public key authentication to PlanetLab node %s failed.' % machinename,
+
+ def _get_command():
+ pass
+
+ def ssh_and_run_(slicename, machinename, command):
+ """Attempt to make a standard OpenSSH client to PL node, and run
+ commands from a .conf file."""
+
+ ## needs a way to specify 'ssh -l <slicename> <machinename>'
+ ## with public key authentication.
+
+ command = PlanetLab.get_command()
+
+ client = paramiko.SSHClient()
+ client.load_system_host_keys()
+ client.connect(machinename)
+
+ stdin, stdout, stderr = client.exec_command(command)
+
+ def send_files_to_node(directory, files):
+ """Attempt to rsync a tree to the PL node."""
+ pass
+
+ def add_unit():
+ pass
+
+ def get_status():
+ pass
diff --git a/ooni/bridget/utils/onion.py b/ooni/bridget/utils/onion.py
new file mode 100644
index 0000000..9d4cae7
--- /dev/null
+++ b/ooni/bridget/utils/onion.py
@@ -0,0 +1,686 @@
+#
+# onion.py
+# ----------
+# Utilities for working with Tor.
+#
+# This code is largely taken from txtorcon and its documentation, and as such
+# any and all credit should go to Meejah. Minor adjustments have been made to
+# use OONI's logging system, and to build custom circuits without actually
+# attaching streams.
+#
+# :author: Meejah, Isis Lovecruft
+# :license: see included LICENSE file
+# :copyright: copyright (c) 2012 The Tor Project, Inc.
+# :version: 0.1.0-alpha
+#
+# XXX TODO add report keys for onion methods
+
+import random
+import sys
+
+from twisted.internet import defer
+from zope.interface import implements
+
+from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher
+from ooni.lib.txtorcon import TorState, TorConfig
+from ooni.utils import log
+from ooni.utils.timer import deferred_timeout, TimeoutError
+
+def parse_data_dir(data_dir):
+ """
+ Parse a string that a has been given as a DataDirectory and determine
+ its absolute path on the filesystem.
+
+ :param data_dir:
+ A directory for Tor's DataDirectory, to be parsed.
+ :return:
+ The absolute path of :param:data_dir.
+ """
+ from os import path, getcwd
+ import sys
+
+ try:
+ assert isinstance(data_dir, str), \
+ "Parameter type(data_dir) must be str"
+ except AssertionError, ae:
+ log.err(ae)
+
+ if data_dir.startswith('~'):
+ data_dir = path.expanduser(data_dir)
+ elif data_dir.startswith('/'):
+ data_dir = path.join(getcwd(), data_dir)
+ elif data_dir.startswith('./'):
+ data_dir = path.abspath(data_dir)
+ else:
+ data_dir = path.join(getcwd(), data_dir)
+
+ try:
+ assert path.isdir(data_dir), "Could not find %s" % data_dir
+ except AssertionError, ae:
+ log.err(ae)
+ sys.exit()
+ else:
+ return data_dir
+
+def write_torrc(conf, data_dir=None):
+ """
+ Create a torrc in our data_dir. If we don't yet have a data_dir, create a
+ temporary one. Any temporary files or folders are added to delete_list.
+
+ :param conf:
+ A :class:`ooni.lib.txtorcon.TorConfig` object, with all configuration
+ values saved.
+ :param data_dir:
+ The Tor DataDirectory to use.
+ :return: torrc, data_dir, delete_list
+ """
+ try:
+ from os import write, close
+ from tempfile import mkstemp, mkdtemp
+ except ImportError, ie:
+ log.err(ie)
+
+ delete_list = []
+
+ if data_dir is None:
+ data_dir = mkdtemp(prefix='bridget-tordata')
+ delete_list.append(data_dir)
+ conf.DataDirectory = data_dir
+
+ (fd, torrc) = mkstemp(dir=data_dir)
+ delete_list.append(torrc)
+ write(fd, conf.create_torrc())
+ close(fd)
+
+ return torrc, data_dir, delete_list
+
+def delete_files_or_dirs(delete_list):
+ """
+ Given a list of files or directories to delete, delete all and suppress
+ all errors.
+
+ :param delete_list:
+ A list of files or directories to delete.
+ """
+ try:
+ from os import unlink
+ from shutil import rmtree
+ except ImportError, ie:
+ log.err(ie)
+
+ for temp in delete_list:
+ try:
+ unlink(temp)
+ except OSError:
+ rmtree(temp, ignore_errors=True)
+
+def remove_node_from_list(node, list):
+ for item in list: ## bridges don't match completely
+ if item.startswith(node): ## due to the :<port>.
+ try:
+ log.msg("Removing %s because it is a public relay" % node)
+ list.remove(item)
+ except ValueError, ve:
+ log.err(ve)
+
+def remove_public_relays(state, bridges):
+ """
+ Remove bridges from our bridge list which are also listed as public
+ relays. This must be called after Tor has fully bootstrapped and we have a
+ :class:`ooni.lib.txtorcon.TorState` with the
+ :attr:`ooni.lib.txtorcon.TorState.routers` attribute assigned.
+
+ XXX Does state.router.values() have all of the relays in the consensus, or
+ just the ones we know about so far?
+
+ XXX FIXME: There is a problem in that Tor needs a Bridge line to already be
+ configured in order to bootstrap. However, after bootstrapping, we grab the
+ microdescriptors of all the relays and check if any of our bridges are
+ listed as public relays. Because of this, the first bridge does not get
+ checked for being a relay.
+ """
+ IPs = map(lambda addr: addr.split(':',1)[0], bridges['all'])
+ both = set(state.routers.values()).intersection(IPs)
+
+ if len(both) > 0:
+ try:
+ updated = map(lambda node: remove_node_from_list(node), both)
+ log.debug("Bridges in both: %s" % both)
+ log.debug("Updated = %s" % updated)
+ #if not updated:
+ # defer.returnValue(state)
+ #else:
+ # defer.returnValue(state)
+ return state
+ except Exception, e:
+ log.err("Removing public relays %s from bridge list failed:\n%s"
+ % (both, e))
+
+def setup_done(proto):
+ log.msg("Setup Complete")
+ state = TorState(proto.tor_protocol)
+ state.post_bootstrap.addCallback(state_complete)
+ state.post_bootstrap.addErrback(setup_fail)
+
+def setup_fail(proto):
+ log.msg("Setup Failed:\n%s" % proto)
+ return proto
+ #reactor.stop()
+
+def state_complete(state):
+ """Called when we've got a TorState."""
+ log.msg("We've completely booted up a Tor version %s at PID %d"
+ % (state.protocol.version, state.tor_pid))
+ log.msg("This Tor has the following %d Circuits:"
+ % len(state.circuits))
+ for circ in state.circuits.values():
+ log.msg("%s" % circ)
+ return state
+
+def updates(_progress, _tag, _summary):
+ """Log updates on the Tor bootstrapping process."""
+ log.msg("%d%%: %s" % (_progress, _summary))
+
+def bootstrap(ctrl):
+ """
+ Bootstrap Tor from an instance of
+ :class:`ooni.lib.txtorcon.TorControlProtocol`.
+ """
+ conf = TorConfig(ctrl)
+ conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail)
+ log.msg("Tor process connected, bootstrapping ...")
+
+def start_tor(reactor, config, control_port, tor_binary, data_dir,
+ report=None, progress=updates,
+ process_cb=None, process_eb=None):
+ """
+ Use a txtorcon.TorConfig() instance, config, to write a torrc to a
+ tempfile in our DataDirectory, data_dir. If data_dir is None, a temp
+ directory will be created. Finally, create a TCP4ClientEndpoint at our
+ control_port, and connect it to our reactor and a spawned Tor
+ process. Compare with :meth:`txtorcon.launch_tor` for differences.
+
+ :param reactor:
+ An instance of class:`twisted.internet.reactor`.
+ :param config:
+ An instance of class:`txtorcon.TorConfig` with all torrc options
+ already configured. ivar:`config.ControlPort`,
+ ivar:`config.SocksPort`, ivar:`config.CookieAuthentication`, should
+ already be set, as well as ivar:`config.UseBridges` and
+ ivar:`config.Bridge` if bridges are to be used.
+ ivar:`txtorcon.DataDirectory` does not need to be set.
+ :param control_port:
+ The port number to use for Tor's ControlPort.
+ :param tor_binary:
+ The full path to the Tor binary to use.
+ :param data_dir:
+ The directory to use as Tor's DataDirectory.
+ :param report:
+ The class:`ooni.plugoo.reports.Report` instance.
+ :param progress:
+ A non-blocking function to handle bootstrapping updates, which takes
+ three parameters: _progress, _tag, and _summary.
+ :param process_cb:
+ The function to callback to after
+ class:`ooni.lib.txtorcon.TorProcessProtocol` returns with the fully
+ bootstrapped Tor process.
+ :param process_eb:
+ The function to errback to if
+ class:`ooni.lib.txtorcon.TorProcessProtocol` fails.
+ :return:
+ The result of the callback of a
+ class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a
+ class:`txtorcon.TorControlProtocol` as .protocol.
+ """
+ try:
+ from functools import partial
+ from twisted.internet.endpoints import TCP4ClientEndpoint
+ from ooni.lib.txtorcon import TorProtocolFactory
+ from ooni.lib.txtorcon import TorProcessProtocol
+ except ImportError, ie:
+ log.err(ie)
+
+ ## TODO: add option to specify an already existing torrc, which
+ ## will require prior parsing to enforce necessary lines
+ (torrc, data_dir, to_delete) = write_torrc(config, data_dir)
+
+ log.msg("Starting Tor ...")
+ log.msg("Using the following as our torrc:\n%s" % config.create_torrc())
+ if report is None:
+ report = {'torrc': config.create_torrc()}
+ else:
+ report.update({'torrc': config.create_torrc()})
+
+ end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port)
+ connection_creator = partial(end_point.connect, TorProtocolFactory())
+ process_protocol = TorProcessProtocol(connection_creator, progress)
+ process_protocol.to_delete = to_delete
+
+ if process_cb is not None and process_eb is not None:
+ process_protocol.connected_cb.addCallbacks(process_cb, process_eb)
+
+ reactor.addSystemEventTrigger('before', 'shutdown',
+ partial(delete_files_or_dirs, to_delete))
+ try:
+ transport = reactor.spawnProcess(process_protocol,
+ tor_binary,
+ args=(tor_binary,'-f',torrc),
+ env={'HOME': data_dir},
+ path=data_dir)
+ transport.closeStdin()
+ except RuntimeError, e:
+ log.err("Starting Tor failed:")
+ process_protocol.connected_cb.errback(e)
+ except NotImplementedError, e:
+ url = "http://starship.python.net/crew/mhammond/win32/Downloads.html"
+ log.msg("Running bridget on Windows requires pywin32: %s" % url)
+ process_protocol.connected_cb.errback(e)
+
+ return process_protocol.connected_cb
+
+ at defer.inlineCallbacks
+def start_tor_filter_nodes(reactor, config, control_port, tor_binary,
+ data_dir, bridges):
+ """
+ Bootstrap a Tor process and return a fully-setup
+ :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges
+ to test in the list of known public relays,
+ :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges
+ which are known public relays.
+
+ :param reactor:
+ The :class:`twisted.internet.reactor`.
+ :param config:
+ An instance of :class:`ooni.lib.txtorcon.TorConfig`.
+ :param control_port:
+ The port to use for Tor's ControlPort. If already configured in
+ the TorConfig instance, this can be given as
+ TorConfig.config.ControlPort.
+ :param tor_binary:
+ The full path to the Tor binary to execute.
+ :param data_dir:
+ The full path to the directory to use as Tor's DataDirectory.
+ :param bridges:
+ A dictionary which has a key 'all' which is a list of bridges to
+ test connecting to, e.g.:
+ bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001']
+ :return:
+ A fully initialized :class:`ooni.lib.txtorcon.TorState`.
+ """
+ setup = yield start_tor(reactor, config, control_port,
+ tor_binary, data_dir,
+ process_cb=setup_done, process_eb=setup_fail)
+ filter_nodes = yield remove_public_relays(setup, bridges)
+ defer.returnValue(filter_nodes)
+
+ at defer.inlineCallbacks
+def start_tor_with_timer(reactor, config, control_port, tor_binary, data_dir,
+ bridges, timeout):
+ """
+ Start bootstrapping a Tor process wrapped with an instance of the class
+ decorator :func:`ooni.utils.timer.deferred_timeout` and complete callbacks
+ to either :func:`setup_done` or :func:`setup_fail`. Return a fully-setup
+ :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges to test
+ in the list of known public relays,
+ :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges which
+ are listed as known public relays.
+
+ :param reactor:
+ The :class:`twisted.internet.reactor`.
+ :param config:
+ An instance of :class:`ooni.lib.txtorcon.TorConfig`.
+ :param control_port:
+ The port to use for Tor's ControlPort. If already configured in
+ the TorConfig instance, this can be given as
+ TorConfig.config.ControlPort.
+ :param tor_binary:
+ The full path to the Tor binary to execute.
+ :param data_dir:
+ The full path to the directory to use as Tor's DataDirectory.
+ :param bridges:
+ A dictionary which has a key 'all' which is a list of bridges to
+ test connecting to, e.g.:
+ bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001']
+ :param timeout:
+ The number of seconds to attempt to bootstrap the Tor process before
+ raising a :class:`ooni.utils.timer.TimeoutError`.
+ :return:
+ If the timeout limit is not exceeded, return a fully initialized
+ :class:`ooni.lib.txtorcon.TorState`, else return None.
+ """
+ error_msg = "Bootstrapping has exceeded the timeout limit..."
+ with_timeout = deferred_timeout(timeout, e=error_msg)(start_tor)
+ try:
+ setup = yield with_timeout(reactor, config, control_port, tor_binary,
+ data_dir, process_cb=setup_done,
+ process_eb=setup_fail)
+ except TimeoutError, te:
+ log.err(te)
+ defer.returnValue(None)
+ #except Exception, e:
+ # log.err(e)
+ # defer.returnValue(None)
+ else:
+ state = yield remove_public_relays(setup, bridges)
+ defer.returnValue(state)
+
+ at defer.inlineCallbacks
+def start_tor_filter_nodes_with_timer(reactor, config, control_port,
+ tor_binary, data_dir, bridges, timeout):
+ """
+ Start bootstrapping a Tor process wrapped with an instance of the class
+ decorator :func:`ooni.utils.timer.deferred_timeout` and complete callbacks
+ to either :func:`setup_done` or :func:`setup_fail`. Then, filter our list
+ of bridges to remove known public relays by calling back to
+ :func:`remove_public_relays`. Return a fully-setup
+ :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges to test
+ in the list of known public relays,
+ :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges which
+ are listed as known public relays.
+
+ :param reactor:
+ The :class:`twisted.internet.reactor`.
+ :param config:
+ An instance of :class:`ooni.lib.txtorcon.TorConfig`.
+ :param control_port:
+ The port to use for Tor's ControlPort. If already configured in
+ the TorConfig instance, this can be given as
+ TorConfig.config.ControlPort.
+ :param tor_binary:
+ The full path to the Tor binary to execute.
+ :param data_dir:
+ The full path to the directory to use as Tor's DataDirectory.
+ :param bridges:
+ A dictionary which has a key 'all' which is a list of bridges to
+ test connecting to, e.g.:
+ bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001']
+ :param timeout:
+ The number of seconds to attempt to bootstrap the Tor process before
+ raising a :class:`ooni.utils.timer.TimeoutError`.
+ :return:
+ If the timeout limit is not exceeded, return a fully initialized
+ :class:`ooni.lib.txtorcon.TorState`, else return None.
+ """
+ error_msg = "Bootstrapping has exceeded the timeout limit..."
+ with_timeout = deferred_timeout(timeout, e=error_msg)(start_tor_filter_nodes)
+ try:
+ state = yield with_timeout(reactor, config, control_port,
+ tor_binary, data_dir, bridges)
+ except TimeoutError, te:
+ log.err(te)
+ defer.returnValue(None)
+ #except Exception, e:
+ # log.err(e)
+ # defer.returnValue(None)
+ else:
+ defer.returnValue(state)
+
+class CustomCircuit(CircuitListenerMixin):
+ """
+ Utility class for controlling circuit building. See
+ 'attach_streams_by_country.py' in the txtorcon documentation.
+
+ :param state:
+ A fully bootstrapped instance of :class:`ooni.lib.txtorcon.TorState`.
+ :param relays:
+ A dictionary containing a key 'all', which is a list of relays to
+ test connecting to.
+ :ivar waiting_circuits:
+ The list of circuits which we are waiting to attach to. You shouldn't
+ need to touch this.
+ """
+ implements(IStreamAttacher)
+
+ def __init__(self, state, relays=None):
+ self.state = state
+ self.waiting_circuits = []
+ self.relays = relays
+
+ def waiting_on(self, circuit):
+ """
+ Whether or not we are waiting on the given circuit before attaching to
+ it.
+
+ :param circuit:
+ An item from :ivar:`ooni.lib.txtorcon.TorState.circuits`.
+ :return:
+ True if we are waiting on the circuit, False if not waiting.
+ """
+ for (circid, d) in self.waiting_circuits:
+ if circuit.id == circid:
+ return True
+ return False
+
+ def circuit_extend(self, circuit, router):
+ "ICircuitListener"
+ if circuit.purpose != 'GENERAL':
+ return
+ if self.waiting_on(circuit):
+ log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex))
+
+ def circuit_built(self, circuit):
+ "ICircuitListener"
+ if circuit.purpose != 'GENERAL':
+ return
+ log.msg("Circuit %s built ..." % circuit.id)
+ log.msg("Full path of %s: %s" % (circuit.id, circuit.path))
+ for (circid, d) in self.waiting_circuits:
+ if circid == circuit.id:
+ self.waiting_circuits.remove((circid, d))
+ d.callback(circuit)
+
+ def circuit_failed(self, circuit, reason):
+ """
+ If building a circuit has failed, try to remove it from our list of
+ :ivar:`waiting_circuits`, else request to build it.
+
+ :param circuit:
+ An item from :ivar:`ooni.lib.txtorcon.TorState.circuits`.
+ :param reason:
+ A :class:`twisted.python.fail.Failure` instance.
+ :return:
+ None
+ """
+ if self.waiting_on(circuit):
+ log.msg("Circuit %s failed for reason %s" % (circuit.id, reason))
+ circid, d = None, None
+ for c in self.waiting_circuits:
+ if c[0] == circuit.id:
+ circid, d = c
+ if d is None:
+ raise Exception("Expected to find circuit.")
+
+ self.waiting_circuits.remove((circid, d))
+ log.msg("Trying to build a circuit for %s" % circid)
+ self.request_circuit_build(d)
+
+ def check_circuit_route(self, router):
+ """
+ Check if a relay is a hop in one of our already built circuits.
+
+ :param router:
+ An item from the list
+ :func:`ooni.lib.txtorcon.TorState.routers.values()`.
+ """
+ for circ in self.state.circuits.values():
+ if router in circ.path:
+ #router.update() ## XXX can i use without args? no.
+ TorInfo.dump(self)
+
+ def request_circuit_build(self, deferred, path=None):
+ """
+ Request a custom circuit.
+
+ :param deferred:
+ A :class:`twisted.internet.defer.Deferred` for this circuit.
+ :param path:
+ A list of router ids to build a circuit from. The length of this
+ list must be at least three.
+ """
+ if path is None:
+
+ pick = self.relays['all'].pop
+ n = self.state.entry_guards.values()
+ choose = random.choice
+
+ first, middle, last = (None for i in range(3))
+
+ if self.relays['remaining']() >= 3:
+ first, middle, last = (pick() for i in range(3))
+ elif self.relays['remaining']() < 3:
+ first = choose(n)
+ middle = pick()
+ if self.relays['remaining'] == 2:
+ middle, last = (pick() for i in range(2))
+ elif self.relay['remaining'] == 1:
+ middle = pick()
+ last = choose(n)
+ else:
+ log.msg("Qu'est-que fuque?")
+ else:
+ middle, last = (random.choice(self.state.routers.values())
+ for i in range(2))
+
+ path = [first, middle, last]
+
+ else:
+ assert isinstance(path, list), \
+ "Circuit path must be a list of relays!"
+ assert len(path) >= 3, \
+ "Circuit path must be at least three hops!"
+
+ log.msg("Requesting a circuit: %s"
+ % '->'.join(map(lambda node: node, path)))
+
+ class AppendWaiting:
+ def __init__(self, attacher, deferred):
+ self.attacher = attacher
+ self.d = deferred
+ def __call__(self, circ):
+ """
+ Return from build_circuit is a Circuit, however,
+ we want to wait until it is built before we can
+ issue an attach on it and callback to the Deferred
+ we issue here.
+ """
+ log.msg("Circuit %s is in progress ..." % circ.id)
+ self.attacher.waiting_circuits.append((circ.id, self.d))
+
+ return self.state.build_circuit(path).addCallback(
+ AppendWaiting(self, deferred)).addErrback(
+ log.err)
+
+class TxtorconImportError(ImportError):
+ """
+ Raised when ooni.lib.txtorcon cannot be imported from. Checks our current
+ working directory and the path given to see if txtorcon has been
+ initialized via /ooni/lib/Makefile.
+ """
+ from os import getcwd, path
+
+ cwd, tx = getcwd(), 'lib/txtorcon/torconfig.py'
+ try:
+ log.msg("Unable to import from ooni.lib.txtorcon")
+ if cwd.endswith('ooni'):
+ check = path.join(cwd, tx)
+ elif cwd.endswith('utils'):
+ check = path.join(cwd, '../'+tx)
+ else:
+ check = path.join(cwd, 'ooni/'+tx)
+ assert path.isfile(check)
+ except:
+ log.msg("Error: Some OONI libraries are missing!")
+ log.msg("Please go to /ooni/lib/ and do \"make all\"")
+
+class PTNoBridgesException(Exception):
+ """Raised when a pluggable transport is specified, but not bridges."""
+ def __init__(self):
+ log.msg("Pluggable transport requires the bridges option")
+ return sys.exit()
+
+class PTNotFoundException(Exception):
+ def __init__(self, transport_type):
+ m = "Pluggable Transport type %s was unaccounted " % transport_type
+ m += "for, please contact isis(at)torproject(dot)org and it will "
+ m += "get included."
+ log.msg("%s" % m)
+ return sys.exit()
+
+ at defer.inlineCallbacks
+def __start_tor_with_timer__(reactor, config, control_port, tor_binary,
+ data_dir, bridges=None, relays=None, timeout=None,
+ retry=None):
+ """
+ A wrapper for :func:`start_tor` which wraps the bootstrapping of a Tor
+ process and its connection to a reactor with a
+ :class:`twisted.internet.defer.Deferred` class decorator utility,
+ :func:`ooni.utils.timer.deferred_timeout`, and a mechanism for resets.
+
+ ## XXX fill me in
+ """
+ raise NotImplementedError
+
+ class RetryException(Exception):
+ pass
+
+ import sys
+ from ooni.utils.timer import deferred_timeout, TimeoutError
+
+ def __make_var__(old, default, _type):
+ if old is not None:
+ assert isinstance(old, _type)
+ new = old
+ else:
+ new = default
+ return new
+
+ reactor = reactor
+ timeout = __make_var__(timeout, 120, int)
+ retry = __make_var__(retry, 1, int)
+
+ with_timeout = deferred_timeout(timeout)(start_tor)
+
+ @defer.inlineCallbacks
+ def __start_tor__(rc=reactor, cf=config, cp=control_port, tb=tor_binary,
+ dd=data_dir, br=bridges, rl=relays, cb=setup_done,
+ eb=setup_fail, af=remove_public_relays, retry=retry):
+ try:
+ setup = yield with_timeout(rc,cf,cp,tb,dd)
+ except TimeoutError:
+ retry -= 1
+ defer.returnValue(retry)
+ else:
+ if setup.callback:
+ setup = yield cb(setup)
+ elif setup.errback:
+ setup = yield eb(setup)
+ else:
+ setup = setup
+
+ if br is not None:
+ state = af(setup,br)
+ else:
+ state = setup
+ defer.returnValue(state)
+
+ @defer.inlineCallbacks
+ def __try_until__(tries):
+ result = yield __start_tor__()
+ try:
+ assert isinstance(result, int)
+ except AssertionError:
+ defer.returnValue(result)
+ else:
+ if result >= 0:
+ tried = yield __try_until__(result)
+ defer.returnValue(tried)
+ else:
+ raise RetryException
+ try:
+ tried = yield __try_until__(retry)
+ except RetryException:
+ log.msg("All retry attempts to bootstrap Tor have timed out.")
+ log.msg("Exiting ...")
+ defer.returnValue(sys.exit())
+ else:
+ defer.returnValue(tried)
diff --git a/ooni/bridget/utils/reports.py b/ooni/bridget/utils/reports.py
new file mode 100644
index 0000000..ae67b13
--- /dev/null
+++ b/ooni/bridget/utils/reports.py
@@ -0,0 +1,144 @@
+from __future__ import with_statement
+
+import os
+import yaml
+
+import itertools
+from ooni.utils import log, date, net
+
+class Report:
+ """This is the ooni-probe reporting mechanism. It allows
+ reporting to multiple destinations and file formats.
+
+ :scp the string of <host>:<port> of an ssh server
+
+ :yaml the filename of a the yaml file to write
+
+ :file the filename of a simple txt file to write
+
+ :tcp the <host>:<port> of a TCP server that will just listen for
+ inbound connection and accept a stream of data (think of it
+ as a `nc -l -p <port> > filename.txt`)
+ """
+ def __init__(self, testname=None, file="report.log",
+ scp=None,
+ tcp=None):
+
+ self.testname = testname
+ self.file = file
+ self.tcp = tcp
+ self.scp = scp
+ #self.config = ooni.config.report
+
+ #if self.config.timestamp:
+ # tmp = self.file.split('.')
+ # self.file = '.'.join(tmp[:-1]) + "-" + \
+ # datetime.now().isoformat('-') + '.' + \
+ # tmp[-1]
+ # print self.file
+
+ self.scp = None
+ self.write_header()
+
+ def write_header(self):
+ pretty_date = date.pretty_date()
+ header = "# OONI Probe Report for Test %s\n" % self.testname
+ header += "# %s\n\n" % pretty_date
+ self._write_to_report(header)
+ # XXX replace this with something proper
+ address = net.getClientAddress()
+ test_details = {'start_time': str(date.now()),
+ 'asn': address['asn'],
+ 'test_name': self.testname,
+ 'addr': address['ip']}
+ self(test_details)
+
+ def _write_to_report(self, dump):
+ reports = []
+
+ if self.file:
+ reports.append("file")
+
+ if self.tcp:
+ reports.append("tcp")
+
+ if self.scp:
+ reports.append("scp")
+
+ #XXX make this non blocking
+ for report in reports:
+ self.send_report(dump, report)
+
+ def __call__(self, data):
+ """
+ This should be invoked every time you wish to write some
+ data to the reporting system
+ """
+ dump = yaml.dump([data])
+ self._write_to_report(dump)
+
+ def file_report(self, data):
+ """
+ This reports to a file in YAML format
+ """
+ with open(self.file, 'a+') as f:
+ f.write(data)
+
+ def send_report(self, data, type):
+ """
+ This sends the report using the
+ specified type.
+ """
+ #print "Reporting %s to %s" % (data, type)
+ log.msg("Reporting to %s" % type)
+ getattr(self, type+"_report").__call__(data)
+
+class NewReport(object):
+ filename = 'report.log'
+ startTime = None
+ endTime = None
+ testName = None
+ ipAddr = None
+ asnAddr = None
+
+ def _open():
+ self.fp = open(self.filename, 'a+')
+
+ @property
+ def header():
+ pretty_date = date.pretty_date()
+ report_header = "# OONI Probe Report for Test %s\n" % self.testName
+ report_header += "# %s\n\n" % pretty_date
+ test_details = {'start_time': self.startTime,
+ 'asn': asnAddr,
+ 'test_name': self.testName,
+ 'addr': ipAddr}
+ report_header += yaml.dump([test_details])
+ return report_header
+
+ def create():
+ """
+ Create a new report by writing it's header.
+ """
+ self.fp = open(self.filename, 'w+')
+ self.fp.write(self.header)
+
+ def exists():
+ """
+ Returns False if the file does not exists.
+ """
+ return os.path.exists(self.filename)
+
+ def write(data):
+ """
+ Write a report to the file.
+
+ :data: python data structure to be written to report.
+ """
+ if not self.exists():
+ self.create()
+ else:
+ self._open()
+ yaml_encoded_data = yaml.dump([data])
+ self.fp.write(yaml_encoded_data)
+ self.fp.close()
diff --git a/ooni/bridget/utils/tests.py b/ooni/bridget/utils/tests.py
new file mode 100644
index 0000000..ea4be0b
--- /dev/null
+++ b/ooni/bridget/utils/tests.py
@@ -0,0 +1,141 @@
+import os
+import yaml
+from zope.interface import Interface, Attribute
+
+import logging
+import itertools
+from twisted.internet import reactor, defer, threads
+## XXX why is this imported and not used?
+from twisted.python import failure
+
+from ooni.utils import log, date
+from ooni.plugoo import assets, work
+from ooni.plugoo.reports import Report
+from ooni.plugoo.interface import ITest
+
+class OONITest(object):
+ """
+ This is the base class for writing OONI Tests.
+
+ It should be used in conjunction with the ITest Interface. It allows the
+ developer to benefit from OONIs reporting system and command line argument
+ parsing system.
+ """
+ name = "oonitest"
+ # By default we set this to False, meaning that we don't block
+ blocking = False
+ reactor = reactor
+ tool = False
+ ended = False
+
+ def __init__(self, local_options, global_options, report, ooninet=None,
+ reactor=reactor):
+ # These are the options that are read through the tests suboptions
+ self.local_options = local_options
+ # These are the options global to all of OONI
+ self.global_options = global_options
+ self.report = report
+ #self.ooninet = ooninet
+ self.reactor = reactor
+ self.result = {}
+ self.initialize()
+ self.assets = self.load_assets()
+
+ def initialize(self):
+ """
+ Override this method if you are interested in having some extra
+ behavior when your test class is instantiated.
+ """
+ pass
+
+ def load_assets(self):
+ """
+ This method should be overriden by the test writer to provide the
+ logic for loading their assets.
+ """
+ return {}
+
+ def __repr__(self):
+ return "<OONITest %s %s %s>" % (self.local_options,
+ self.global_options,
+ self.assets)
+
+ def end(self):
+ """
+ State that the current test should finish.
+ """
+ self.ended = True
+
+ def finished(self, return_value):
+ """
+ The Test has finished running, we must now calculate the test runtime
+ and add all time data to the report.
+ """
+ #self.ooninet.report(result)
+ self.end_time = date.now()
+ result = self.result
+ result['start_time'] = str(self.start_time)
+ result['end_time'] = str(self.end_time)
+ result['run_time'] = str(self.end_time - self.start_time)
+ result['return_value'] = return_value
+ log.msg("FINISHED %s" % result)
+ self.report(result)
+ return result
+
+ def _do_experiment(self, args):
+ """
+ A wrapper around the launch of experiment.
+ If we are running a blocking test experiment will be run in a thread if
+ not we expect it to return a Deferred.
+
+ @param args: the asset line(s) that we are working on.
+
+ returns a deferred.
+ """
+ if self.blocking:
+ self.d = threads.deferToThread(self.experiment, args)
+ else:
+ self.d = self.experiment(args)
+
+ self.d.addCallback(self.control, args)
+ self.d.addCallback(self.finished)
+ self.d.addErrback(self.finished)
+ return self.d
+
+ def control(self, result, args):
+ """
+ Run the control.
+
+ @param result: what was returned by experiment.
+
+ @param args: the asset(s) lines that we are working on.
+ """
+ log.msg("Doing control")
+ return result
+
+ def experiment(self, args):
+ """
+ Run the experiment. This sample implementation returns a deferred,
+ making it a non-blocking test.
+
+ @param args: the asset(s) lines that we are working on.
+ """
+ log.msg("Doing experiment")
+ d = defer.Deferred()
+ return d
+
+ def startTest(self, args):
+ """
+ This method is invoked by the worker to start the test with one line of
+ the asset file.
+
+ @param args: the asset(s) lines that we are working on.
+ """
+ self.start_time = date.now()
+
+ if self.shortName:
+ log.msg("Starting test %s" % self.shortName)
+ else:
+ log.msg("Starting test %s" % self.__class__)
+
+ return self._do_experiment(args)
diff --git a/ooni/bridget/utils/work.py b/ooni/bridget/utils/work.py
new file mode 100644
index 0000000..c329c20
--- /dev/null
+++ b/ooni/bridget/utils/work.py
@@ -0,0 +1,147 @@
+# -*- coding: UTF-8
+"""
+ work.py
+ **********
+
+ This contains all code related to generating
+ Units of Work and processing it.
+
+ :copyright: (c) 2012 by Arturo Filastò.
+ :license: see LICENSE for more details.
+
+"""
+import itertools
+import yaml
+from datetime import datetime
+
+from zope.interface import Interface, Attribute
+
+from twisted.python import failure
+from twisted.internet import reactor, defer
+
+class Worker(object):
+ """
+ This is the core of OONI. It takes as input Work Units and
+ runs them concurrently.
+ """
+ def __init__(self, maxconcurrent=10, reactor=reactor):
+ """
+ @param maxconcurrent: how many test instances should be run
+ concurrently.
+ """
+ self.reactor = reactor
+ self.maxconcurrent = maxconcurrent
+ self._running = 0
+ self._queued = []
+
+ def _run(self, r):
+ """
+ Check if we should start another test because we are below maximum
+ concurrency.
+
+ This function is called every time a test finishes running.
+
+ @param r: the return value of a previous test.
+ """
+ if self._running > 0:
+ self._running -= 1
+
+ if self._running < self.maxconcurrent and self._queued:
+ workunit, d = self._queued.pop(0)
+ asset, test, idx = workunit
+ while test.ended and workunit:
+ try:
+ workunit, d = self._queued.pop(0)
+ asset, test, idx = workunit
+ except:
+ workunit = None
+
+ if not test.ended:
+ self._running += 1
+ actuald = test.startTest(asset).addBoth(self._run)
+
+ if isinstance(r, failure.Failure):
+ # XXX probably we should be doing something to retry test running
+ r.trap()
+
+ if self._running == 0 and not self._queued:
+ self.reactor.stop()
+
+ return r
+
+ def push(self, workunit):
+ """
+ Add a test to the test queue and run it if we are not maxed out on
+ concurrency.
+
+ @param workunit: a tuple containing the (asset, test, idx), where asset
+ is the line of the asset(s) we are working on, test
+ is an instantiated test and idx is the index we are
+ currently at.
+ """
+ if self._running < self.maxconcurrent:
+ asset, test, idx = workunit
+ if not test.ended:
+ self._running += 1
+ return test.startTest(asset).addBoth(self._run)
+
+ d = defer.Deferred()
+ self._queued.append((workunit, d))
+ return d
+
+class WorkGenerator(object):
+ """
+ Factory responsible for creating units of work.
+
+ This shall be run on the machine running OONI-cli. The returned WorkUnits
+ can either be run locally or on a remote OONI Node or Network Node.
+ """
+ size = 10
+
+ def __init__(self, test, arguments=None, start=None):
+ self.Test = test
+
+ if self.Test.assets and self.Test.assets.values()[0]:
+ self.assetGenerator = itertools.product(*self.Test.assets.values())
+ else:
+ self.assetGenerator = None
+
+ self.assetNames = self.Test.assets.keys()
+
+ self.idx = 0
+ self.end = False
+ if start:
+ self.skip(start)
+
+ def __iter__(self):
+ return self
+
+ def skip(self, start):
+ """
+ Skip the first x number of lines of the asset.
+
+ @param start: int how many items we should skip.
+ """
+ for j in xrange(0, start-1):
+ for i in xrange(0, self.size):
+ self.assetGenerator.next()
+ self.idx += 1
+
+ def next(self):
+ if self.end:
+ raise StopIteration
+
+ if not self.assetGenerator:
+ self.end = True
+ return ({}, self.Test, self.idx)
+
+ try:
+ asset = self.assetGenerator.next()
+ ret = {}
+ for i, v in enumerate(asset):
+ ret[self.assetNames[i]] = v
+ except StopIteration:
+ raise StopIteration
+
+ self.idx += 1
+ return (ret, self.Test, self.idx)
diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py
deleted file mode 100644
index 5ff7b3f..0000000
--- a/ooni/plugins/bridget.py
+++ /dev/null
@@ -1,500 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-#
-# +-----------+
-# | BRIDGET |
-# | +--------------------------------------------+
-# +--------| Use a Tor process to test making a Tor |
-# | connection to a list of bridges or relays. |
-# +--------------------------------------------+
-#
-# :authors: Isis Lovecruft, Arturo Filasto
-# :licence: see included LICENSE
-# :version: 0.1.0-alpha
-
-from __future__ import with_statement
-from functools import partial
-from random import randint
-
-import os
-import sys
-
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from twisted.internet import defer, error, reactor
-from zope.interface import implements
-
-from ooni.utils import log, date
-from ooni.utils.config import ValueChecker
-
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.plugoo.assets import Asset, MissingAssetException
-from ooni.utils.onion import TxtorconImportError
-from ooni.utils.onion import PTNoBridgesException, PTNotFoundException
-
-try:
- from ooni.utils.onion import parse_data_dir
-except:
- log.msg("Please go to /ooni/lib and do 'make txtorcon' to run this test!")
-
-class RandomPortException(Exception):
- """Raised when using a random port conflicts with configured ports."""
- def __init__(self):
- log.msg("Unable to use random and specific ports simultaneously")
- return sys.exit()
-
-class BridgetArgs(usage.Options):
- """Commandline options."""
- allowed = "Port to use for Tor's %s, must be between 1024 and 65535."
- sock_check = ValueChecker(allowed % "SocksPort").port_check
- ctrl_check = ValueChecker(allowed % "ControlPort").port_check
-
- optParameters = [
- ['bridges', 'b', None,
- 'File listing bridge IP:ORPorts to test'],
- ['relays', 'f', None,
- 'File listing relay IPs to test'],
- ['socks', 's', 9049, None, sock_check],
- ['control', 'c', 9052, None, ctrl_check],
- ['torpath', 'p', None,
- 'Path to the Tor binary to use'],
- ['datadir', 'd', None,
- 'Tor DataDirectory to use'],
- ['transport', 't', None,
- 'Tor ClientTransportPlugin'],
- ['resume', 'r', 0,
- 'Resume at this index']]
- optFlags = [['random', 'x', 'Use random ControlPort and SocksPort']]
-
- def postOptions(self):
- if not self['bridges'] and not self['relays']:
- raise MissingAssetException(
- "Bridget can't run without bridges or relays to test!")
- if self['transport']:
- ValueChecker.uid_check(
- "Can't run bridget as root with pluggable transports!")
- if not self['bridges']:
- raise PTNoBridgesException
- if self['socks'] or self['control']:
- if self['random']:
- raise RandomPortException
- if self['datadir']:
- ValueChecker.dir_check(self['datadir'])
- if self['torpath']:
- ValueChecker.file_check(self['torpath'])
-
-class BridgetAsset(Asset):
- """Class for parsing bridget Assets ignoring commented out lines."""
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
- def parse_line(self, line):
- if line.startswith('#'):
- return
- else:
- return line.replace('\n','')
-
-class BridgetTest(OONITest):
- """
- XXX fill me in
-
- :ivar config:
- An :class:`ooni.lib.txtorcon.TorConfig` instance.
- :ivar relays:
- A list of all provided relays to test.
- :ivar bridges:
- A list of all provided bridges to test.
- :ivar socks_port:
- Integer for Tor's SocksPort.
- :ivar control_port:
- Integer for Tor's ControlPort.
- :ivar transport:
- String defining the Tor's ClientTransportPlugin, for testing
- a bridge's pluggable transport functionality.
- :ivar tor_binary:
- Path to the Tor binary to use, e.g. \'/usr/sbin/tor\'
- """
- implements(IPlugin, ITest)
-
- shortName = "bridget"
- description = "Use a Tor process to test connecting to bridges or relays"
- requirements = None
- options = BridgetArgs
- blocking = False
-
- def initialize(self):
- """
- Extra initialization steps. We only want one child Tor process
- running, so we need to deal with most of the TorConfig() only once,
- before the experiment runs.
- """
- self.socks_port = 9049
- self.control_port = 9052
- self.circuit_timeout = 90
- self.tor_binary = '/usr/sbin/tor'
- self.data_directory = None
-
- def __make_asset_list__(opt, lst):
- log.msg("Loading information from %s ..." % opt)
- with open(opt) as opt_file:
- for line in opt_file.readlines():
- if line.startswith('#'):
- continue
- else:
- lst.append(line.replace('\n',''))
-
- def __count_remaining__(which):
- total, reach, unreach = map(lambda x: which[x],
- ['all', 'reachable', 'unreachable'])
- count = len(total) - reach() - unreach()
- return count
-
- ## XXX should we do report['bridges_up'].append(self.bridges['current'])
- self.bridges = {}
- self.bridges['all'], self.bridges['up'], self.bridges['down'] = \
- ([] for i in range(3))
- self.bridges['reachable'] = lambda: len(self.bridges['up'])
- self.bridges['unreachable'] = lambda: len(self.bridges['down'])
- self.bridges['remaining'] = lambda: __count_remaining__(self.bridges)
- self.bridges['current'] = None
- self.bridges['pt_type'] = None
- self.bridges['use_pt'] = False
-
- self.relays = {}
- self.relays['all'], self.relays['up'], self.relays['down'] = \
- ([] for i in range(3))
- self.relays['reachable'] = lambda: len(self.relays['up'])
- self.relays['unreachable'] = lambda: len(self.relays['down'])
- self.relays['remaining'] = lambda: __count_remaining__(self.relays)
- self.relays['current'] = None
-
- if self.local_options:
- try:
- from ooni.lib.txtorcon import TorConfig
- except ImportError:
- raise TxtorconImportError
- else:
- self.config = TorConfig()
- finally:
- options = self.local_options
-
- if options['bridges']:
- self.config.UseBridges = 1
- __make_asset_list__(options['bridges'], self.bridges['all'])
- if options['relays']:
- ## first hop must be in TorState().guards
- self.config.EntryNodes = ','.join(relay_list)
- __make_asset_list__(options['relays'], self.relays['all'])
- if options['socks']:
- self.socks_port = options['socks']
- if options['control']:
- self.control_port = options['control']
- if options['random']:
- log.msg("Using randomized ControlPort and SocksPort ...")
- self.socks_port = randint(1024, 2**16)
- self.control_port = randint(1024, 2**16)
- if options['torpath']:
- self.tor_binary = options['torpath']
- if options['datadir']:
- self.data_directory = parse_data_dir(options['datadir'])
- if options['transport']:
- ## ClientTransportPlugin transport exec pathtobinary [options]
- ## XXX we need a better way to deal with all PTs
- log.msg("Using ClientTransportPlugin %s" % options['transport'])
- self.bridges['use_pt'] = True
- [self.bridges['pt_type'], pt_exec] = \
- options['transport'].split(' ', 1)
-
- if self.bridges['pt_type'] == "obfs2":
- self.config.ClientTransportPlugin = \
- self.bridges['pt_type'] + " " + pt_exec
- else:
- raise PTNotFoundException
-
- self.config.SocksPort = self.socks_port
- self.config.ControlPort = self.control_port
- self.config.CookieAuthentication = 1
-
- def __load_assets__(self):
- """
- Load bridges and/or relays from files given in user options. Bridges
- should be given in the form IP:ORport. We don't want to load these as
- assets, because it's inefficient to start a Tor process for each one.
-
- We cannot use the Asset model, because that model calls
- self.experiment() with the current Assets, which would be one relay
- and one bridge, then it gives the defer.Deferred returned from
- self.experiment() to self.control(), which means that, for each
- (bridge, relay) pair, experiment gets called again, which instantiates
- an additional Tor process that attempts to bind to the same
- ports. Thus, additionally instantiated Tor processes return with
- RuntimeErrors, which break the final defer.chainDeferred.callback(),
- sending it into the errback chain.
- """
- assets = {}
- if self.local_options:
- if self.local_options['bridges']:
- assets.update({'bridge':
- BridgetAsset(self.local_options['bridges'])})
- if self.local_options['relays']:
- assets.update({'relay':
- BridgetAsset(self.local_options['relays'])})
- return assets
-
- def experiment(self, args):
- """
- if bridges:
- 1. configure first bridge line
- 2a. configure data_dir, if it doesn't exist
- 2b. write torrc to a tempfile in data_dir
- 3. start tor } if any of these
- 4. remove bridges which are public relays } fail, add current
- 5. SIGHUP for each bridge } bridge to unreach-
- } able bridges.
- if relays:
- 1a. configure the data_dir, if it doesn't exist
- 1b. write torrc to a tempfile in data_dir
- 2. start tor
- 3. remove any of our relays which are already part of current
- circuits
- 4a. attach CustomCircuit() to self.state
- 4b. RELAY_EXTEND for each relay } if this fails, add
- } current relay to list
- } of unreachable relays
- 5.
- if bridges and relays:
- 1. configure first bridge line
- 2a. configure data_dir if it doesn't exist
- 2b. write torrc to a tempfile in data_dir
- 3. start tor
- 4. remove bridges which are public relays
- 5. remove any of our relays which are already part of current
- circuits
- 6a. attach CustomCircuit() to self.state
- 6b. for each bridge, build three circuits, with three
- relays each
- 6c. RELAY_EXTEND for each relay } if this fails, add
- } current relay to list
- } of unreachable relays
-
- :param args:
- The :class:`BridgetAsset` line currently being used. Except that it
- in Bridget it doesn't, so it should be ignored and avoided.
- """
- try:
- from ooni.utils import process
- from ooni.utils.onion import remove_public_relays, start_tor
- from ooni.utils.onion import start_tor_filter_nodes
- from ooni.utils.onion import setup_fail, setup_done
- from ooni.utils.onion import CustomCircuit
- from ooni.utils.timer import deferred_timeout, TimeoutError
- from ooni.lib.txtorcon import TorConfig, TorState
- except ImportError:
- raise TxtorconImportError
- except TxtorconImportError, tie:
- log.err(tie)
- sys.exit()
-
- def reconfigure_done(state, bridges):
- """
- Append :ivar:`bridges['current']` to the list
- :ivar:`bridges['up'].
- """
- log.msg("Reconfiguring with 'Bridge %s' successful"
- % bridges['current'])
- bridges['up'].append(bridges['current'])
- return state
-
- def reconfigure_fail(state, bridges):
- """
- Append :ivar:`bridges['current']` to the list
- :ivar:`bridges['down'].
- """
- log.msg("Reconfiguring TorConfig with parameters %s failed"
- % state)
- bridges['down'].append(bridges['current'])
- return state
-
- @defer.inlineCallbacks
- def reconfigure_bridge(state, bridges):
- """
- Rewrite the Bridge line in our torrc. If use of pluggable
- transports was specified, rewrite the line as:
- Bridge <transport_type> <IP>:<ORPort>
- Otherwise, rewrite in the standard form:
- Bridge <IP>:<ORPort>
-
- :param state:
- A fully bootstrapped instance of
- :class:`ooni.lib.txtorcon.TorState`.
- :param bridges:
- A dictionary of bridges containing the following keys:
-
- bridges['remaining'] :: A function returning and int for the
- number of remaining bridges to test.
- bridges['current'] :: A string containing the <IP>:<ORPort>
- of the current bridge.
- bridges['use_pt'] :: A boolean, True if we're testing
- bridges with a pluggable transport;
- False otherwise.
- bridges['pt_type'] :: If :ivar:`bridges['use_pt'] is True,
- this is a string containing the type
- of pluggable transport to test.
- :return:
- :param:`state`
- """
- log.msg("Current Bridge: %s" % bridges['current'])
- log.msg("We now have %d bridges remaining to test..."
- % bridges['remaining']())
- try:
- if bridges['use_pt'] is False:
- controller_response = yield state.protocol.set_conf(
- 'Bridge', bridges['current'])
- elif bridges['use_pt'] and bridges['pt_type'] is not None:
- controller_reponse = yield state.protocol.set_conf(
- 'Bridge', bridges['pt_type'] +' '+ bridges['current'])
- else:
- raise PTNotFoundException
-
- if controller_response == 'OK':
- finish = yield reconfigure_done(state, bridges)
- else:
- log.err("SETCONF for %s responded with error:\n %s"
- % (bridges['current'], controller_response))
- finish = yield reconfigure_fail(state, bridges)
-
- defer.returnValue(finish)
-
- except Exception, e:
- log.err("Reconfiguring torrc with Bridge line %s failed:\n%s"
- % (bridges['current'], e))
- defer.returnValue(None)
-
- def attacher_extend_circuit(attacher, deferred, router):
- ## XXX todo write me
- ## state.attacher.extend_circuit
- raise NotImplemented
- #attacher.extend_circuit
-
- def state_attach(state, path):
- log.msg("Setting up custom circuit builder...")
- attacher = CustomCircuit(state)
- state.set_attacher(attacher, reactor)
- state.add_circuit_listener(attacher)
- return state
-
- ## OLD
- #for circ in state.circuits.values():
- # for relay in circ.path:
- # try:
- # relay_list.remove(relay)
- # except KeyError:
- # continue
- ## XXX how do we attach to circuits with bridges?
- d = defer.Deferred()
- attacher.request_circuit_build(d)
- return d
-
- def state_attach_fail(state):
- log.err("Attaching custom circuit builder failed: %s" % state)
-
- log.msg("Bridget: initiating test ... ") ## Start the experiment
-
- ## if we've at least one bridge, and our config has no 'Bridge' line
- if self.bridges['remaining']() >= 1 \
- and not 'Bridge' in self.config.config:
-
- ## configure our first bridge line
- self.bridges['current'] = self.bridges['all'][0]
- self.config.Bridge = self.bridges['current']
- ## avoid starting several
- self.config.save() ## processes
- assert self.config.config.has_key('Bridge'), "No Bridge Line"
-
- ## start tor and remove bridges which are public relays
- from ooni.utils.onion import start_tor_filter_nodes
- state = start_tor_filter_nodes(reactor, self.config,
- self.control_port, self.tor_binary,
- self.data_directory, self.bridges)
- #controller = defer.Deferred()
- #controller.addCallback(singleton_semaphore, tor)
- #controller.addErrback(setup_fail)
- #bootstrap = defer.gatherResults([controller, filter_bridges],
- # consumeErrors=True)
-
- if state is not None:
- log.debug("state:\n%s" % state)
- log.debug("Current callbacks on TorState():\n%s"
- % state.callbacks)
-
- ## if we've got more bridges
- if self.bridges['remaining']() >= 2:
- #all = []
- for bridge in self.bridges['all'][1:]:
- self.bridges['current'] = bridge
- #new = defer.Deferred()
- #new.addCallback(reconfigure_bridge, state, self.bridges)
- #all.append(new)
- #check_remaining = defer.DeferredList(all, consumeErrors=True)
- #state.chainDeferred(check_remaining)
- state.addCallback(reconfigure_bridge, self.bridges)
-
- if self.relays['remaining']() > 0:
- while self.relays['remaining']() >= 3:
- #path = list(self.relays.pop() for i in range(3))
- #log.msg("Trying path %s" % '->'.join(map(lambda node:
- # node, path)))
- self.relays['current'] = self.relays['all'].pop()
- for circ in state.circuits.values():
- for node in circ.path:
- if node == self.relays['current']:
- self.relays['up'].append(self.relays['current'])
- if len(circ.path) < 3:
- try:
- ext = attacher_extend_circuit(state.attacher, circ,
- self.relays['current'])
- ext.addCallback(attacher_extend_circuit_done,
- state.attacher, circ,
- self.relays['current'])
- except Exception, e:
- log.err("Extend circuit failed: %s" % e)
- else:
- continue
-
- #state.callback(all)
- #self.reactor.run()
- return state
-
- def startTest(self, args):
- """
- Local override of :meth:`OONITest.startTest` to bypass calling
- self.control.
-
- :param args:
- The current line of :class:`Asset`, not used but kept for
- compatibility reasons.
- :return:
- A fired deferred which callbacks :meth:`experiment` and
- :meth:`OONITest.finished`.
- """
- self.start_time = date.now()
- self.d = self.experiment(args)
- self.d.addErrback(log.err)
- self.d.addCallbacks(self.finished, log.err)
- return self.d
-
-## So that getPlugins() can register the Test:
-#bridget = BridgetTest(None, None, None)
-
-
-## ISIS' NOTES
-## -----------
-## TODO:
-## x cleanup documentation
-## x add DataDirectory option
-## x check if bridges are public relays
-## o take bridge_desc file as input, also be able to give same
-## format as output
-## x Add asynchronous timeout for deferred, so that we don't wait
-## o Add assychronous timout for deferred, so that we don't wait
-## forever for bridges that don't work.
More information about the tor-commits
mailing list