[tor-commits] [ooni-probe/master] * Refactor all the bridget tests into a better dir structure

commit 04a61965d73cb96745e898c5bfd4cf097a090803 Author: Isis Lovecruft isis@torproject.org Date: Tue Oct 23 10:44:46 2012 +0000

* Refactor all the bridget tests into a better dir structure --- bin/canary | 27 ++ ooni/bridget/__init__.py | 14 + ooni/bridget/custodiet.py | 421 ++++++++++++++++++++++++ ooni/bridget/tests/__init__.py | 14 + ooni/bridget/tests/bridget.py | 499 ++++++++++++++++++++++++++++ ooni/bridget/utils/__init__.py | 1 + ooni/bridget/utils/inputs.py | 174 ++++++++++ ooni/bridget/utils/interface.py | 54 +++ ooni/bridget/utils/log.py | 98 ++++++ ooni/bridget/utils/nodes.py | 176 ++++++++++ ooni/bridget/utils/onion.py | 686 +++++++++++++++++++++++++++++++++++++++ ooni/bridget/utils/reports.py | 144 ++++++++ ooni/bridget/utils/tests.py | 141 ++++++++ ooni/bridget/utils/work.py | 147 +++++++++ ooni/plugins/bridget.py | 500 ---------------------------- 15 files changed, 2596 insertions(+), 500 deletions(-)

diff --git a/bin/canary b/bin/canary new file mode 100755 index 0000000..1473ae4 --- /dev/null +++ b/bin/canary @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +############################################################################### +# +# canary +# ----------------- +# Test Tor bridge reachability. +# +# :authors: Isis Lovecruft +# :copyright: 2012 Isis Lovecruft, The Tor Project +# :licence: see included LICENSE file +# :version: 0.2.0-beta +############################################################################### + +import os, sys +import copy_reg + +# Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain. +sys.path[:] = map(os.path.abspath, sys.path) +sys.path.insert(0, os.path.abspath(os.getcwd())) + +# This is a hack to overcome a bug in python +from ooni.utils.hacks import patched_reduce_ex +copy_reg._reduce_ex = patched_reduce_ex + +from ooni.bridget import spelunker +spelunker.descend() diff --git a/ooni/bridget/__init__.py b/ooni/bridget/__init__.py new file mode 100644 index 0000000..4648d77 --- /dev/null +++ b/ooni/bridget/__init__.py @@ -0,0 +1,14 @@ +#-*- coding: utf-8 -*- + +#import os, sys +#import copy_reg + +## Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain. +#sys.path[:] = map(os.path.abspath, sys.path) +#sys.path.insert(0, os.path.abspath(os.getcwd())) + +## This is a hack to overcome a bug in python +#from ooni.utils.hacks import patched_reduce_ex +#copy_reg._reduce_ex = patched_reduce_ex + +__all__ = ['custodiet'] diff --git a/ooni/bridget/custodiet.py b/ooni/bridget/custodiet.py new file mode 100755 index 0000000..8cbcfce --- /dev/null +++ b/ooni/bridget/custodiet.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 +# +# custodiet +# ********* +# +# "...quis custodiet ipsos custodes?" +# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.) +# +# "'Hand me the Custodian,' Goodchild demands, inserting the waiflike +# robot into Bambara's opened navel. 'Providing conscience for those who +# have none.' Goodchild and the other Breen government agents disappear +# into the surrounding desert in a vehicle, kicking up cloud of white dust. +# Bambara awakens, and, patting the dust from his clothing, turns to +# greet a one-armed child. 'Hi, my name's Bambara; I'm a +# thirty-six-year-old Virgo and a former killer, who's hobbies include +# performing recreational autopsies, defecating, and drinking rum. I've +# recently been given a conscience, and would very much like to help you.' +# Cut to Bambara and the child, now with one of Bambara's arms, leaving +# a surgical clinic." +# - AeonFlux, "The Purge" (sometime in the late 90s) +# +# :copyright: (c) 2012 Isis Lovecruft +# :license: see LICENSE for more details. +# :version: 0.1.0-beta +# + +# ooniprobe.py imports +import sys +from signal import SIGTERM, signal +from pprint import pprint + +from twisted.python import usage +from twisted.internet import reactor +from twisted.plugin import getPlugins + +from zope.interface.verify import verifyObject +from zope.interface.exceptions import BrokenImplementation +from zope.interface.exceptions import BrokenMethodImplementation + +from ooni.bridget.tests import bridget +from ooni.bridget.utils import log, tests, work, reports +from ooni.bridget.utils.interface import ITest +from ooni.utils.logo import getlogo + +# runner.py imports +import os +import types +import time +import inspect +import yaml + +from twisted.internet import defer, reactor +from twisted.python import reflect, failure, usage +from twisted.python import log as tlog + +from twisted.trial import unittest +from twisted.trial.runner import TrialRunner, TestLoader +from twisted.trial.runner import isPackage, isTestCase, ErrorHolder +from twisted.trial.runner import filenameToModule, _importFromFile + +from ooni import nettest +from ooni.inputunit import InputUnitFactory +from ooni.nettest import InputTestSuite +from ooni.plugoo import tests as oonitests +from ooni.reporter import ReporterFactory +from ooni.utils import log, geodata, date +from ooni.utils.legacy import LegacyOONITest +from ooni.utils.legacy import start_legacy_test, adapt_legacy_test + + +__version__ = "0.1.0-beta" + + +#def retrieve_plugoo(): +# """ +# Get all the plugins that implement the ITest interface and get the data +# associated to them into a dict. +# """ +# interface = ITest +# d = {} +# error = False +# for p in getPlugins(interface, plugins): +# try: +# verifyObject(interface, p) +# d[p.shortName] = p +# except BrokenImplementation, bi: +# print "Plugin Broken" +# print bi +# error = True +# if error != False: +# print "Plugin Loaded!" +# return d +# +#plugoo = retrieve_plugoo() + +""" + +ai to watch over which tests to run - custodiet + + * runTest() or getPrefixMethodNames() to run the tests in order for each + test (esp. the tcp and icmp parts) to be oonicompat we should use the + test_icmp_ping API framework for those. + + * should handle calling + +tests to run: + echo + syn + fin + conn + tls + tor +need fakebridge - canary + +""" + +def runTest(test, options, global_options, reactor=reactor): + """ + Run an OONI probe test by name. + + @param test: a string specifying the test name as specified inside of + shortName. + + @param options: the local options to be passed to the test. + + @param global_options: the global options for OONI + """ + parallelism = int(global_options['parallelism']) + worker = work.Worker(parallelism, reactor=reactor) + test_class = plugoo[test].__class__ + report = reports.Report(test, global_options['output']) + + log_to_stdout = True + if global_options['quiet']: + log_to_stdout = False + + log.start(log_to_stdout, + global_options['log'], + global_options['verbosity']) + + resume = 0 + if not options: + options = {} + if 'resume' in options: + resume = options['resume'] + + test = test_class(options, global_options, report, reactor=reactor) + if test.tool: + test.runTool() + return True + + if test.ended: + print "Ending test" + return None + + wgen = work.WorkGenerator(test, + dict(options), + start=resume) + for x in wgen: + worker.push(x) + +class MainOptions(usage.Options): + tests = [bridget, ] + subCommands = [] + for test in tests: + print test + testopt = getattr(test, 'options') + subCommands.append([test, None, testopt, "Run the %s test" % test]) + + optFlags = [ + ['quiet', 'q', "Don't log to stdout"] + ] + + optParameters = [ + ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], + #['target-node', 't', 'localhost:31415', 'Select target node'], + ['output', 'o', 'bridge.log', "Specify output report file"], + ['reportfile', 'o', 'bridge.log', "Specify output log file"], + ['verbosity', 'v', 1, "Specify the logging level"], + ] + + def opt_version(self): + """ + Display OONI version and exit. + """ + print "OONI version:", __version__ + sys.exit(0) + + def __str__(self): + """ + Hack to get the sweet ascii art into the help output and replace the + strings "Commands" with "Tests". + """ + return getlogo() + '\n' + self.getSynopsis() + '\n' + \ + self.getUsage(width=None).replace("Commands:", "Tests:") + + + +def isTestCase(thing): + try: + return issubclass(thing, unittest.TestCase) + except TypeError: + return False + +def isLegacyTest(obj): + """ + Returns True if the test in question is written using the OONITest legacy + class. + We do this for backward compatibility of the OONIProbe API. + """ + try: + if issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest: + return True + else: + return False + except TypeError: + return False + +def processTest(obj, config): + """ + Process the parameters and :class:`twisted.python.usage.Options` of a + :class:`ooni.nettest.Nettest`. + + :param obj: + An uninstantiated old test, which should be a subclass of + :class:`ooni.plugoo.tests.OONITest`. + :param config: + A configured and instantiated :class:`twisted.python.usage.Options` + class. + """ + + inputFile = obj.inputFile + + if obj.optParameters or inputFile: + if not obj.optParameters: + obj.optParameters = [] + + if inputFile: + obj.optParameters.append(inputFile) + + class Options(usage.Options): + optParameters = obj.optParameters + + options = Options() + options.parseOptions(config['subArgs']) + obj.localOptions = options + + if inputFile: + obj.inputFile = options[inputFile[0]] + try: + tmp_obj = obj() + tmp_obj.getOptions() + except usage.UsageError: + options.opt_help() + + return obj + +def findTestClassesFromConfig(config): + """ + Takes as input the command line config parameters and returns the test + case classes. + If it detects that a certain test class is using the old OONIProbe format, + then it will adapt it to the new testing system. + + :param config: + A configured and instantiated :class:`twisted.python.usage.Options` + class. + :return: + A list of class objects found in a file or module given on the + commandline. + """ + + filename = config['test'] + classes = [] + + module = filenameToModule(filename) + for name, val in inspect.getmembers(module): + if isTestCase(val): + classes.append(processTest(val, config)) + elif isLegacyTest(val): + classes.append(adapt_legacy_test(val, config)) + return classes + +def makeTestCases(klass, tests, methodPrefix): + """ + Takes a class some tests and returns the test cases. methodPrefix is how + the test case functions should be prefixed with. + """ + + cases = [] + for test in tests: + cases.append(klass(methodPrefix+test)) + return cases + +def loadTestsAndOptions(classes, config): + """ + Takes a list of classes and returns their testcases and options. + Legacy tests will be adapted. + """ + + methodPrefix = 'test' + suiteFactory = InputTestSuite + options = [] + testCases = [] + names = [] + + _old_klass_type = LegacyOONITest + + for klass in classes: + if isinstance(klass, _old_klass_type): + try: + cases = start_legacy_test(klass) + #cases.callback() + if cases: + print cases + return [], [] + testCases.append(cases) + except Exception, e: + log.err(e) + else: + try: + opts = klass.local_options + options.append(opts) + except AttributeError, ae: + options.append([]) + log.err(ae) + elif not isinstance(klass, _old_klass_type): + tests = reflect.prefixedMethodNames(klass, methodPrefix) + if tests: + cases = makeTestCases(klass, tests, methodPrefix) + testCases.append(cases) + try: + k = klass() + opts = k.getOptions() + options.append(opts) + except AttributeError, ae: + options.append([]) + log.err(ae) + else: + try: + raise RuntimeError, "Class is some strange type!" + except RuntimeError, re: + log.err(re) + + return testCases, options + +class ORunner(object): + """ + This is a specialized runner used by the ooniprobe command line tool. + I am responsible for reading the inputs from the test files and splitting + them in input units. I also create all the report instances required to run + the tests. + """ + def __init__(self, cases, options=None, config=None, *arg, **kw): + self.baseSuite = InputTestSuite + self.cases = cases + self.options = options + + try: + assert len(options) != 0, "Length of options is zero!" + except AssertionError, ae: + self.inputs = [] + log.err(ae) + else: + try: + first = options.pop(0) + except: + first = {} + if 'inputs' in first: + self.inputs = options['inputs'] + else: + log.msg("Could not find inputs!") + log.msg("options[0] = %s" % first) + self.inputs = [None] + + try: + reportFile = open(config['reportfile'], 'a+') + except: + filename = 'report_'+date.timestamp()+'.yaml' + reportFile = open(filename, 'a+') + self.reporterFactory = ReporterFactory(reportFile, + testSuite=self.baseSuite(self.cases)) + + def runWithInputUnit(self, inputUnit): + idx = 0 + result = self.reporterFactory.create() + + for inputs in inputUnit: + result.reporterFactory = self.reporterFactory + + suite = self.baseSuite(self.cases) + suite.input = inputs + suite(result, idx) + + # XXX refactor all of this index bullshit to avoid having to pass + # this index around. Probably what I want to do is go and make + # changes to report to support the concept of having multiple runs + # of the same test. + # We currently need to do this addition in order to get the number + # of times the test cases that have run inside of the test suite. + idx += (suite._idx - idx) + + result.done() + + def run(self): + self.reporterFactory.options = self.options + for inputUnit in InputUnitFactory(self.inputs): + self.runWithInputUnit(inputUnit) + +if __name__ == "__main__": + config = Options() + config.parseOptions() + + if not config.subCommand: + config.opt_help() + signal(SIGTERM) + #sys.exit(1) + + runTest(config.subCommand, config.subOptions, config) + reactor.run() diff --git a/ooni/bridget/tests/__init__.py b/ooni/bridget/tests/__init__.py new file mode 100644 index 0000000..9ecc88d --- /dev/null +++ b/ooni/bridget/tests/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: UTF-8 +# +# bridget/tests/__init__.py +# ************************* +# +# "...quis custodiet ipsos custodes?" +# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.) +# +# :copyright: (c) 2012 Isis Lovecruft +# :license: see LICENSE for more details. +# :version: 0.1.0-beta +# + +all = ['bridget'] diff --git a/ooni/bridget/tests/bridget.py b/ooni/bridget/tests/bridget.py new file mode 100644 index 0000000..a334747 --- /dev/null +++ b/ooni/bridget/tests/bridget.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# +-----------+ +# | BRIDGET | +# | +--------------------------------------------+ +# +--------| Use a Tor process to test making a Tor | +# | connection to a list of bridges or relays. | +# +--------------------------------------------+ +# +# :authors: Isis Lovecruft, Arturo Filasto +# :licence: see included LICENSE +# :version: 0.1.0-alpha + +from __future__ import with_statement +from functools import partial +from random import randint + +import os +import sys + +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor +from zope.interface import implements + +from ooni.utils import log, date +from ooni.utils.config import ValueChecker + +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset, MissingAssetException +from ooni.utils.onion import TxtorconImportError +from ooni.utils.onion import PTNoBridgesException, PTNotFoundException + +try: + from ooni.utils.onion import parse_data_dir +except: + log.msg("Please go to /ooni/lib and do 'make txtorcon' to run this test!") + +class RandomPortException(Exception): + """Raised when using a random port conflicts with configured ports.""" + def __init__(self): + log.msg("Unable to use random and specific ports simultaneously") + return sys.exit() + +class BridgetArgs(usage.Options): + """Commandline options.""" + allowed = "Port to use for Tor's %s, must be between 1024 and 65535." + sock_check = ValueChecker(allowed % "SocksPort").port_check + ctrl_check = ValueChecker(allowed % "ControlPort").port_check + + optParameters = [ + ['bridges', 'b', None, + 'File listing bridge IP:ORPorts to test'], + ['relays', 'f', None, + 'File listing relay IPs to test'], + ['socks', 's', 9049, None, sock_check], + ['control', 'c', 9052, None, ctrl_check], + ['torpath', 'p', None, + 'Path to the Tor binary to use'], + ['datadir', 'd', None, + 'Tor DataDirectory to use'], + ['transport', 't', None, + 'Tor ClientTransportPlugin'], + ['resume', 'r', 0, + 'Resume at this index']] + optFlags = [['random', 'x', 'Use random ControlPort and SocksPort']] + + def postOptions(self): + if not self['bridges'] and not self['relays']: + raise MissingAssetException( + "Bridget can't run without bridges or relays to test!") + if self['transport']: + ValueChecker.uid_check( + "Can't run bridget as root with pluggable transports!") + if not self['bridges']: + raise PTNoBridgesException + if self['socks'] or self['control']: + if self['random']: + raise RandomPortException + if self['datadir']: + ValueChecker.dir_check(self['datadir']) + if self['torpath']: + ValueChecker.file_check(self['torpath']) + +class BridgetAsset(Asset): + """Class for parsing bridget Assets ignoring commented out lines.""" + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + if line.startswith('#'): + return + else: + return line.replace('\n','') + +class BridgetTest(OONITest): + """ + XXX fill me in + + :ivar config: + An :class:`ooni.lib.txtorcon.TorConfig` instance. + :ivar relays: + A list of all provided relays to test. + :ivar bridges: + A list of all provided bridges to test. + :ivar socks_port: + Integer for Tor's SocksPort. + :ivar control_port: + Integer for Tor's ControlPort. + :ivar transport: + String defining the Tor's ClientTransportPlugin, for testing + a bridge's pluggable transport functionality. + :ivar tor_binary: + Path to the Tor binary to use, e.g. '/usr/sbin/tor' + """ + implements(IPlugin, ITest) + + shortName = "bridget" + description = "Use a Tor process to test connecting to bridges or relays" + requirements = None + options = BridgetArgs + blocking = False + + def initialize(self): + """ + Extra initialization steps. We only want one child Tor process + running, so we need to deal with most of the TorConfig() only once, + before the experiment runs. + """ + self.socks_port = 9049 + self.control_port = 9052 + self.circuit_timeout = 90 + self.tor_binary = '/usr/sbin/tor' + self.data_directory = None + + def __make_asset_list__(opt, lst): + log.msg("Loading information from %s ..." % opt) + with open(opt) as opt_file: + for line in opt_file.readlines(): + if line.startswith('#'): + continue + else: + lst.append(line.replace('\n','')) + + def __count_remaining__(which): + total, reach, unreach = map(lambda x: which[x], + ['all', 'reachable', 'unreachable']) + count = len(total) - reach() - unreach() + return count + + ## XXX should we do report['bridges_up'].append(self.bridges['current']) + self.bridges = {} + self.bridges['all'], self.bridges['up'], self.bridges['down'] = \ + ([] for i in range(3)) + self.bridges['reachable'] = lambda: len(self.bridges['up']) + self.bridges['unreachable'] = lambda: len(self.bridges['down']) + self.bridges['remaining'] = lambda: __count_remaining__(self.bridges) + self.bridges['current'] = None + self.bridges['pt_type'] = None + self.bridges['use_pt'] = False + + self.relays = {} + self.relays['all'], self.relays['up'], self.relays['down'] = \ + ([] for i in range(3)) + self.relays['reachable'] = lambda: len(self.relays['up']) + self.relays['unreachable'] = lambda: len(self.relays['down']) + self.relays['remaining'] = lambda: __count_remaining__(self.relays) + self.relays['current'] = None + + if self.local_options: + try: + from ooni.lib.txtorcon import TorConfig + except ImportError: + raise TxtorconImportError + else: + self.config = TorConfig() + finally: + options = self.local_options + + if options['bridges']: + self.config.UseBridges = 1 + __make_asset_list__(options['bridges'], self.bridges['all']) + if options['relays']: + ## first hop must be in TorState().guards + self.config.EntryNodes = ','.join(relay_list) + __make_asset_list__(options['relays'], self.relays['all']) + if options['socks']: + self.socks_port = options['socks'] + if options['control']: + self.control_port = options['control'] + if options['random']: + log.msg("Using randomized ControlPort and SocksPort ...") + self.socks_port = randint(1024, 2**16) + self.control_port = randint(1024, 2**16) + if options['torpath']: + self.tor_binary = options['torpath'] + if options['datadir']: + self.data_directory = parse_data_dir(options['datadir']) + if options['transport']: + ## ClientTransportPlugin transport exec pathtobinary [options] + ## XXX we need a better way to deal with all PTs + log.msg("Using ClientTransportPlugin %s" % options['transport']) + self.bridges['use_pt'] = True + [self.bridges['pt_type'], pt_exec] = \ + options['transport'].split(' ', 1) + + if self.bridges['pt_type'] == "obfs2": + self.config.ClientTransportPlugin = \ + self.bridges['pt_type'] + " " + pt_exec + else: + raise PTNotFoundException + + self.config.SocksPort = self.socks_port + self.config.ControlPort = self.control_port + self.config.CookieAuthentication = 1 + + def __load_assets__(self): + """ + Load bridges and/or relays from files given in user options. Bridges + should be given in the form IP:ORport. We don't want to load these as + assets, because it's inefficient to start a Tor process for each one. + + We cannot use the Asset model, because that model calls + self.experiment() with the current Assets, which would be one relay + and one bridge, then it gives the defer.Deferred returned from + self.experiment() to self.control(), which means that, for each + (bridge, relay) pair, experiment gets called again, which instantiates + an additional Tor process that attempts to bind to the same + ports. Thus, additionally instantiated Tor processes return with + RuntimeErrors, which break the final defer.chainDeferred.callback(), + sending it into the errback chain. + """ + assets = {} + if self.local_options: + if self.local_options['bridges']: + assets.update({'bridge': + BridgetAsset(self.local_options['bridges'])}) + if self.local_options['relays']: + assets.update({'relay': + BridgetAsset(self.local_options['relays'])}) + return assets + + def experiment(self, args): + """ + if bridges: + 1. configure first bridge line + 2a. configure data_dir, if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor } if any of these + 4. remove bridges which are public relays } fail, add current + 5. SIGHUP for each bridge } bridge to unreach- + } able bridges. + if relays: + 1a. configure the data_dir, if it doesn't exist + 1b. write torrc to a tempfile in data_dir + 2. start tor + 3. remove any of our relays which are already part of current + circuits + 4a. attach CustomCircuit() to self.state + 4b. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays + 5. + if bridges and relays: + 1. configure first bridge line + 2a. configure data_dir if it doesn't exist + 2b. write torrc to a tempfile in data_dir + 3. start tor + 4. remove bridges which are public relays + 5. remove any of our relays which are already part of current + circuits + 6a. attach CustomCircuit() to self.state + 6b. for each bridge, build three circuits, with three + relays each + 6c. RELAY_EXTEND for each relay } if this fails, add + } current relay to list + } of unreachable relays + + :param args: + The :class:`BridgetAsset` line currently being used. Except that it + in Bridget it doesn't, so it should be ignored and avoided. + """ + try: + from ooni.utils import process + from ooni.utils.onion import remove_public_relays, start_tor + from ooni.utils.onion import start_tor_filter_nodes + from ooni.utils.onion import setup_fail, setup_done + from ooni.utils.onion import CustomCircuit + from ooni.utils.timer import deferred_timeout, TimeoutError + from ooni.lib.txtorcon import TorConfig, TorState + except ImportError: + raise TxtorconImportError + except TxtorconImportError, tie: + log.err(tie) + sys.exit() + + def reconfigure_done(state, bridges): + """ + Append :ivar:`bridges['current']` to the list + :ivar:`bridges['up']. + """ + log.msg("Reconfiguring with 'Bridge %s' successful" + % bridges['current']) + bridges['up'].append(bridges['current']) + return state + + def reconfigure_fail(state, bridges): + """ + Append :ivar:`bridges['current']` to the list + :ivar:`bridges['down']. + """ + log.msg("Reconfiguring TorConfig with parameters %s failed" + % state) + bridges['down'].append(bridges['current']) + return state + + @defer.inlineCallbacks + def reconfigure_bridge(state, bridges): + """ + Rewrite the Bridge line in our torrc. If use of pluggable + transports was specified, rewrite the line as: + Bridge <transport_type> <IP>:<ORPort> + Otherwise, rewrite in the standard form: + Bridge <IP>:<ORPort> + + :param state: + A fully bootstrapped instance of + :class:`ooni.lib.txtorcon.TorState`. + :param bridges: + A dictionary of bridges containing the following keys: + + bridges['remaining'] :: A function returning and int for the + number of remaining bridges to test. + bridges['current'] :: A string containing the <IP>:<ORPort> + of the current bridge. + bridges['use_pt'] :: A boolean, True if we're testing + bridges with a pluggable transport; + False otherwise. + bridges['pt_type'] :: If :ivar:`bridges['use_pt'] is True, + this is a string containing the type + of pluggable transport to test. + :return: + :param:`state` + """ + log.msg("Current Bridge: %s" % bridges['current']) + log.msg("We now have %d bridges remaining to test..." + % bridges['remaining']()) + try: + if bridges['use_pt'] is False: + controller_response = yield state.protocol.set_conf( + 'Bridge', bridges['current']) + elif bridges['use_pt'] and bridges['pt_type'] is not None: + controller_reponse = yield state.protocol.set_conf( + 'Bridge', bridges['pt_type'] +' '+ bridges['current']) + else: + raise PTNotFoundException + + if controller_response == 'OK': + finish = yield reconfigure_done(state, bridges) + else: + log.err("SETCONF for %s responded with error:\n %s" + % (bridges['current'], controller_response)) + finish = yield reconfigure_fail(state, bridges) + + defer.returnValue(finish) + + except Exception, e: + log.err("Reconfiguring torrc with Bridge line %s failed:\n%s" + % (bridges['current'], e)) + defer.returnValue(None) + + def attacher_extend_circuit(attacher, deferred, router): + ## XXX todo write me + ## state.attacher.extend_circuit + raise NotImplemented + #attacher.extend_circuit + + def state_attach(state, path): + log.msg("Setting up custom circuit builder...") + attacher = CustomCircuit(state) + state.set_attacher(attacher, reactor) + state.add_circuit_listener(attacher) + return state + + ## OLD + #for circ in state.circuits.values(): + # for relay in circ.path: + # try: + # relay_list.remove(relay) + # except KeyError: + # continue + ## XXX how do we attach to circuits with bridges? + d = defer.Deferred() + attacher.request_circuit_build(d) + return d + + def state_attach_fail(state): + log.err("Attaching custom circuit builder failed: %s" % state) + + log.msg("Bridget: initiating test ... ") ## Start the experiment + + ## if we've at least one bridge, and our config has no 'Bridge' line + if self.bridges['remaining']() >= 1 \ + and not 'Bridge' in self.config.config: + + ## configure our first bridge line + self.bridges['current'] = self.bridges['all'][0] + self.config.Bridge = self.bridges['current'] + ## avoid starting several + self.config.save() ## processes + assert self.config.config.has_key('Bridge'), "No Bridge Line" + + ## start tor and remove bridges which are public relays + from ooni.utils.onion import start_tor_filter_nodes + state = start_tor_filter_nodes(reactor, self.config, + self.control_port, self.tor_binary, + self.data_directory, self.bridges) + #controller = defer.Deferred() + #controller.addCallback(singleton_semaphore, tor) + #controller.addErrback(setup_fail) + #bootstrap = defer.gatherResults([controller, filter_bridges], + # consumeErrors=True) + + if state is not None: + log.debug("state:\n%s" % state) + log.debug("Current callbacks on TorState():\n%s" + % state.callbacks) + + ## if we've got more bridges + if self.bridges['remaining']() >= 2: + #all = [] + for bridge in self.bridges['all'][1:]: + self.bridges['current'] = bridge + #new = defer.Deferred() + #new.addCallback(reconfigure_bridge, state, self.bridges) + #all.append(new) + #check_remaining = defer.DeferredList(all, consumeErrors=True) + #state.chainDeferred(check_remaining) + state.addCallback(reconfigure_bridge, self.bridges) + + if self.relays['remaining']() > 0: + while self.relays['remaining']() >= 3: + #path = list(self.relays.pop() for i in range(3)) + #log.msg("Trying path %s" % '->'.join(map(lambda node: + # node, path))) + self.relays['current'] = self.relays['all'].pop() + for circ in state.circuits.values(): + for node in circ.path: + if node == self.relays['current']: + self.relays['up'].append(self.relays['current']) + if len(circ.path) < 3: + try: + ext = attacher_extend_circuit(state.attacher, circ, + self.relays['current']) + ext.addCallback(attacher_extend_circuit_done, + state.attacher, circ, + self.relays['current']) + except Exception, e: + log.err("Extend circuit failed: %s" % e) + else: + continue + + #state.callback(all) + #self.reactor.run() + return state + + def startTest(self, args): + """ + Local override of :meth:`OONITest.startTest` to bypass calling + self.control. + + :param args: + The current line of :class:`Asset`, not used but kept for + compatibility reasons. + :return: + A fired deferred which callbacks :meth:`experiment` and + :meth:`OONITest.finished`. + """ + self.start_time = date.now() + self.d = self.experiment(args) + self.d.addErrback(log.err) + self.d.addCallbacks(self.finished, log.err) + return self.d + +## So that getPlugins() can register the Test: +#bridget = BridgetTest(None, None, None) + +## ISIS' NOTES +## ----------- +## TODO: +## x cleanup documentation +## x add DataDirectory option +## x check if bridges are public relays +## o take bridge_desc file as input, also be able to give same +## format as output +## x Add asynchronous timeout for deferred, so that we don't wait +## o Add assychronous timout for deferred, so that we don't wait +## forever for bridges that don't work. diff --git a/ooni/bridget/utils/__init__.py b/ooni/bridget/utils/__init__.py new file mode 100644 index 0000000..92893d6 --- /dev/null +++ b/ooni/bridget/utils/__init__.py @@ -0,0 +1 @@ +all = ['inputs', 'log', 'onion', 'tests', 'interface', 'nodes', 'reports', 'work'] diff --git a/ooni/bridget/utils/inputs.py b/ooni/bridget/utils/inputs.py new file mode 100644 index 0000000..fe058cc --- /dev/null +++ b/ooni/bridget/utils/inputs.py @@ -0,0 +1,174 @@ +#-*- coding: utf-8 -*- +# +# inputs.py +# ********* +# +# "...quis custodiet ipsos custodes?" +# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.) +# +# :copyright: (c) 2012 Isis Lovecruft +# :license: see LICENSE for more details. +# :version: 0.1.0-beta +# + +#from types import FunctionType, FileType +import types + +from ooni.bridget import log +from ooni.utils import date, Storage + +class InputFile: + """ + This is a class describing a file used to store Tor bridge or relays + inputs. It is a python iterator object, allowing it to be efficiently + looped. + + This class should not be used directly, but rather its subclasses, + BridgeFile and RelayFile should be used instead. + """ + + def __init__(self, file, **kw): + """ + ## This is an InputAsset file, created because you tried to pass a + ## non-existent filename to a test. + ## + ## To use this file, place one input to be tested per line. Each + ## test takes different inputs. Lines which are commented out with + ## a '#' are not used. + """ + self.file = file + self.eof = False + self.all = Storage() + + for key, value in input_dict: + self.all[key] = value + + try: + self.handler = open(self.file, 'r') + except IOError: + with open(self.file, 'w') as explain: + for line in self.__init__.__doc__: + explain.writeline(line) + self.handler = open(self.file, 'r') + try: + assert isinstance(self.handler, file), "That's not a file!" + except AssertionError, ae: + log.err(ae) + + # def __handler__(self): + # """ + # Attempt to open InputFile.file and check that it is actually a file. + # If it's not, create it and add an explaination for how InputFile files + # should be used. + + # :return: + # A :type:`file` which has been opened in read-only mode. + # """ + # try: + # handler = open(self.file, 'r') + # except IOError, ioerror: ## not the hacker <(A)3 + # log.err(ioerror) + # explanation = ( + # with open(self.file, 'w') as explain: + # for line in explanation: + # explain.writeline(line) + # handler = open(self.file, 'r') + # try: + # assert isinstance(handler, file), "That's not a file!" + # except AssertionError, ae: + # log.err(ae) + # else: + # return handler + + def __iter__(next, StopIteration): + """ + Returns the next input from the file. + """ + #return self.next() + return self + + def len(self): + """ + Returns the number of the lines in the InputFile. + """ + with open(self.file, 'r') as input_file: + lines = input_file.readlines() + for number, line in enumerate(lines): + self.input_dict[number] = line + return number + 1 + + def next(self): + try: + return self.next_input() + except: + raise StopIteration + + def next_input(self): + """ + Return the next input. + """ + line = self.handler.readline() + if line: + parsed_line = self.parse_line(line) + if parsed_line: + return parsed_line + else: + self.fh.seek(0) + raise StopIteration + + def default_parser(self, line): + """ + xxx fill me in + """ + if not line.startswith('#'): + return line.replace('\n', '') + else: + return False + + def parse_line(self, line): + """ + Override this method if you need line by line parsing of an Asset. + + The default parsing action is to ignore lines which are commented out + with a '#', and to strip the newline character from the end of the + line. + + If the line was commented out return an empty string instead. + + If a subclass Foo incorporates another class Bar, when Bar is not + also a subclass of InputFile, and Bar.parse_line() exists, then + do not overwrite Bar's parse_line method. + """ + assert not hasattr(super(InputFile, self), 'parse_line') + + if self.parser is None: + if not line.startswith('#'): + return line.replace('\n', '') + else: + return '' + else: + try: + assert isinstance(self.parser, FunctionType),"Not a function!" + except AssertionError, ae: + log.err(ae) + else: + return self.parser(line) + +class BridgeFile(InputFile): + """ + xxx fill me in + """ + def __init__(self, **kw): + super(BridgeFile, self).__init__(**kw) + +class MissingInputException(Exception): + """ + + Raised when an :class:`InputFile` necessary for running the Test is + missing. + + """ + def __init__(self, error_message): + print error_message + import sys + return sys.exit() diff --git a/ooni/bridget/utils/interface.py b/ooni/bridget/utils/interface.py new file mode 100644 index 0000000..aa55436 --- /dev/null +++ b/ooni/bridget/utils/interface.py @@ -0,0 +1,54 @@ +from zope.interface import implements, Interface, Attribute + +class ITest(Interface): + """ + This interface represents an OONI test. It fires a deferred on completion. + """ + + shortName = Attribute("""A short user facing description for this test""") + description = Attribute("""A string containing a longer description for the test""") + + requirements = Attribute("""What is required to run this this test, for example raw socket access or UDP or TCP""") + + options = Attribute("""These are the arguments to be passed to the test for it's execution""") + + blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""") + + def control(experiment_result, args): + """ + @param experiment_result: The result returned by the experiment method. + + @param args: the keys of this dict are the names of the assets passed in + from load_assets. The value is one item of the asset. + + Must return a dict containing what should be written to the report. + Anything returned by control ends up inside of the YAMLOONI report. + """ + + def experiment(args): + """ + Perform all the operations that are necessary to running a test. + + @param args: the keys of this dict are the names of the assets passed in + from load_assets. The value is one item of the asset. + + Must return a dict containing the values to be passed to control. + """ + + def load_assets(): + """ + Load the assets that should be passed to the Test. These are the inputs + to the OONI test. + Must return a dict that has as keys the asset names and values the + asset contents. + If the test does not have any assets it should return an empty dict. + """ + + def end(): + """ + This can be called at any time to terminate the execution of all of + these test instances. + + What this means is that no more test instances with new parameters will + be created. A report will be written. + """ diff --git a/ooni/bridget/utils/log.py b/ooni/bridget/utils/log.py new file mode 100644 index 0000000..eef50d8 --- /dev/null +++ b/ooni/bridget/utils/log.py @@ -0,0 +1,98 @@ +""" +OONI logging facility. +""" +from sys import stderr, stdout + +from twisted.python import log, util +from twisted.python.failure import Failure + +def _get_log_level(level): + english = ['debug', 'info', 'warn', 'err', 'crit'] + + levels = dict(zip(range(len(english)), english)) + number = dict(zip(english, range(len(english)))) + + if not level: + return number['info'] + else: + ve = "Unknown log level: %s\n" % level + ve += "Allowed levels: %s\n" % [word for word in english] + + if type(level) is int: + if 0 <= level <= 4: + return level + elif type(level) is str: + if number.has_key(level.lower()): + return number[level] + else: + raise ValueError, ve + else: + raise ValueError, ve + +class OONITestFailure(Failure): + """ + For handling Exceptions asynchronously. + + Can be given an Exception as an argument, else will use the + most recent Exception from the current stack frame. + """ + def __init__(self, exception=None, _type=None, + _traceback=None, _capture=False): + Failure.__init__(self, exc_type=_type, + exc_tb=_traceback, captureVars=_capture) + +class OONILogObserver(log.FileLogObserver): + """ + Supports logging level verbosity. + """ + def __init__(self, logfile, verb=None): + log.FileLogObserver.__init__(self, logfile) + self.level = _get_log_level(verb) if verb is not None else 1 + assert type(self.level) is int + + def emit(self, eventDict): + if 'logLevel' in eventDict: + msgLvl = _get_log_level(eventDict['logLevel']) + assert type(msgLvl) is int + ## only log our level and higher + if self.level <= msgLvl: + text = log.textFromEventDict(eventDict) + else: + text = None + else: + text = log.textFromEventDict(eventDict) + + if text is None: + return + + timeStr = self.formatTime(eventDict['time']) + fmtDict = {'system': eventDict['system'], + 'text': text.replace('\n','\n\t')} + msgStr = log._safeFormat("[%(system)s] %(text)s\n", fmtDict) + + util.untilConcludes(self.write, timeStr + " " + msgStr) + util.untilConcludes(self.flush) + +def start(logfile=None, verbosity=None): + if log.defaultObserver: + verbosity = _get_log_level(verbosity) + + ## Always log to file, keep level at info + file = open(logfile, 'a') if logfile else stderr + OONILogObserver(file, "info").start() + + log.msg("Starting OONI...") + +def debug(message, level="debug", **kw): + print "[%s] %s" % (level, message) + ## If we want debug messages in the logfile: + #log.msg(message, logLevel=level, **kw) + +def msg(message, level="info", **kw): + log.msg(message, logLevel=level, **kw) + +def err(message, level="err", **kw): + log.err(logLevel=level, **kw) + +def fail(message, exception, level="crit", **kw): + log.failure(message, OONITestFailure(exception, **kw), logLevel=level) diff --git a/ooni/bridget/utils/nodes.py b/ooni/bridget/utils/nodes.py new file mode 100644 index 0000000..155f183 --- /dev/null +++ b/ooni/bridget/utils/nodes.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 +""" + nodes + ***** + + This contains all the code related to Nodes + both network and code execution. + + :copyright: (c) 2012 by Arturo Filastò, Isis Lovecruft + :license: see LICENSE for more details. + +""" + +import os +from binascii import hexlify + +try: + import paramiko +except: + print "Error: module paramiko is not installed." +from pprint import pprint +import sys +import socks +import xmlrpclib + +class Node(object): + def __init__(self, address, port): + self.address = address + self.port = port + +class LocalNode(object): + def __init__(self): + pass + +""" +[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5") +[]: node_socket = node.wrap_socket() +""" +class NetworkNode(Node): + def __init__(self, address, port, node_type="SOCKS5", auth_creds=None): + self.node = Node(address,port) + + # XXX support for multiple types + # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...) + self.node_type = node_type + # type-specific authentication credentials + self.auth_creds = auth_creds + + def _get_socksipy_socket(self, proxy_type, auth_creds): + import socks + s = socks.socksocket() + # auth_creds[0] -> username + # auth_creds[1] -> password + s.setproxy(proxy_type, self.node.address, self.node.port, + self.auth_creds[0], self.auth_creds[1]) + return s + + def _get_socket_wrapper(self): + if (self.node_type.startswith("SOCKS")): # SOCKS proxies + if (self.node_type != "SOCKS5"): + proxy_type = socks.PROXY_TYPE_SOCKS5 + elif (self.node_type != "SOCKS4"): + proxy_type = socks.PROXY_TYPE_SOCKS4 + else: + print "We don't know this proxy type." + sys.exit(1) + + return self._get_socksipy_socket(proxy_type) + elif (self.node_type == "HTTP"): # HTTP proxies + return self._get_socksipy_socket(PROXY_TYPE_HTTP) + else: # Unknown proxies + print "We don't know this proxy type." + sys.exit(1) + + def wrap_socket(self): + return self._get_socket_wrapper() + +class CodeExecNode(Node): + def __init__(self, address, port, node_type, auth_creds): + self.node = Node(address,port) + + # node type (SSH proxy, etc.) + self.node_type = node_type + # type-specific authentication credentials + self.auth_creds = auth_creds + + def add_unit(self): + pass + + def get_status(self): + pass + +class PlanetLab(CodeExecNode): + def __init__(self, address, auth_creds, ooni): + self.auth_creds = auth_creds + + self.config = ooni.utils.config + self.logger = ooni.logger + self.name = "PlanetLab" + + def _api_auth(self): + api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/') + auth = {} + ## should be changed to separate node.conf file + auth['Username'] = self.config.main.pl_username + auth['AuthString'] = self.config.main.pl_password + auth['AuthMethod'] = "password" + authorized = api_server.AuthCheck(auth) + + if authorized: + print 'We are authorized!' + return auth + else: + print 'Authorization failed. Please check your settings for pl_username and pl_password in the ooni-probe.conf file.' + + def _search_for_nodes(self, node_filter=None): + api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True) + node_filter = {'hostname': '*.cert.org.cn'} + return_fields = ['hostname', 'site_id'] + all_nodes = api_server.GetNodes(self.api_auth(), node_filter, boot_state_filter) + pprint(all_nodes) + return all_nodes + + def _add_nodes_to_slice(self): + api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True) + all_nodes = self.search_for_nodes() + for node in all_nodes: + api_server.AddNode(self.api_auth(), node['site_id'], all_nodes) + print 'Adding nodes %s' % node['hostname'] + + def _auth_login(slicename, machinename): + """Attempt to authenticate to the given PL node, slicename and + machinename, using any of the private keys in ~/.ssh/ """ + + agent = paramiko.Agent() + agent_keys = agent.get_keys() + if len(agent_keys) == 0: + return + + for key in agent_keys: + print 'Trying ssh-agent key %s' % hexlify(key.get_fingerprint()), + try: + paramiko.transport.auth_publickey(machinename, slicename) + print 'Public key authentication to PlanetLab node %s successful.' % machinename, + return + except paramiko.SSHException: + print 'Public key authentication to PlanetLab node %s failed.' % machinename, + + def _get_command(): + pass + + def ssh_and_run_(slicename, machinename, command): + """Attempt to make a standard OpenSSH client to PL node, and run + commands from a .conf file.""" + + ## needs a way to specify 'ssh -l <slicename> <machinename>' + ## with public key authentication. + + command = PlanetLab.get_command() + + client = paramiko.SSHClient() + client.load_system_host_keys() + client.connect(machinename) + + stdin, stdout, stderr = client.exec_command(command) + + def send_files_to_node(directory, files): + """Attempt to rsync a tree to the PL node.""" + pass + + def add_unit(): + pass + + def get_status(): + pass diff --git a/ooni/bridget/utils/onion.py b/ooni/bridget/utils/onion.py new file mode 100644 index 0000000..9d4cae7 --- /dev/null +++ b/ooni/bridget/utils/onion.py @@ -0,0 +1,686 @@ +# +# onion.py +# ---------- +# Utilities for working with Tor. +# +# This code is largely taken from txtorcon and its documentation, and as such +# any and all credit should go to Meejah. Minor adjustments have been made to +# use OONI's logging system, and to build custom circuits without actually +# attaching streams. +# +# :author: Meejah, Isis Lovecruft +# :license: see included LICENSE file +# :copyright: copyright (c) 2012 The Tor Project, Inc. +# :version: 0.1.0-alpha +# +# XXX TODO add report keys for onion methods + +import random +import sys + +from twisted.internet import defer +from zope.interface import implements + +from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +from ooni.lib.txtorcon import TorState, TorConfig +from ooni.utils import log +from ooni.utils.timer import deferred_timeout, TimeoutError + +def parse_data_dir(data_dir): + """ + Parse a string that a has been given as a DataDirectory and determine + its absolute path on the filesystem. + + :param data_dir: + A directory for Tor's DataDirectory, to be parsed. + :return: + The absolute path of :param:data_dir. + """ + from os import path, getcwd + import sys + + try: + assert isinstance(data_dir, str), \ + "Parameter type(data_dir) must be str" + except AssertionError, ae: + log.err(ae) + + if data_dir.startswith('~'): + data_dir = path.expanduser(data_dir) + elif data_dir.startswith('/'): + data_dir = path.join(getcwd(), data_dir) + elif data_dir.startswith('./'): + data_dir = path.abspath(data_dir) + else: + data_dir = path.join(getcwd(), data_dir) + + try: + assert path.isdir(data_dir), "Could not find %s" % data_dir + except AssertionError, ae: + log.err(ae) + sys.exit() + else: + return data_dir + +def write_torrc(conf, data_dir=None): + """ + Create a torrc in our data_dir. If we don't yet have a data_dir, create a + temporary one. Any temporary files or folders are added to delete_list. + + :param conf: + A :class:`ooni.lib.txtorcon.TorConfig` object, with all configuration + values saved. + :param data_dir: + The Tor DataDirectory to use. + :return: torrc, data_dir, delete_list + """ + try: + from os import write, close + from tempfile import mkstemp, mkdtemp + except ImportError, ie: + log.err(ie) + + delete_list = [] + + if data_dir is None: + data_dir = mkdtemp(prefix='bridget-tordata') + delete_list.append(data_dir) + conf.DataDirectory = data_dir + + (fd, torrc) = mkstemp(dir=data_dir) + delete_list.append(torrc) + write(fd, conf.create_torrc()) + close(fd) + + return torrc, data_dir, delete_list + +def delete_files_or_dirs(delete_list): + """ + Given a list of files or directories to delete, delete all and suppress + all errors. + + :param delete_list: + A list of files or directories to delete. + """ + try: + from os import unlink + from shutil import rmtree + except ImportError, ie: + log.err(ie) + + for temp in delete_list: + try: + unlink(temp) + except OSError: + rmtree(temp, ignore_errors=True) + +def remove_node_from_list(node, list): + for item in list: ## bridges don't match completely + if item.startswith(node): ## due to the :<port>. + try: + log.msg("Removing %s because it is a public relay" % node) + list.remove(item) + except ValueError, ve: + log.err(ve) + +def remove_public_relays(state, bridges): + """ + Remove bridges from our bridge list which are also listed as public + relays. This must be called after Tor has fully bootstrapped and we have a + :class:`ooni.lib.txtorcon.TorState` with the + :attr:`ooni.lib.txtorcon.TorState.routers` attribute assigned. + + XXX Does state.router.values() have all of the relays in the consensus, or + just the ones we know about so far? + + XXX FIXME: There is a problem in that Tor needs a Bridge line to already be + configured in order to bootstrap. However, after bootstrapping, we grab the + microdescriptors of all the relays and check if any of our bridges are + listed as public relays. Because of this, the first bridge does not get + checked for being a relay. + """ + IPs = map(lambda addr: addr.split(':',1)[0], bridges['all']) + both = set(state.routers.values()).intersection(IPs) + + if len(both) > 0: + try: + updated = map(lambda node: remove_node_from_list(node), both) + log.debug("Bridges in both: %s" % both) + log.debug("Updated = %s" % updated) + #if not updated: + # defer.returnValue(state) + #else: + # defer.returnValue(state) + return state + except Exception, e: + log.err("Removing public relays %s from bridge list failed:\n%s" + % (both, e)) + +def setup_done(proto): + log.msg("Setup Complete") + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete) + state.post_bootstrap.addErrback(setup_fail) + +def setup_fail(proto): + log.msg("Setup Failed:\n%s" % proto) + return proto + #reactor.stop() + +def state_complete(state): + """Called when we've got a TorState.""" + log.msg("We've completely booted up a Tor version %s at PID %d" + % (state.protocol.version, state.tor_pid)) + log.msg("This Tor has the following %d Circuits:" + % len(state.circuits)) + for circ in state.circuits.values(): + log.msg("%s" % circ) + return state + +def updates(_progress, _tag, _summary): + """Log updates on the Tor bootstrapping process.""" + log.msg("%d%%: %s" % (_progress, _summary)) + +def bootstrap(ctrl): + """ + Bootstrap Tor from an instance of + :class:`ooni.lib.txtorcon.TorControlProtocol`. + """ + conf = TorConfig(ctrl) + conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) + log.msg("Tor process connected, bootstrapping ...") + +def start_tor(reactor, config, control_port, tor_binary, data_dir, + report=None, progress=updates, + process_cb=None, process_eb=None): + """ + Use a txtorcon.TorConfig() instance, config, to write a torrc to a + tempfile in our DataDirectory, data_dir. If data_dir is None, a temp + directory will be created. Finally, create a TCP4ClientEndpoint at our + control_port, and connect it to our reactor and a spawned Tor + process. Compare with :meth:`txtorcon.launch_tor` for differences. + + :param reactor: + An instance of class:`twisted.internet.reactor`. + :param config: + An instance of class:`txtorcon.TorConfig` with all torrc options + already configured. ivar:`config.ControlPort`, + ivar:`config.SocksPort`, ivar:`config.CookieAuthentication`, should + already be set, as well as ivar:`config.UseBridges` and + ivar:`config.Bridge` if bridges are to be used. + ivar:`txtorcon.DataDirectory` does not need to be set. + :param control_port: + The port number to use for Tor's ControlPort. + :param tor_binary: + The full path to the Tor binary to use. + :param data_dir: + The directory to use as Tor's DataDirectory. + :param report: + The class:`ooni.plugoo.reports.Report` instance. + :param progress: + A non-blocking function to handle bootstrapping updates, which takes + three parameters: _progress, _tag, and _summary. + :param process_cb: + The function to callback to after + class:`ooni.lib.txtorcon.TorProcessProtocol` returns with the fully + bootstrapped Tor process. + :param process_eb: + The function to errback to if + class:`ooni.lib.txtorcon.TorProcessProtocol` fails. + :return: + The result of the callback of a + class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a + class:`txtorcon.TorControlProtocol` as .protocol. + """ + try: + from functools import partial + from twisted.internet.endpoints import TCP4ClientEndpoint + from ooni.lib.txtorcon import TorProtocolFactory + from ooni.lib.txtorcon import TorProcessProtocol + except ImportError, ie: + log.err(ie) + + ## TODO: add option to specify an already existing torrc, which + ## will require prior parsing to enforce necessary lines + (torrc, data_dir, to_delete) = write_torrc(config, data_dir) + + log.msg("Starting Tor ...") + log.msg("Using the following as our torrc:\n%s" % config.create_torrc()) + if report is None: + report = {'torrc': config.create_torrc()} + else: + report.update({'torrc': config.create_torrc()}) + + end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port) + connection_creator = partial(end_point.connect, TorProtocolFactory()) + process_protocol = TorProcessProtocol(connection_creator, progress) + process_protocol.to_delete = to_delete + + if process_cb is not None and process_eb is not None: + process_protocol.connected_cb.addCallbacks(process_cb, process_eb) + + reactor.addSystemEventTrigger('before', 'shutdown', + partial(delete_files_or_dirs, to_delete)) + try: + transport = reactor.spawnProcess(process_protocol, + tor_binary, + args=(tor_binary,'-f',torrc), + env={'HOME': data_dir}, + path=data_dir) + transport.closeStdin() + except RuntimeError, e: + log.err("Starting Tor failed:") + process_protocol.connected_cb.errback(e) + except NotImplementedError, e: + url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" + log.msg("Running bridget on Windows requires pywin32: %s" % url) + process_protocol.connected_cb.errback(e) + + return process_protocol.connected_cb + +@defer.inlineCallbacks +def start_tor_filter_nodes(reactor, config, control_port, tor_binary, + data_dir, bridges): + """ + Bootstrap a Tor process and return a fully-setup + :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges + to test in the list of known public relays, + :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges + which are known public relays. + + :param reactor: + The :class:`twisted.internet.reactor`. + :param config: + An instance of :class:`ooni.lib.txtorcon.TorConfig`. + :param control_port: + The port to use for Tor's ControlPort. If already configured in + the TorConfig instance, this can be given as + TorConfig.config.ControlPort. + :param tor_binary: + The full path to the Tor binary to execute. + :param data_dir: + The full path to the directory to use as Tor's DataDirectory. + :param bridges: + A dictionary which has a key 'all' which is a list of bridges to + test connecting to, e.g.: + bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001'] + :return: + A fully initialized :class:`ooni.lib.txtorcon.TorState`. + """ + setup = yield start_tor(reactor, config, control_port, + tor_binary, data_dir, + process_cb=setup_done, process_eb=setup_fail) + filter_nodes = yield remove_public_relays(setup, bridges) + defer.returnValue(filter_nodes) + +@defer.inlineCallbacks +def start_tor_with_timer(reactor, config, control_port, tor_binary, data_dir, + bridges, timeout): + """ + Start bootstrapping a Tor process wrapped with an instance of the class + decorator :func:`ooni.utils.timer.deferred_timeout` and complete callbacks + to either :func:`setup_done` or :func:`setup_fail`. Return a fully-setup + :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges to test + in the list of known public relays, + :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges which + are listed as known public relays. + + :param reactor: + The :class:`twisted.internet.reactor`. + :param config: + An instance of :class:`ooni.lib.txtorcon.TorConfig`. + :param control_port: + The port to use for Tor's ControlPort. If already configured in + the TorConfig instance, this can be given as + TorConfig.config.ControlPort. + :param tor_binary: + The full path to the Tor binary to execute. + :param data_dir: + The full path to the directory to use as Tor's DataDirectory. + :param bridges: + A dictionary which has a key 'all' which is a list of bridges to + test connecting to, e.g.: + bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001'] + :param timeout: + The number of seconds to attempt to bootstrap the Tor process before + raising a :class:`ooni.utils.timer.TimeoutError`. + :return: + If the timeout limit is not exceeded, return a fully initialized + :class:`ooni.lib.txtorcon.TorState`, else return None. + """ + error_msg = "Bootstrapping has exceeded the timeout limit..." + with_timeout = deferred_timeout(timeout, e=error_msg)(start_tor) + try: + setup = yield with_timeout(reactor, config, control_port, tor_binary, + data_dir, process_cb=setup_done, + process_eb=setup_fail) + except TimeoutError, te: + log.err(te) + defer.returnValue(None) + #except Exception, e: + # log.err(e) + # defer.returnValue(None) + else: + state = yield remove_public_relays(setup, bridges) + defer.returnValue(state) + +@defer.inlineCallbacks +def start_tor_filter_nodes_with_timer(reactor, config, control_port, + tor_binary, data_dir, bridges, timeout): + """ + Start bootstrapping a Tor process wrapped with an instance of the class + decorator :func:`ooni.utils.timer.deferred_timeout` and complete callbacks + to either :func:`setup_done` or :func:`setup_fail`. Then, filter our list + of bridges to remove known public relays by calling back to + :func:`remove_public_relays`. Return a fully-setup + :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges to test + in the list of known public relays, + :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges which + are listed as known public relays. + + :param reactor: + The :class:`twisted.internet.reactor`. + :param config: + An instance of :class:`ooni.lib.txtorcon.TorConfig`. + :param control_port: + The port to use for Tor's ControlPort. If already configured in + the TorConfig instance, this can be given as + TorConfig.config.ControlPort. + :param tor_binary: + The full path to the Tor binary to execute. + :param data_dir: + The full path to the directory to use as Tor's DataDirectory. + :param bridges: + A dictionary which has a key 'all' which is a list of bridges to + test connecting to, e.g.: + bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001'] + :param timeout: + The number of seconds to attempt to bootstrap the Tor process before + raising a :class:`ooni.utils.timer.TimeoutError`. + :return: + If the timeout limit is not exceeded, return a fully initialized + :class:`ooni.lib.txtorcon.TorState`, else return None. + """ + error_msg = "Bootstrapping has exceeded the timeout limit..." + with_timeout = deferred_timeout(timeout, e=error_msg)(start_tor_filter_nodes) + try: + state = yield with_timeout(reactor, config, control_port, + tor_binary, data_dir, bridges) + except TimeoutError, te: + log.err(te) + defer.returnValue(None) + #except Exception, e: + # log.err(e) + # defer.returnValue(None) + else: + defer.returnValue(state) + +class CustomCircuit(CircuitListenerMixin): + """ + Utility class for controlling circuit building. See + 'attach_streams_by_country.py' in the txtorcon documentation. + + :param state: + A fully bootstrapped instance of :class:`ooni.lib.txtorcon.TorState`. + :param relays: + A dictionary containing a key 'all', which is a list of relays to + test connecting to. + :ivar waiting_circuits: + The list of circuits which we are waiting to attach to. You shouldn't + need to touch this. + """ + implements(IStreamAttacher) + + def __init__(self, state, relays=None): + self.state = state + self.waiting_circuits = [] + self.relays = relays + + def waiting_on(self, circuit): + """ + Whether or not we are waiting on the given circuit before attaching to + it. + + :param circuit: + An item from :ivar:`ooni.lib.txtorcon.TorState.circuits`. + :return: + True if we are waiting on the circuit, False if not waiting. + """ + for (circid, d) in self.waiting_circuits: + if circuit.id == circid: + return True + return False + + def circuit_extend(self, circuit, router): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + if self.waiting_on(circuit): + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) + + def circuit_built(self, circuit): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + log.msg("Circuit %s built ..." % circuit.id) + log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) + for (circid, d) in self.waiting_circuits: + if circid == circuit.id: + self.waiting_circuits.remove((circid, d)) + d.callback(circuit) + + def circuit_failed(self, circuit, reason): + """ + If building a circuit has failed, try to remove it from our list of + :ivar:`waiting_circuits`, else request to build it. + + :param circuit: + An item from :ivar:`ooni.lib.txtorcon.TorState.circuits`. + :param reason: + A :class:`twisted.python.fail.Failure` instance. + :return: + None + """ + if self.waiting_on(circuit): + log.msg("Circuit %s failed for reason %s" % (circuit.id, reason)) + circid, d = None, None + for c in self.waiting_circuits: + if c[0] == circuit.id: + circid, d = c + if d is None: + raise Exception("Expected to find circuit.") + + self.waiting_circuits.remove((circid, d)) + log.msg("Trying to build a circuit for %s" % circid) + self.request_circuit_build(d) + + def check_circuit_route(self, router): + """ + Check if a relay is a hop in one of our already built circuits. + + :param router: + An item from the list + :func:`ooni.lib.txtorcon.TorState.routers.values()`. + """ + for circ in self.state.circuits.values(): + if router in circ.path: + #router.update() ## XXX can i use without args? no. + TorInfo.dump(self) + + def request_circuit_build(self, deferred, path=None): + """ + Request a custom circuit. + + :param deferred: + A :class:`twisted.internet.defer.Deferred` for this circuit. + :param path: + A list of router ids to build a circuit from. The length of this + list must be at least three. + """ + if path is None: + + pick = self.relays['all'].pop + n = self.state.entry_guards.values() + choose = random.choice + + first, middle, last = (None for i in range(3)) + + if self.relays['remaining']() >= 3: + first, middle, last = (pick() for i in range(3)) + elif self.relays['remaining']() < 3: + first = choose(n) + middle = pick() + if self.relays['remaining'] == 2: + middle, last = (pick() for i in range(2)) + elif self.relay['remaining'] == 1: + middle = pick() + last = choose(n) + else: + log.msg("Qu'est-que fuque?") + else: + middle, last = (random.choice(self.state.routers.values()) + for i in range(2)) + + path = [first, middle, last] + + else: + assert isinstance(path, list), \ + "Circuit path must be a list of relays!" + assert len(path) >= 3, \ + "Circuit path must be at least three hops!" + + log.msg("Requesting a circuit: %s" + % '->'.join(map(lambda node: node, path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + def __call__(self, circ): + """ + Return from build_circuit is a Circuit, however, + we want to wait until it is built before we can + issue an attach on it and callback to the Deferred + we issue here. + """ + log.msg("Circuit %s is in progress ..." % circ.id) + self.attacher.waiting_circuits.append((circ.id, self.d)) + + return self.state.build_circuit(path).addCallback( + AppendWaiting(self, deferred)).addErrback( + log.err) + +class TxtorconImportError(ImportError): + """ + Raised when ooni.lib.txtorcon cannot be imported from. Checks our current + working directory and the path given to see if txtorcon has been + initialized via /ooni/lib/Makefile. + """ + from os import getcwd, path + + cwd, tx = getcwd(), 'lib/txtorcon/torconfig.py' + try: + log.msg("Unable to import from ooni.lib.txtorcon") + if cwd.endswith('ooni'): + check = path.join(cwd, tx) + elif cwd.endswith('utils'): + check = path.join(cwd, '../'+tx) + else: + check = path.join(cwd, 'ooni/'+tx) + assert path.isfile(check) + except: + log.msg("Error: Some OONI libraries are missing!") + log.msg("Please go to /ooni/lib/ and do "make all"") + +class PTNoBridgesException(Exception): + """Raised when a pluggable transport is specified, but not bridges.""" + def __init__(self): + log.msg("Pluggable transport requires the bridges option") + return sys.exit() + +class PTNotFoundException(Exception): + def __init__(self, transport_type): + m = "Pluggable Transport type %s was unaccounted " % transport_type + m += "for, please contact isis(at)torproject(dot)org and it will " + m += "get included." + log.msg("%s" % m) + return sys.exit() + +@defer.inlineCallbacks +def __start_tor_with_timer__(reactor, config, control_port, tor_binary, + data_dir, bridges=None, relays=None, timeout=None, + retry=None): + """ + A wrapper for :func:`start_tor` which wraps the bootstrapping of a Tor + process and its connection to a reactor with a + :class:`twisted.internet.defer.Deferred` class decorator utility, + :func:`ooni.utils.timer.deferred_timeout`, and a mechanism for resets. + + ## XXX fill me in + """ + raise NotImplementedError + + class RetryException(Exception): + pass + + import sys + from ooni.utils.timer import deferred_timeout, TimeoutError + + def __make_var__(old, default, _type): + if old is not None: + assert isinstance(old, _type) + new = old + else: + new = default + return new + + reactor = reactor + timeout = __make_var__(timeout, 120, int) + retry = __make_var__(retry, 1, int) + + with_timeout = deferred_timeout(timeout)(start_tor) + + @defer.inlineCallbacks + def __start_tor__(rc=reactor, cf=config, cp=control_port, tb=tor_binary, + dd=data_dir, br=bridges, rl=relays, cb=setup_done, + eb=setup_fail, af=remove_public_relays, retry=retry): + try: + setup = yield with_timeout(rc,cf,cp,tb,dd) + except TimeoutError: + retry -= 1 + defer.returnValue(retry) + else: + if setup.callback: + setup = yield cb(setup) + elif setup.errback: + setup = yield eb(setup) + else: + setup = setup + + if br is not None: + state = af(setup,br) + else: + state = setup + defer.returnValue(state) + + @defer.inlineCallbacks + def __try_until__(tries): + result = yield __start_tor__() + try: + assert isinstance(result, int) + except AssertionError: + defer.returnValue(result) + else: + if result >= 0: + tried = yield __try_until__(result) + defer.returnValue(tried) + else: + raise RetryException + try: + tried = yield __try_until__(retry) + except RetryException: + log.msg("All retry attempts to bootstrap Tor have timed out.") + log.msg("Exiting ...") + defer.returnValue(sys.exit()) + else: + defer.returnValue(tried) diff --git a/ooni/bridget/utils/reports.py b/ooni/bridget/utils/reports.py new file mode 100644 index 0000000..ae67b13 --- /dev/null +++ b/ooni/bridget/utils/reports.py @@ -0,0 +1,144 @@ +from __future__ import with_statement + +import os +import yaml + +import itertools +from ooni.utils import log, date, net + +class Report: + """This is the ooni-probe reporting mechanism. It allows + reporting to multiple destinations and file formats. + + :scp the string of <host>:<port> of an ssh server + + :yaml the filename of a the yaml file to write + + :file the filename of a simple txt file to write + + :tcp the <host>:<port> of a TCP server that will just listen for + inbound connection and accept a stream of data (think of it + as a `nc -l -p <port> > filename.txt`) + """ + def __init__(self, testname=None, file="report.log", + scp=None, + tcp=None): + + self.testname = testname + self.file = file + self.tcp = tcp + self.scp = scp + #self.config = ooni.config.report + + #if self.config.timestamp: + # tmp = self.file.split('.') + # self.file = '.'.join(tmp[:-1]) + "-" + \ + # datetime.now().isoformat('-') + '.' + \ + # tmp[-1] + # print self.file + + self.scp = None + self.write_header() + + def write_header(self): + pretty_date = date.pretty_date() + header = "# OONI Probe Report for Test %s\n" % self.testname + header += "# %s\n\n" % pretty_date + self._write_to_report(header) + # XXX replace this with something proper + address = net.getClientAddress() + test_details = {'start_time': str(date.now()), + 'asn': address['asn'], + 'test_name': self.testname, + 'addr': address['ip']} + self(test_details) + + def _write_to_report(self, dump): + reports = [] + + if self.file: + reports.append("file") + + if self.tcp: + reports.append("tcp") + + if self.scp: + reports.append("scp") + + #XXX make this non blocking + for report in reports: + self.send_report(dump, report) + + def __call__(self, data): + """ + This should be invoked every time you wish to write some + data to the reporting system + """ + dump = yaml.dump([data]) + self._write_to_report(dump) + + def file_report(self, data): + """ + This reports to a file in YAML format + """ + with open(self.file, 'a+') as f: + f.write(data) + + def send_report(self, data, type): + """ + This sends the report using the + specified type. + """ + #print "Reporting %s to %s" % (data, type) + log.msg("Reporting to %s" % type) + getattr(self, type+"_report").__call__(data) + +class NewReport(object): + filename = 'report.log' + startTime = None + endTime = None + testName = None + ipAddr = None + asnAddr = None + + def _open(): + self.fp = open(self.filename, 'a+') + + @property + def header(): + pretty_date = date.pretty_date() + report_header = "# OONI Probe Report for Test %s\n" % self.testName + report_header += "# %s\n\n" % pretty_date + test_details = {'start_time': self.startTime, + 'asn': asnAddr, + 'test_name': self.testName, + 'addr': ipAddr} + report_header += yaml.dump([test_details]) + return report_header + + def create(): + """ + Create a new report by writing it's header. + """ + self.fp = open(self.filename, 'w+') + self.fp.write(self.header) + + def exists(): + """ + Returns False if the file does not exists. + """ + return os.path.exists(self.filename) + + def write(data): + """ + Write a report to the file. + + :data: python data structure to be written to report. + """ + if not self.exists(): + self.create() + else: + self._open() + yaml_encoded_data = yaml.dump([data]) + self.fp.write(yaml_encoded_data) + self.fp.close() diff --git a/ooni/bridget/utils/tests.py b/ooni/bridget/utils/tests.py new file mode 100644 index 0000000..ea4be0b --- /dev/null +++ b/ooni/bridget/utils/tests.py @@ -0,0 +1,141 @@ +import os +import yaml +from zope.interface import Interface, Attribute + +import logging +import itertools +from twisted.internet import reactor, defer, threads +## XXX why is this imported and not used? +from twisted.python import failure + +from ooni.utils import log, date +from ooni.plugoo import assets, work +from ooni.plugoo.reports import Report +from ooni.plugoo.interface import ITest + +class OONITest(object): + """ + This is the base class for writing OONI Tests. + + It should be used in conjunction with the ITest Interface. It allows the + developer to benefit from OONIs reporting system and command line argument + parsing system. + """ + name = "oonitest" + # By default we set this to False, meaning that we don't block + blocking = False + reactor = reactor + tool = False + ended = False + + def __init__(self, local_options, global_options, report, ooninet=None, + reactor=reactor): + # These are the options that are read through the tests suboptions + self.local_options = local_options + # These are the options global to all of OONI + self.global_options = global_options + self.report = report + #self.ooninet = ooninet + self.reactor = reactor + self.result = {} + self.initialize() + self.assets = self.load_assets() + + def initialize(self): + """ + Override this method if you are interested in having some extra + behavior when your test class is instantiated. + """ + pass + + def load_assets(self): + """ + This method should be overriden by the test writer to provide the + logic for loading their assets. + """ + return {} + + def __repr__(self): + return "<OONITest %s %s %s>" % (self.local_options, + self.global_options, + self.assets) + + def end(self): + """ + State that the current test should finish. + """ + self.ended = True + + def finished(self, return_value): + """ + The Test has finished running, we must now calculate the test runtime + and add all time data to the report. + """ + #self.ooninet.report(result) + self.end_time = date.now() + result = self.result + result['start_time'] = str(self.start_time) + result['end_time'] = str(self.end_time) + result['run_time'] = str(self.end_time - self.start_time) + result['return_value'] = return_value + log.msg("FINISHED %s" % result) + self.report(result) + return result + + def _do_experiment(self, args): + """ + A wrapper around the launch of experiment. + If we are running a blocking test experiment will be run in a thread if + not we expect it to return a Deferred. + + @param args: the asset line(s) that we are working on. + + returns a deferred. + """ + if self.blocking: + self.d = threads.deferToThread(self.experiment, args) + else: + self.d = self.experiment(args) + + self.d.addCallback(self.control, args) + self.d.addCallback(self.finished) + self.d.addErrback(self.finished) + return self.d + + def control(self, result, args): + """ + Run the control. + + @param result: what was returned by experiment. + + @param args: the asset(s) lines that we are working on. + """ + log.msg("Doing control") + return result + + def experiment(self, args): + """ + Run the experiment. This sample implementation returns a deferred, + making it a non-blocking test. + + @param args: the asset(s) lines that we are working on. + """ + log.msg("Doing experiment") + d = defer.Deferred() + return d + + def startTest(self, args): + """ + This method is invoked by the worker to start the test with one line of + the asset file. + + @param args: the asset(s) lines that we are working on. + """ + self.start_time = date.now() + + if self.shortName: + log.msg("Starting test %s" % self.shortName) + else: + log.msg("Starting test %s" % self.__class__) + + return self._do_experiment(args) diff --git a/ooni/bridget/utils/work.py b/ooni/bridget/utils/work.py new file mode 100644 index 0000000..c329c20 --- /dev/null +++ b/ooni/bridget/utils/work.py @@ -0,0 +1,147 @@ +# -*- coding: UTF-8 +""" + work.py + ********** + + This contains all code related to generating + Units of Work and processing it. + + :copyright: (c) 2012 by Arturo Filastò. + :license: see LICENSE for more details. + +""" +import itertools +import yaml +from datetime import datetime + +from zope.interface import Interface, Attribute + +from twisted.python import failure +from twisted.internet import reactor, defer + +class Worker(object): + """ + This is the core of OONI. It takes as input Work Units and + runs them concurrently. + """ + def __init__(self, maxconcurrent=10, reactor=reactor): + """ + @param maxconcurrent: how many test instances should be run + concurrently. + """ + self.reactor = reactor + self.maxconcurrent = maxconcurrent + self._running = 0 + self._queued = [] + + def _run(self, r): + """ + Check if we should start another test because we are below maximum + concurrency. + + This function is called every time a test finishes running. + + @param r: the return value of a previous test. + """ + if self._running > 0: + self._running -= 1 + + if self._running < self.maxconcurrent and self._queued: + workunit, d = self._queued.pop(0) + asset, test, idx = workunit + while test.ended and workunit: + try: + workunit, d = self._queued.pop(0) + asset, test, idx = workunit + except: + workunit = None + + if not test.ended: + self._running += 1 + actuald = test.startTest(asset).addBoth(self._run) + + if isinstance(r, failure.Failure): + # XXX probably we should be doing something to retry test running + r.trap() + + if self._running == 0 and not self._queued: + self.reactor.stop() + + return r + + def push(self, workunit): + """ + Add a test to the test queue and run it if we are not maxed out on + concurrency. + + @param workunit: a tuple containing the (asset, test, idx), where asset + is the line of the asset(s) we are working on, test + is an instantiated test and idx is the index we are + currently at. + """ + if self._running < self.maxconcurrent: + asset, test, idx = workunit + if not test.ended: + self._running += 1 + return test.startTest(asset).addBoth(self._run) + + d = defer.Deferred() + self._queued.append((workunit, d)) + return d + +class WorkGenerator(object): + """ + Factory responsible for creating units of work. + + This shall be run on the machine running OONI-cli. The returned WorkUnits + can either be run locally or on a remote OONI Node or Network Node. + """ + size = 10 + + def __init__(self, test, arguments=None, start=None): + self.Test = test + + if self.Test.assets and self.Test.assets.values()[0]: + self.assetGenerator = itertools.product(*self.Test.assets.values()) + else: + self.assetGenerator = None + + self.assetNames = self.Test.assets.keys() + + self.idx = 0 + self.end = False + if start: + self.skip(start) + + def __iter__(self): + return self + + def skip(self, start): + """ + Skip the first x number of lines of the asset. + + @param start: int how many items we should skip. + """ + for j in xrange(0, start-1): + for i in xrange(0, self.size): + self.assetGenerator.next() + self.idx += 1 + + def next(self): + if self.end: + raise StopIteration + + if not self.assetGenerator: + self.end = True + return ({}, self.Test, self.idx) + + try: + asset = self.assetGenerator.next() + ret = {} + for i, v in enumerate(asset): + ret[self.assetNames[i]] = v + except StopIteration: + raise StopIteration + + self.idx += 1 + return (ret, self.Test, self.idx) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py deleted file mode 100644 index 5ff7b3f..0000000 --- a/ooni/plugins/bridget.py +++ /dev/null @@ -1,500 +0,0 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- -# -# +-----------+ -# | BRIDGET | -# | +--------------------------------------------+ -# +--------| Use a Tor process to test making a Tor | -# | connection to a list of bridges or relays. | -# +--------------------------------------------+ -# -# :authors: Isis Lovecruft, Arturo Filasto -# :licence: see included LICENSE -# :version: 0.1.0-alpha - -from __future__ import with_statement -from functools import partial -from random import randint - -import os -import sys - -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import defer, error, reactor -from zope.interface import implements - -from ooni.utils import log, date -from ooni.utils.config import ValueChecker - -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset, MissingAssetException -from ooni.utils.onion import TxtorconImportError -from ooni.utils.onion import PTNoBridgesException, PTNotFoundException - -try: - from ooni.utils.onion import parse_data_dir -except: - log.msg("Please go to /ooni/lib and do 'make txtorcon' to run this test!") - -class RandomPortException(Exception): - """Raised when using a random port conflicts with configured ports.""" - def __init__(self): - log.msg("Unable to use random and specific ports simultaneously") - return sys.exit() - -class BridgetArgs(usage.Options): - """Commandline options.""" - allowed = "Port to use for Tor's %s, must be between 1024 and 65535." - sock_check = ValueChecker(allowed % "SocksPort").port_check - ctrl_check = ValueChecker(allowed % "ControlPort").port_check - - optParameters = [ - ['bridges', 'b', None, - 'File listing bridge IP:ORPorts to test'], - ['relays', 'f', None, - 'File listing relay IPs to test'], - ['socks', 's', 9049, None, sock_check], - ['control', 'c', 9052, None, ctrl_check], - ['torpath', 'p', None, - 'Path to the Tor binary to use'], - ['datadir', 'd', None, - 'Tor DataDirectory to use'], - ['transport', 't', None, - 'Tor ClientTransportPlugin'], - ['resume', 'r', 0, - 'Resume at this index']] - optFlags = [['random', 'x', 'Use random ControlPort and SocksPort']] - - def postOptions(self): - if not self['bridges'] and not self['relays']: - raise MissingAssetException( - "Bridget can't run without bridges or relays to test!") - if self['transport']: - ValueChecker.uid_check( - "Can't run bridget as root with pluggable transports!") - if not self['bridges']: - raise PTNoBridgesException - if self['socks'] or self['control']: - if self['random']: - raise RandomPortException - if self['datadir']: - ValueChecker.dir_check(self['datadir']) - if self['torpath']: - ValueChecker.file_check(self['torpath']) - -class BridgetAsset(Asset): - """Class for parsing bridget Assets ignoring commented out lines.""" - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - if line.startswith('#'): - return - else: - return line.replace('\n','') - -class BridgetTest(OONITest): - """ - XXX fill me in - - :ivar config: - An :class:`ooni.lib.txtorcon.TorConfig` instance. - :ivar relays: - A list of all provided relays to test. - :ivar bridges: - A list of all provided bridges to test. - :ivar socks_port: - Integer for Tor's SocksPort. - :ivar control_port: - Integer for Tor's ControlPort. - :ivar transport: - String defining the Tor's ClientTransportPlugin, for testing - a bridge's pluggable transport functionality. - :ivar tor_binary: - Path to the Tor binary to use, e.g. '/usr/sbin/tor' - """ - implements(IPlugin, ITest) - - shortName = "bridget" - description = "Use a Tor process to test connecting to bridges or relays" - requirements = None - options = BridgetArgs - blocking = False - - def initialize(self): - """ - Extra initialization steps. We only want one child Tor process - running, so we need to deal with most of the TorConfig() only once, - before the experiment runs. - """ - self.socks_port = 9049 - self.control_port = 9052 - self.circuit_timeout = 90 - self.tor_binary = '/usr/sbin/tor' - self.data_directory = None - - def __make_asset_list__(opt, lst): - log.msg("Loading information from %s ..." % opt) - with open(opt) as opt_file: - for line in opt_file.readlines(): - if line.startswith('#'): - continue - else: - lst.append(line.replace('\n','')) - - def __count_remaining__(which): - total, reach, unreach = map(lambda x: which[x], - ['all', 'reachable', 'unreachable']) - count = len(total) - reach() - unreach() - return count - - ## XXX should we do report['bridges_up'].append(self.bridges['current']) - self.bridges = {} - self.bridges['all'], self.bridges['up'], self.bridges['down'] = \ - ([] for i in range(3)) - self.bridges['reachable'] = lambda: len(self.bridges['up']) - self.bridges['unreachable'] = lambda: len(self.bridges['down']) - self.bridges['remaining'] = lambda: __count_remaining__(self.bridges) - self.bridges['current'] = None - self.bridges['pt_type'] = None - self.bridges['use_pt'] = False - - self.relays = {} - self.relays['all'], self.relays['up'], self.relays['down'] = \ - ([] for i in range(3)) - self.relays['reachable'] = lambda: len(self.relays['up']) - self.relays['unreachable'] = lambda: len(self.relays['down']) - self.relays['remaining'] = lambda: __count_remaining__(self.relays) - self.relays['current'] = None - - if self.local_options: - try: - from ooni.lib.txtorcon import TorConfig - except ImportError: - raise TxtorconImportError - else: - self.config = TorConfig() - finally: - options = self.local_options - - if options['bridges']: - self.config.UseBridges = 1 - __make_asset_list__(options['bridges'], self.bridges['all']) - if options['relays']: - ## first hop must be in TorState().guards - self.config.EntryNodes = ','.join(relay_list) - __make_asset_list__(options['relays'], self.relays['all']) - if options['socks']: - self.socks_port = options['socks'] - if options['control']: - self.control_port = options['control'] - if options['random']: - log.msg("Using randomized ControlPort and SocksPort ...") - self.socks_port = randint(1024, 2**16) - self.control_port = randint(1024, 2**16) - if options['torpath']: - self.tor_binary = options['torpath'] - if options['datadir']: - self.data_directory = parse_data_dir(options['datadir']) - if options['transport']: - ## ClientTransportPlugin transport exec pathtobinary [options] - ## XXX we need a better way to deal with all PTs - log.msg("Using ClientTransportPlugin %s" % options['transport']) - self.bridges['use_pt'] = True - [self.bridges['pt_type'], pt_exec] = \ - options['transport'].split(' ', 1) - - if self.bridges['pt_type'] == "obfs2": - self.config.ClientTransportPlugin = \ - self.bridges['pt_type'] + " " + pt_exec - else: - raise PTNotFoundException - - self.config.SocksPort = self.socks_port - self.config.ControlPort = self.control_port - self.config.CookieAuthentication = 1 - - def __load_assets__(self): - """ - Load bridges and/or relays from files given in user options. Bridges - should be given in the form IP:ORport. We don't want to load these as - assets, because it's inefficient to start a Tor process for each one. - - We cannot use the Asset model, because that model calls - self.experiment() with the current Assets, which would be one relay - and one bridge, then it gives the defer.Deferred returned from - self.experiment() to self.control(), which means that, for each - (bridge, relay) pair, experiment gets called again, which instantiates - an additional Tor process that attempts to bind to the same - ports. Thus, additionally instantiated Tor processes return with - RuntimeErrors, which break the final defer.chainDeferred.callback(), - sending it into the errback chain. - """ - assets = {} - if self.local_options: - if self.local_options['bridges']: - assets.update({'bridge': - BridgetAsset(self.local_options['bridges'])}) - if self.local_options['relays']: - assets.update({'relay': - BridgetAsset(self.local_options['relays'])}) - return assets - - def experiment(self, args): - """ - if bridges: - 1. configure first bridge line - 2a. configure data_dir, if it doesn't exist - 2b. write torrc to a tempfile in data_dir - 3. start tor } if any of these - 4. remove bridges which are public relays } fail, add current - 5. SIGHUP for each bridge } bridge to unreach- - } able bridges. - if relays: - 1a. configure the data_dir, if it doesn't exist - 1b. write torrc to a tempfile in data_dir - 2. start tor - 3. remove any of our relays which are already part of current - circuits - 4a. attach CustomCircuit() to self.state - 4b. RELAY_EXTEND for each relay } if this fails, add - } current relay to list - } of unreachable relays - 5. - if bridges and relays: - 1. configure first bridge line - 2a. configure data_dir if it doesn't exist - 2b. write torrc to a tempfile in data_dir - 3. start tor - 4. remove bridges which are public relays - 5. remove any of our relays which are already part of current - circuits - 6a. attach CustomCircuit() to self.state - 6b. for each bridge, build three circuits, with three - relays each - 6c. RELAY_EXTEND for each relay } if this fails, add - } current relay to list - } of unreachable relays - - :param args: - The :class:`BridgetAsset` line currently being used. Except that it - in Bridget it doesn't, so it should be ignored and avoided. - """ - try: - from ooni.utils import process - from ooni.utils.onion import remove_public_relays, start_tor - from ooni.utils.onion import start_tor_filter_nodes - from ooni.utils.onion import setup_fail, setup_done - from ooni.utils.onion import CustomCircuit - from ooni.utils.timer import deferred_timeout, TimeoutError - from ooni.lib.txtorcon import TorConfig, TorState - except ImportError: - raise TxtorconImportError - except TxtorconImportError, tie: - log.err(tie) - sys.exit() - - def reconfigure_done(state, bridges): - """ - Append :ivar:`bridges['current']` to the list - :ivar:`bridges['up']. - """ - log.msg("Reconfiguring with 'Bridge %s' successful" - % bridges['current']) - bridges['up'].append(bridges['current']) - return state - - def reconfigure_fail(state, bridges): - """ - Append :ivar:`bridges['current']` to the list - :ivar:`bridges['down']. - """ - log.msg("Reconfiguring TorConfig with parameters %s failed" - % state) - bridges['down'].append(bridges['current']) - return state - - @defer.inlineCallbacks - def reconfigure_bridge(state, bridges): - """ - Rewrite the Bridge line in our torrc. If use of pluggable - transports was specified, rewrite the line as: - Bridge <transport_type> <IP>:<ORPort> - Otherwise, rewrite in the standard form: - Bridge <IP>:<ORPort> - - :param state: - A fully bootstrapped instance of - :class:`ooni.lib.txtorcon.TorState`. - :param bridges: - A dictionary of bridges containing the following keys: - - bridges['remaining'] :: A function returning and int for the - number of remaining bridges to test. - bridges['current'] :: A string containing the <IP>:<ORPort> - of the current bridge. - bridges['use_pt'] :: A boolean, True if we're testing - bridges with a pluggable transport; - False otherwise. - bridges['pt_type'] :: If :ivar:`bridges['use_pt'] is True, - this is a string containing the type - of pluggable transport to test. - :return: - :param:`state` - """ - log.msg("Current Bridge: %s" % bridges['current']) - log.msg("We now have %d bridges remaining to test..." - % bridges['remaining']()) - try: - if bridges['use_pt'] is False: - controller_response = yield state.protocol.set_conf( - 'Bridge', bridges['current']) - elif bridges['use_pt'] and bridges['pt_type'] is not None: - controller_reponse = yield state.protocol.set_conf( - 'Bridge', bridges['pt_type'] +' '+ bridges['current']) - else: - raise PTNotFoundException - - if controller_response == 'OK': - finish = yield reconfigure_done(state, bridges) - else: - log.err("SETCONF for %s responded with error:\n %s" - % (bridges['current'], controller_response)) - finish = yield reconfigure_fail(state, bridges) - - defer.returnValue(finish) - - except Exception, e: - log.err("Reconfiguring torrc with Bridge line %s failed:\n%s" - % (bridges['current'], e)) - defer.returnValue(None) - - def attacher_extend_circuit(attacher, deferred, router): - ## XXX todo write me - ## state.attacher.extend_circuit - raise NotImplemented - #attacher.extend_circuit - - def state_attach(state, path): - log.msg("Setting up custom circuit builder...") - attacher = CustomCircuit(state) - state.set_attacher(attacher, reactor) - state.add_circuit_listener(attacher) - return state - - ## OLD - #for circ in state.circuits.values(): - # for relay in circ.path: - # try: - # relay_list.remove(relay) - # except KeyError: - # continue - ## XXX how do we attach to circuits with bridges? - d = defer.Deferred() - attacher.request_circuit_build(d) - return d - - def state_attach_fail(state): - log.err("Attaching custom circuit builder failed: %s" % state) - - log.msg("Bridget: initiating test ... ") ## Start the experiment - - ## if we've at least one bridge, and our config has no 'Bridge' line - if self.bridges['remaining']() >= 1 \ - and not 'Bridge' in self.config.config: - - ## configure our first bridge line - self.bridges['current'] = self.bridges['all'][0] - self.config.Bridge = self.bridges['current'] - ## avoid starting several - self.config.save() ## processes - assert self.config.config.has_key('Bridge'), "No Bridge Line" - - ## start tor and remove bridges which are public relays - from ooni.utils.onion import start_tor_filter_nodes - state = start_tor_filter_nodes(reactor, self.config, - self.control_port, self.tor_binary, - self.data_directory, self.bridges) - #controller = defer.Deferred() - #controller.addCallback(singleton_semaphore, tor) - #controller.addErrback(setup_fail) - #bootstrap = defer.gatherResults([controller, filter_bridges], - # consumeErrors=True) - - if state is not None: - log.debug("state:\n%s" % state) - log.debug("Current callbacks on TorState():\n%s" - % state.callbacks) - - ## if we've got more bridges - if self.bridges['remaining']() >= 2: - #all = [] - for bridge in self.bridges['all'][1:]: - self.bridges['current'] = bridge - #new = defer.Deferred() - #new.addCallback(reconfigure_bridge, state, self.bridges) - #all.append(new) - #check_remaining = defer.DeferredList(all, consumeErrors=True) - #state.chainDeferred(check_remaining) - state.addCallback(reconfigure_bridge, self.bridges) - - if self.relays['remaining']() > 0: - while self.relays['remaining']() >= 3: - #path = list(self.relays.pop() for i in range(3)) - #log.msg("Trying path %s" % '->'.join(map(lambda node: - # node, path))) - self.relays['current'] = self.relays['all'].pop() - for circ in state.circuits.values(): - for node in circ.path: - if node == self.relays['current']: - self.relays['up'].append(self.relays['current']) - if len(circ.path) < 3: - try: - ext = attacher_extend_circuit(state.attacher, circ, - self.relays['current']) - ext.addCallback(attacher_extend_circuit_done, - state.attacher, circ, - self.relays['current']) - except Exception, e: - log.err("Extend circuit failed: %s" % e) - else: - continue - - #state.callback(all) - #self.reactor.run() - return state - - def startTest(self, args): - """ - Local override of :meth:`OONITest.startTest` to bypass calling - self.control. - - :param args: - The current line of :class:`Asset`, not used but kept for - compatibility reasons. - :return: - A fired deferred which callbacks :meth:`experiment` and - :meth:`OONITest.finished`. - """ - self.start_time = date.now() - self.d = self.experiment(args) - self.d.addErrback(log.err) - self.d.addCallbacks(self.finished, log.err) - return self.d - -## So that getPlugins() can register the Test: -#bridget = BridgetTest(None, None, None) - - -## ISIS' NOTES -## ----------- -## TODO: -## x cleanup documentation -## x add DataDirectory option -## x check if bridges are public relays -## o take bridge_desc file as input, also be able to give same -## format as output -## x Add asynchronous timeout for deferred, so that we don't wait -## o Add assychronous timout for deferred, so that we don't wait -## forever for bridges that don't work.