[tor-commits] [ooni-probe/master] * Moving bridget around to be better integrated with the new ooni structure.

commit 276aeea3a524018df7a4535add3e2ad83f755e33 Author: Isis Lovecruft isis@torproject.org Date: Sun Nov 4 12:23:09 2012 +0000

* Moving bridget around to be better integrated with the new ooni structure. --- ooni/bridget/__init__.py | 14 - ooni/bridget/custodiet.py | 421 --------------------- ooni/bridget/tests/__init__.py | 14 - ooni/bridget/tests/bridget.py | 499 ------------------------- ooni/bridget/tests/echo.py | 205 ----------- ooni/bridget/tests/tls-handshake.py | 32 -- ooni/bridget/utils/__init__.py | 1 - ooni/bridget/utils/inputs.py | 174 --------- ooni/bridget/utils/interface.py | 54 --- ooni/bridget/utils/log.py | 98 ----- ooni/bridget/utils/nodes.py | 176 --------- ooni/bridget/utils/onion.py | 686 ----------------------------------- ooni/bridget/utils/reports.py | 144 -------- ooni/bridget/utils/tests.py | 141 ------- ooni/bridget/utils/work.py | 147 -------- 15 files changed, 0 insertions(+), 2806 deletions(-)

diff --git a/ooni/bridget/__init__.py b/ooni/bridget/__init__.py deleted file mode 100644 index 4648d77..0000000 --- a/ooni/bridget/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -#-*- coding: utf-8 -*- - -#import os, sys -#import copy_reg - -## Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain. -#sys.path[:] = map(os.path.abspath, sys.path) -#sys.path.insert(0, os.path.abspath(os.getcwd())) - -## This is a hack to overcome a bug in python -#from ooni.utils.hacks import patched_reduce_ex -#copy_reg._reduce_ex = patched_reduce_ex - -__all__ = ['custodiet'] diff --git a/ooni/bridget/custodiet.py b/ooni/bridget/custodiet.py deleted file mode 100755 index 8cbcfce..0000000 --- a/ooni/bridget/custodiet.py +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -# -# custodiet -# ********* -# -# "...quis custodiet ipsos custodes?" -# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.) -# -# "'Hand me the Custodian,' Goodchild demands, inserting the waiflike -# robot into Bambara's opened navel. 'Providing conscience for those who -# have none.' Goodchild and the other Breen government agents disappear -# into the surrounding desert in a vehicle, kicking up cloud of white dust. -# Bambara awakens, and, patting the dust from his clothing, turns to -# greet a one-armed child. 'Hi, my name's Bambara; I'm a -# thirty-six-year-old Virgo and a former killer, who's hobbies include -# performing recreational autopsies, defecating, and drinking rum. I've -# recently been given a conscience, and would very much like to help you.' -# Cut to Bambara and the child, now with one of Bambara's arms, leaving -# a surgical clinic." -# - AeonFlux, "The Purge" (sometime in the late 90s) -# -# :copyright: (c) 2012 Isis Lovecruft -# :license: see LICENSE for more details. -# :version: 0.1.0-beta -# - -# ooniprobe.py imports -import sys -from signal import SIGTERM, signal -from pprint import pprint - -from twisted.python import usage -from twisted.internet import reactor -from twisted.plugin import getPlugins - -from zope.interface.verify import verifyObject -from zope.interface.exceptions import BrokenImplementation -from zope.interface.exceptions import BrokenMethodImplementation - -from ooni.bridget.tests import bridget -from ooni.bridget.utils import log, tests, work, reports -from ooni.bridget.utils.interface import ITest -from ooni.utils.logo import getlogo - -# runner.py imports -import os -import types -import time -import inspect -import yaml - -from twisted.internet import defer, reactor -from twisted.python import reflect, failure, usage -from twisted.python import log as tlog - -from twisted.trial import unittest -from twisted.trial.runner import TrialRunner, TestLoader -from twisted.trial.runner import isPackage, isTestCase, ErrorHolder -from twisted.trial.runner import filenameToModule, _importFromFile - -from ooni import nettest -from ooni.inputunit import InputUnitFactory -from ooni.nettest import InputTestSuite -from ooni.plugoo import tests as oonitests -from ooni.reporter import ReporterFactory -from ooni.utils import log, geodata, date -from ooni.utils.legacy import LegacyOONITest -from ooni.utils.legacy import start_legacy_test, adapt_legacy_test - - -__version__ = "0.1.0-beta" - - -#def retrieve_plugoo(): -# """ -# Get all the plugins that implement the ITest interface and get the data -# associated to them into a dict. -# """ -# interface = ITest -# d = {} -# error = False -# for p in getPlugins(interface, plugins): -# try: -# verifyObject(interface, p) -# d[p.shortName] = p -# except BrokenImplementation, bi: -# print "Plugin Broken" -# print bi -# error = True -# if error != False: -# print "Plugin Loaded!" -# return d -# -#plugoo = retrieve_plugoo() - -""" - -ai to watch over which tests to run - custodiet - - * runTest() or getPrefixMethodNames() to run the tests in order for each - test (esp. the tcp and icmp parts) to be oonicompat we should use the - test_icmp_ping API framework for those. - - * should handle calling - -tests to run: - echo - syn - fin - conn - tls - tor -need fakebridge - canary - -""" - -def runTest(test, options, global_options, reactor=reactor): - """ - Run an OONI probe test by name. - - @param test: a string specifying the test name as specified inside of - shortName. - - @param options: the local options to be passed to the test. - - @param global_options: the global options for OONI - """ - parallelism = int(global_options['parallelism']) - worker = work.Worker(parallelism, reactor=reactor) - test_class = plugoo[test].__class__ - report = reports.Report(test, global_options['output']) - - log_to_stdout = True - if global_options['quiet']: - log_to_stdout = False - - log.start(log_to_stdout, - global_options['log'], - global_options['verbosity']) - - resume = 0 - if not options: - options = {} - if 'resume' in options: - resume = options['resume'] - - test = test_class(options, global_options, report, reactor=reactor) - if test.tool: - test.runTool() - return True - - if test.ended: - print "Ending test" - return None - - wgen = work.WorkGenerator(test, - dict(options), - start=resume) - for x in wgen: - worker.push(x) - -class MainOptions(usage.Options): - tests = [bridget, ] - subCommands = [] - for test in tests: - print test - testopt = getattr(test, 'options') - subCommands.append([test, None, testopt, "Run the %s test" % test]) - - optFlags = [ - ['quiet', 'q', "Don't log to stdout"] - ] - - optParameters = [ - ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], - #['target-node', 't', 'localhost:31415', 'Select target node'], - ['output', 'o', 'bridge.log', "Specify output report file"], - ['reportfile', 'o', 'bridge.log', "Specify output log file"], - ['verbosity', 'v', 1, "Specify the logging level"], - ] - - def opt_version(self): - """ - Display OONI version and exit. - """ - print "OONI version:", __version__ - sys.exit(0) - - def __str__(self): - """ - Hack to get the sweet ascii art into the help output and replace the - strings "Commands" with "Tests". - """ - return getlogo() + '\n' + self.getSynopsis() + '\n' + \ - self.getUsage(width=None).replace("Commands:", "Tests:") - - - -def isTestCase(thing): - try: - return issubclass(thing, unittest.TestCase) - except TypeError: - return False - -def isLegacyTest(obj): - """ - Returns True if the test in question is written using the OONITest legacy - class. - We do this for backward compatibility of the OONIProbe API. - """ - try: - if issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest: - return True - else: - return False - except TypeError: - return False - -def processTest(obj, config): - """ - Process the parameters and :class:`twisted.python.usage.Options` of a - :class:`ooni.nettest.Nettest`. - - :param obj: - An uninstantiated old test, which should be a subclass of - :class:`ooni.plugoo.tests.OONITest`. - :param config: - A configured and instantiated :class:`twisted.python.usage.Options` - class. - """ - - inputFile = obj.inputFile - - if obj.optParameters or inputFile: - if not obj.optParameters: - obj.optParameters = [] - - if inputFile: - obj.optParameters.append(inputFile) - - class Options(usage.Options): - optParameters = obj.optParameters - - options = Options() - options.parseOptions(config['subArgs']) - obj.localOptions = options - - if inputFile: - obj.inputFile = options[inputFile[0]] - try: - tmp_obj = obj() - tmp_obj.getOptions() - except usage.UsageError: - options.opt_help() - - return obj - -def findTestClassesFromConfig(config): - """ - Takes as input the command line config parameters and returns the test - case classes. - If it detects that a certain test class is using the old OONIProbe format, - then it will adapt it to the new testing system. - - :param config: - A configured and instantiated :class:`twisted.python.usage.Options` - class. - :return: - A list of class objects found in a file or module given on the - commandline. - """ - - filename = config['test'] - classes = [] - - module = filenameToModule(filename) - for name, val in inspect.getmembers(module): - if isTestCase(val): - classes.append(processTest(val, config)) - elif isLegacyTest(val): - classes.append(adapt_legacy_test(val, config)) - return classes - -def makeTestCases(klass, tests, methodPrefix): - """ - Takes a class some tests and returns the test cases. methodPrefix is how - the test case functions should be prefixed with. - """ - - cases = [] - for test in tests: - cases.append(klass(methodPrefix+test)) - return cases - -def loadTestsAndOptions(classes, config): - """ - Takes a list of classes and returns their testcases and options. - Legacy tests will be adapted. - """ - - methodPrefix = 'test' - suiteFactory = InputTestSuite - options = [] - testCases = [] - names = [] - - _old_klass_type = LegacyOONITest - - for klass in classes: - if isinstance(klass, _old_klass_type): - try: - cases = start_legacy_test(klass) - #cases.callback() - if cases: - print cases - return [], [] - testCases.append(cases) - except Exception, e: - log.err(e) - else: - try: - opts = klass.local_options - options.append(opts) - except AttributeError, ae: - options.append([]) - log.err(ae) - elif not isinstance(klass, _old_klass_type): - tests = reflect.prefixedMethodNames(klass, methodPrefix) - if tests: - cases = makeTestCases(klass, tests, methodPrefix) - testCases.append(cases) - try: - k = klass() - opts = k.getOptions() - options.append(opts) - except AttributeError, ae: - options.append([]) - log.err(ae) - else: - try: - raise RuntimeError, "Class is some strange type!" - except RuntimeError, re: - log.err(re) - - return testCases, options - -class ORunner(object): - """ - This is a specialized runner used by the ooniprobe command line tool. - I am responsible for reading the inputs from the test files and splitting - them in input units. I also create all the report instances required to run - the tests. - """ - def __init__(self, cases, options=None, config=None, *arg, **kw): - self.baseSuite = InputTestSuite - self.cases = cases - self.options = options - - try: - assert len(options) != 0, "Length of options is zero!" - except AssertionError, ae: - self.inputs = [] - log.err(ae) - else: - try: - first = options.pop(0) - except: - first = {} - if 'inputs' in first: - self.inputs = options['inputs'] - else: - log.msg("Could not find inputs!") - log.msg("options[0] = %s" % first) - self.inputs = [None] - - try: - reportFile = open(config['reportfile'], 'a+') - except: - filename = 'report_'+date.timestamp()+'.yaml' - reportFile = open(filename, 'a+') - self.reporterFactory = ReporterFactory(reportFile, - testSuite=self.baseSuite(self.cases)) - - def runWithInputUnit(self, inputUnit): - idx = 0 - result = self.reporterFactory.create() - - for inputs in inputUnit: - result.reporterFactory = self.reporterFactory - - suite = self.baseSuite(self.cases) - suite.input = inputs - suite(result, idx) - - # XXX refactor all of this index bullshit to avoid having to pass - # this index around. Probably what I want to do is go and make - # changes to report to support the concept of having multiple runs - # of the same test. - # We currently need to do this addition in order to get the number - # of times the test cases that have run inside of the test suite. - idx += (suite._idx - idx) - - result.done() - - def run(self): - self.reporterFactory.options = self.options - for inputUnit in InputUnitFactory(self.inputs): - self.runWithInputUnit(inputUnit) - -if __name__ == "__main__": - config = Options() - config.parseOptions() - - if not config.subCommand: - config.opt_help() - signal(SIGTERM) - #sys.exit(1) - - runTest(config.subCommand, config.subOptions, config) - reactor.run() diff --git a/ooni/bridget/tests/__init__.py b/ooni/bridget/tests/__init__.py deleted file mode 100644 index 9ecc88d..0000000 --- a/ooni/bridget/tests/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: UTF-8 -# -# bridget/tests/__init__.py -# ************************* -# -# "...quis custodiet ipsos custodes?" -# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.) -# -# :copyright: (c) 2012 Isis Lovecruft -# :license: see LICENSE for more details. -# :version: 0.1.0-beta -# - -all = ['bridget'] diff --git a/ooni/bridget/tests/bridget.py b/ooni/bridget/tests/bridget.py deleted file mode 100644 index a334747..0000000 --- a/ooni/bridget/tests/bridget.py +++ /dev/null @@ -1,499 +0,0 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- -# -# +-----------+ -# | BRIDGET | -# | +--------------------------------------------+ -# +--------| Use a Tor process to test making a Tor | -# | connection to a list of bridges or relays. | -# +--------------------------------------------+ -# -# :authors: Isis Lovecruft, Arturo Filasto -# :licence: see included LICENSE -# :version: 0.1.0-alpha - -from __future__ import with_statement -from functools import partial -from random import randint - -import os -import sys - -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import defer, error, reactor -from zope.interface import implements - -from ooni.utils import log, date -from ooni.utils.config import ValueChecker - -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset, MissingAssetException -from ooni.utils.onion import TxtorconImportError -from ooni.utils.onion import PTNoBridgesException, PTNotFoundException - -try: - from ooni.utils.onion import parse_data_dir -except: - log.msg("Please go to /ooni/lib and do 'make txtorcon' to run this test!") - -class RandomPortException(Exception): - """Raised when using a random port conflicts with configured ports.""" - def __init__(self): - log.msg("Unable to use random and specific ports simultaneously") - return sys.exit() - -class BridgetArgs(usage.Options): - """Commandline options.""" - allowed = "Port to use for Tor's %s, must be between 1024 and 65535." - sock_check = ValueChecker(allowed % "SocksPort").port_check - ctrl_check = ValueChecker(allowed % "ControlPort").port_check - - optParameters = [ - ['bridges', 'b', None, - 'File listing bridge IP:ORPorts to test'], - ['relays', 'f', None, - 'File listing relay IPs to test'], - ['socks', 's', 9049, None, sock_check], - ['control', 'c', 9052, None, ctrl_check], - ['torpath', 'p', None, - 'Path to the Tor binary to use'], - ['datadir', 'd', None, - 'Tor DataDirectory to use'], - ['transport', 't', None, - 'Tor ClientTransportPlugin'], - ['resume', 'r', 0, - 'Resume at this index']] - optFlags = [['random', 'x', 'Use random ControlPort and SocksPort']] - - def postOptions(self): - if not self['bridges'] and not self['relays']: - raise MissingAssetException( - "Bridget can't run without bridges or relays to test!") - if self['transport']: - ValueChecker.uid_check( - "Can't run bridget as root with pluggable transports!") - if not self['bridges']: - raise PTNoBridgesException - if self['socks'] or self['control']: - if self['random']: - raise RandomPortException - if self['datadir']: - ValueChecker.dir_check(self['datadir']) - if self['torpath']: - ValueChecker.file_check(self['torpath']) - -class BridgetAsset(Asset): - """Class for parsing bridget Assets ignoring commented out lines.""" - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - if line.startswith('#'): - return - else: - return line.replace('\n','') - -class BridgetTest(OONITest): - """ - XXX fill me in - - :ivar config: - An :class:`ooni.lib.txtorcon.TorConfig` instance. - :ivar relays: - A list of all provided relays to test. - :ivar bridges: - A list of all provided bridges to test. - :ivar socks_port: - Integer for Tor's SocksPort. - :ivar control_port: - Integer for Tor's ControlPort. - :ivar transport: - String defining the Tor's ClientTransportPlugin, for testing - a bridge's pluggable transport functionality. - :ivar tor_binary: - Path to the Tor binary to use, e.g. '/usr/sbin/tor' - """ - implements(IPlugin, ITest) - - shortName = "bridget" - description = "Use a Tor process to test connecting to bridges or relays" - requirements = None - options = BridgetArgs - blocking = False - - def initialize(self): - """ - Extra initialization steps. We only want one child Tor process - running, so we need to deal with most of the TorConfig() only once, - before the experiment runs. - """ - self.socks_port = 9049 - self.control_port = 9052 - self.circuit_timeout = 90 - self.tor_binary = '/usr/sbin/tor' - self.data_directory = None - - def __make_asset_list__(opt, lst): - log.msg("Loading information from %s ..." % opt) - with open(opt) as opt_file: - for line in opt_file.readlines(): - if line.startswith('#'): - continue - else: - lst.append(line.replace('\n','')) - - def __count_remaining__(which): - total, reach, unreach = map(lambda x: which[x], - ['all', 'reachable', 'unreachable']) - count = len(total) - reach() - unreach() - return count - - ## XXX should we do report['bridges_up'].append(self.bridges['current']) - self.bridges = {} - self.bridges['all'], self.bridges['up'], self.bridges['down'] = \ - ([] for i in range(3)) - self.bridges['reachable'] = lambda: len(self.bridges['up']) - self.bridges['unreachable'] = lambda: len(self.bridges['down']) - self.bridges['remaining'] = lambda: __count_remaining__(self.bridges) - self.bridges['current'] = None - self.bridges['pt_type'] = None - self.bridges['use_pt'] = False - - self.relays = {} - self.relays['all'], self.relays['up'], self.relays['down'] = \ - ([] for i in range(3)) - self.relays['reachable'] = lambda: len(self.relays['up']) - self.relays['unreachable'] = lambda: len(self.relays['down']) - self.relays['remaining'] = lambda: __count_remaining__(self.relays) - self.relays['current'] = None - - if self.local_options: - try: - from ooni.lib.txtorcon import TorConfig - except ImportError: - raise TxtorconImportError - else: - self.config = TorConfig() - finally: - options = self.local_options - - if options['bridges']: - self.config.UseBridges = 1 - __make_asset_list__(options['bridges'], self.bridges['all']) - if options['relays']: - ## first hop must be in TorState().guards - self.config.EntryNodes = ','.join(relay_list) - __make_asset_list__(options['relays'], self.relays['all']) - if options['socks']: - self.socks_port = options['socks'] - if options['control']: - self.control_port = options['control'] - if options['random']: - log.msg("Using randomized ControlPort and SocksPort ...") - self.socks_port = randint(1024, 2**16) - self.control_port = randint(1024, 2**16) - if options['torpath']: - self.tor_binary = options['torpath'] - if options['datadir']: - self.data_directory = parse_data_dir(options['datadir']) - if options['transport']: - ## ClientTransportPlugin transport exec pathtobinary [options] - ## XXX we need a better way to deal with all PTs - log.msg("Using ClientTransportPlugin %s" % options['transport']) - self.bridges['use_pt'] = True - [self.bridges['pt_type'], pt_exec] = \ - options['transport'].split(' ', 1) - - if self.bridges['pt_type'] == "obfs2": - self.config.ClientTransportPlugin = \ - self.bridges['pt_type'] + " " + pt_exec - else: - raise PTNotFoundException - - self.config.SocksPort = self.socks_port - self.config.ControlPort = self.control_port - self.config.CookieAuthentication = 1 - - def __load_assets__(self): - """ - Load bridges and/or relays from files given in user options. Bridges - should be given in the form IP:ORport. We don't want to load these as - assets, because it's inefficient to start a Tor process for each one. - - We cannot use the Asset model, because that model calls - self.experiment() with the current Assets, which would be one relay - and one bridge, then it gives the defer.Deferred returned from - self.experiment() to self.control(), which means that, for each - (bridge, relay) pair, experiment gets called again, which instantiates - an additional Tor process that attempts to bind to the same - ports. Thus, additionally instantiated Tor processes return with - RuntimeErrors, which break the final defer.chainDeferred.callback(), - sending it into the errback chain. - """ - assets = {} - if self.local_options: - if self.local_options['bridges']: - assets.update({'bridge': - BridgetAsset(self.local_options['bridges'])}) - if self.local_options['relays']: - assets.update({'relay': - BridgetAsset(self.local_options['relays'])}) - return assets - - def experiment(self, args): - """ - if bridges: - 1. configure first bridge line - 2a. configure data_dir, if it doesn't exist - 2b. write torrc to a tempfile in data_dir - 3. start tor } if any of these - 4. remove bridges which are public relays } fail, add current - 5. SIGHUP for each bridge } bridge to unreach- - } able bridges. - if relays: - 1a. configure the data_dir, if it doesn't exist - 1b. write torrc to a tempfile in data_dir - 2. start tor - 3. remove any of our relays which are already part of current - circuits - 4a. attach CustomCircuit() to self.state - 4b. RELAY_EXTEND for each relay } if this fails, add - } current relay to list - } of unreachable relays - 5. - if bridges and relays: - 1. configure first bridge line - 2a. configure data_dir if it doesn't exist - 2b. write torrc to a tempfile in data_dir - 3. start tor - 4. remove bridges which are public relays - 5. remove any of our relays which are already part of current - circuits - 6a. attach CustomCircuit() to self.state - 6b. for each bridge, build three circuits, with three - relays each - 6c. RELAY_EXTEND for each relay } if this fails, add - } current relay to list - } of unreachable relays - - :param args: - The :class:`BridgetAsset` line currently being used. Except that it - in Bridget it doesn't, so it should be ignored and avoided. - """ - try: - from ooni.utils import process - from ooni.utils.onion import remove_public_relays, start_tor - from ooni.utils.onion import start_tor_filter_nodes - from ooni.utils.onion import setup_fail, setup_done - from ooni.utils.onion import CustomCircuit - from ooni.utils.timer import deferred_timeout, TimeoutError - from ooni.lib.txtorcon import TorConfig, TorState - except ImportError: - raise TxtorconImportError - except TxtorconImportError, tie: - log.err(tie) - sys.exit() - - def reconfigure_done(state, bridges): - """ - Append :ivar:`bridges['current']` to the list - :ivar:`bridges['up']. - """ - log.msg("Reconfiguring with 'Bridge %s' successful" - % bridges['current']) - bridges['up'].append(bridges['current']) - return state - - def reconfigure_fail(state, bridges): - """ - Append :ivar:`bridges['current']` to the list - :ivar:`bridges['down']. - """ - log.msg("Reconfiguring TorConfig with parameters %s failed" - % state) - bridges['down'].append(bridges['current']) - return state - - @defer.inlineCallbacks - def reconfigure_bridge(state, bridges): - """ - Rewrite the Bridge line in our torrc. If use of pluggable - transports was specified, rewrite the line as: - Bridge <transport_type> <IP>:<ORPort> - Otherwise, rewrite in the standard form: - Bridge <IP>:<ORPort> - - :param state: - A fully bootstrapped instance of - :class:`ooni.lib.txtorcon.TorState`. - :param bridges: - A dictionary of bridges containing the following keys: - - bridges['remaining'] :: A function returning and int for the - number of remaining bridges to test. - bridges['current'] :: A string containing the <IP>:<ORPort> - of the current bridge. - bridges['use_pt'] :: A boolean, True if we're testing - bridges with a pluggable transport; - False otherwise. - bridges['pt_type'] :: If :ivar:`bridges['use_pt'] is True, - this is a string containing the type - of pluggable transport to test. - :return: - :param:`state` - """ - log.msg("Current Bridge: %s" % bridges['current']) - log.msg("We now have %d bridges remaining to test..." - % bridges['remaining']()) - try: - if bridges['use_pt'] is False: - controller_response = yield state.protocol.set_conf( - 'Bridge', bridges['current']) - elif bridges['use_pt'] and bridges['pt_type'] is not None: - controller_reponse = yield state.protocol.set_conf( - 'Bridge', bridges['pt_type'] +' '+ bridges['current']) - else: - raise PTNotFoundException - - if controller_response == 'OK': - finish = yield reconfigure_done(state, bridges) - else: - log.err("SETCONF for %s responded with error:\n %s" - % (bridges['current'], controller_response)) - finish = yield reconfigure_fail(state, bridges) - - defer.returnValue(finish) - - except Exception, e: - log.err("Reconfiguring torrc with Bridge line %s failed:\n%s" - % (bridges['current'], e)) - defer.returnValue(None) - - def attacher_extend_circuit(attacher, deferred, router): - ## XXX todo write me - ## state.attacher.extend_circuit - raise NotImplemented - #attacher.extend_circuit - - def state_attach(state, path): - log.msg("Setting up custom circuit builder...") - attacher = CustomCircuit(state) - state.set_attacher(attacher, reactor) - state.add_circuit_listener(attacher) - return state - - ## OLD - #for circ in state.circuits.values(): - # for relay in circ.path: - # try: - # relay_list.remove(relay) - # except KeyError: - # continue - ## XXX how do we attach to circuits with bridges? - d = defer.Deferred() - attacher.request_circuit_build(d) - return d - - def state_attach_fail(state): - log.err("Attaching custom circuit builder failed: %s" % state) - - log.msg("Bridget: initiating test ... ") ## Start the experiment - - ## if we've at least one bridge, and our config has no 'Bridge' line - if self.bridges['remaining']() >= 1 \ - and not 'Bridge' in self.config.config: - - ## configure our first bridge line - self.bridges['current'] = self.bridges['all'][0] - self.config.Bridge = self.bridges['current'] - ## avoid starting several - self.config.save() ## processes - assert self.config.config.has_key('Bridge'), "No Bridge Line" - - ## start tor and remove bridges which are public relays - from ooni.utils.onion import start_tor_filter_nodes - state = start_tor_filter_nodes(reactor, self.config, - self.control_port, self.tor_binary, - self.data_directory, self.bridges) - #controller = defer.Deferred() - #controller.addCallback(singleton_semaphore, tor) - #controller.addErrback(setup_fail) - #bootstrap = defer.gatherResults([controller, filter_bridges], - # consumeErrors=True) - - if state is not None: - log.debug("state:\n%s" % state) - log.debug("Current callbacks on TorState():\n%s" - % state.callbacks) - - ## if we've got more bridges - if self.bridges['remaining']() >= 2: - #all = [] - for bridge in self.bridges['all'][1:]: - self.bridges['current'] = bridge - #new = defer.Deferred() - #new.addCallback(reconfigure_bridge, state, self.bridges) - #all.append(new) - #check_remaining = defer.DeferredList(all, consumeErrors=True) - #state.chainDeferred(check_remaining) - state.addCallback(reconfigure_bridge, self.bridges) - - if self.relays['remaining']() > 0: - while self.relays['remaining']() >= 3: - #path = list(self.relays.pop() for i in range(3)) - #log.msg("Trying path %s" % '->'.join(map(lambda node: - # node, path))) - self.relays['current'] = self.relays['all'].pop() - for circ in state.circuits.values(): - for node in circ.path: - if node == self.relays['current']: - self.relays['up'].append(self.relays['current']) - if len(circ.path) < 3: - try: - ext = attacher_extend_circuit(state.attacher, circ, - self.relays['current']) - ext.addCallback(attacher_extend_circuit_done, - state.attacher, circ, - self.relays['current']) - except Exception, e: - log.err("Extend circuit failed: %s" % e) - else: - continue - - #state.callback(all) - #self.reactor.run() - return state - - def startTest(self, args): - """ - Local override of :meth:`OONITest.startTest` to bypass calling - self.control. - - :param args: - The current line of :class:`Asset`, not used but kept for - compatibility reasons. - :return: - A fired deferred which callbacks :meth:`experiment` and - :meth:`OONITest.finished`. - """ - self.start_time = date.now() - self.d = self.experiment(args) - self.d.addErrback(log.err) - self.d.addCallbacks(self.finished, log.err) - return self.d - -## So that getPlugins() can register the Test: -#bridget = BridgetTest(None, None, None) - -## ISIS' NOTES -## ----------- -## TODO: -## x cleanup documentation -## x add DataDirectory option -## x check if bridges are public relays -## o take bridge_desc file as input, also be able to give same -## format as output -## x Add asynchronous timeout for deferred, so that we don't wait -## o Add assychronous timout for deferred, so that we don't wait -## forever for bridges that don't work. diff --git a/ooni/bridget/tests/echo.py b/ooni/bridget/tests/echo.py deleted file mode 100644 index 7f3217a..0000000 --- a/ooni/bridget/tests/echo.py +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# +---------+ -# | echo.py | -# +---------+ -# A simple ICMP-8 ping test. -# -# :author: Isis Lovecruft -# :version: 0.0.1-pre-alpha -# :license: (c) 2012 Isis Lovecruft -# see attached LICENCE file -# - -import os -import sys - -from pprint import pprint - -from twisted.internet import reactor -from twisted.plugin import IPlugin -from twisted.python import usage -from ooni.nettest import NetTestCase -from ooni.utils import log, Storage -from ooni.utils.net import PermissionsError, IfaceError - -try: - from scapy.all import sr1, IP, ICMP ## XXX v4/v6? - from ooni.lib import txscapy - from ooni.lib.txscapy import txsr, txsend - from ooni.templates.scapyt import ScapyTest -except: - log.msg("This test requires scapy, see www.secdev.org/projects/scapy") - -## xxx TODO: move these to a utility function for determining OSes -LINUX=sys.platform.startswith("linux") -OPENBSD=sys.platform.startswith("openbsd") -FREEBSD=sys.platform.startswith("freebsd") -NETBSD=sys.platform.startswith("netbsd") -DARWIN=sys.platform.startswith("darwin") -SOLARIS=sys.platform.startswith("sunos") -WINDOWS=sys.platform.startswith("win32") - -class EchoTest(ScapyTest): - """ - xxx fill me in - """ - name = 'echo' - author = 'Isis Lovecruft isis@torproject.org' - description = 'A simple ICMP-8 test to see if a host is reachable.' - version = '0.0.1' - inputFile = ['file', 'f', None, 'File of list of IPs to ping'] - requirements = None - report = Storage() - - optParameters = [ - ['interface', 'i', None, 'Network interface to use'], - ['count', 'c', 5, 'Number of packets to send', int], - ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], - ['ttl', 'l', 25, 'Set the IP Time to Live', int], - ['timeout', 't', 2, 'Seconds until timeout if no response', int], - ['pcap', 'p', None, 'Save pcap to this file'], - ['receive', 'r', True, 'Receive response packets'] - ] - - def setUpClass(self, *a, **kw): - ''' - :ivar ifaces: - Struct returned from getifaddrs(3) and turned into a tuple in the - form (*ifa_name, AF_FAMILY, *ifa_addr) - ''' - super(EchoTest, self).__init__(*a, **kw) - - ## allow subclasses which register/implement external classes - ## to define their own reactor without overrides: - if not hasattr(super(EchoTest, self), 'reactor'): - log.debug("%s test: Didn't find reactor!" % self.name) - self.reactor = reactor - - if self.localOptions: - log.debug("%s localOptions found" % self.name) - log.debug("%s test options: %s" % (self.name, self.subOptions)) - self.local_options = self.localOptions.parseOptions(self.subOptions) - for key, value in self.local_options: - log.debug("Set attribute %s[%s] = %s" % (self.name, key, value)) - setattr(self, key, value) - - ## xxx is this now .subOptions? - #self.inputFile = self.localOptions['file'] - self.timeout *= 1000 ## convert to milliseconds - - if not self.interface: - log.msg("No network interface specified!") - log.debug("OS detected: %s" % sys.platform) - if LINUX or OPENBSD or NETBSD or FREEBSD or DARWIN or SOLARIS: - from twisted.internet.test import _posixifaces - log.msg("Attempting to discover network interfaces...") - ifaces = _posixifaces._interfaces() - elif WINDOWS: - from twisted.internet.test import _win32ifaces - log.msg("Attempting to discover network interfaces...") - ifaces = _win32ifaces._interfaces() - else: - log.debug("Client OS %s not accounted for!" % sys.platform) - log.debug("Unable to discover network interfaces...") - ifaces = [('lo', '')] - - ## found = {'eth0': '1.1.1.1'} - found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] - log.info("Found interfaces:\n%s" % pprint(found)) - self.interfaces = self.tryInterfaces(found) - else: - ## xxx need a way to check that iface exists, is up, and - ## we have permissions on it - log.debug("Our interface has been set to %s" % self.interface) - - if self.pcap: - try: - self.pcapfile = open(self.pcap, 'a+') - except: - log.msg("Unable to write to pcap file %s" % self.pcap) - self.pcapfile = None - - try: - assert os.path.isfile(self.file) - fp = open(self.file, 'r') - except Exception, e: - hosts = ['8.8.8.8', '38.229.72.14'] - log.err(e) - else: - self.inputs = self.inputProcessor(fp) - self.removePorts(hosts) - - log.debug("Initialization of %s test completed with:\n%s" - % (self.name, ''.join(self.__dict__))) - - @staticmethod - def inputParser(inputs): - log.debug("Removing possible ports from host addresses...") - log.debug("Initial inputs:\n%s" % pprint(inputs)) - - assert isinstance(inputs, list) - hosts = [h.rsplit(':', 1)[0] for h in inputs] - log.debug("Inputs converted to:\n%s" % hosts) - - return hosts - - def tryInterfaces(self, ifaces): - try: - from scapy.all import sr1 ## we want this check to be blocking - except: - log.msg("This test requires scapy: www.secdev.org/projects/scapy") - raise SystemExit - - ifup = {} - while ifaces: - for ifname, ifaddr in ifaces: - log.debug("Currently testing network capabilities of interface" - + "%s by sending a packet to our address %s" - % (ifname, ifaddr)) - try: - pkt = IP(dst=ifaddr)/ICMP() - ans, unans = sr(pkt, iface=ifname, timeout=self.timeout) - except Exception, e: - raise PermissionsError if e.find("Errno 1") else log.err(e) - else: - ## xxx i think this logic might be wrong - log.debug("Interface test packet\n%s\n\n%s" - % (pkt.summary(), pkt.show2())) - if ans.summary(): - log.info("Received answer for test packet on interface" - +"%s :\n%s" % (ifname, ans.summary())) - ifup.update(ifname, ifaddr) - else: - log.info("Our interface test packet was unanswered:\n%s" - % unans.summary()) - - if len(ifup) > 0: - log.msg("Discovered the following working network interfaces: %s" - % ifup) - return ifup - else: - raise IfaceError("Could not find a working network interface.") - - def buildPackets(self): - log.debug("self.input is %s" % self.input) - log.debug("self.hosts is %s" % self.hosts) - for addr in self.input: - packet = IP(dst=self.input)/ICMP() - self.request.append(packet) - return packet - - def test_icmp(self): - if self.recieve: - self.buildPackets() - all = [] - for packet in self.request: - d = self.sendReceivePackets(packets=packet) - all.append(d) - self.response.update({packet: d}) - d_list = defer.DeferredList(all) - return d_list - else: - d = self.sendPackets() - return d diff --git a/ooni/bridget/tests/tls-handshake.py b/ooni/bridget/tests/tls-handshake.py deleted file mode 100644 index eba950e..0000000 --- a/ooni/bridget/tests/tls-handshake.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -import subprocess -from subprocess import PIPE -serverport = "129.21.124.215:443" -# a subset of those from firefox -ciphers = [ - "ECDHE-ECDSA-AES256-SHA", - "ECDHE-RSA-AES256-SHA", - "DHE-RSA-CAMELLIA256-SHA", - "DHE-DSS-CAMELLIA256-SHA", - "DHE-RSA-AES256-SHA", - "DHE-DSS-AES256-SHA", - "ECDH-ECDSA-AES256-CBC-SHA", - "ECDH-RSA-AES256-CBC-SHA", - "CAMELLIA256-SHA", - "AES256-SHA", - "ECDHE-ECDSA-RC4-SHA", - "ECDHE-ECDSA-AES128-SHA", - "ECDHE-RSA-RC4-SHA", - "ECDHE-RSA-AES128-SHA", - "DHE-RSA-CAMELLIA128-SHA", - "DHE-DSS-CAMELLIA128-SHA" -] -def checkBridgeConnection(host, port) - cipher_arg = ":".join(ciphers) - cmd = ["openssl", "s_client", "-connect", "%s:%s" % (host,port)] - cmd += ["-cipher", cipher_arg] - proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE,stdin=PIPE) - out, error = proc.communicate() - success = "Cipher is DHE-RSA-AES256-SHA" in out - return success diff --git a/ooni/bridget/utils/__init__.py b/ooni/bridget/utils/__init__.py deleted file mode 100644 index 92893d6..0000000 --- a/ooni/bridget/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -all = ['inputs', 'log', 'onion', 'tests', 'interface', 'nodes', 'reports', 'work'] diff --git a/ooni/bridget/utils/inputs.py b/ooni/bridget/utils/inputs.py deleted file mode 100644 index fe058cc..0000000 --- a/ooni/bridget/utils/inputs.py +++ /dev/null @@ -1,174 +0,0 @@ -#-*- coding: utf-8 -*- -# -# inputs.py -# ********* -# -# "...quis custodiet ipsos custodes?" -# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.) -# -# :copyright: (c) 2012 Isis Lovecruft -# :license: see LICENSE for more details. -# :version: 0.1.0-beta -# - -#from types import FunctionType, FileType -import types - -from ooni.bridget import log -from ooni.utils import date, Storage - -class InputFile: - """ - This is a class describing a file used to store Tor bridge or relays - inputs. It is a python iterator object, allowing it to be efficiently - looped. - - This class should not be used directly, but rather its subclasses, - BridgeFile and RelayFile should be used instead. - """ - - def __init__(self, file, **kw): - """ - ## This is an InputAsset file, created because you tried to pass a - ## non-existent filename to a test. - ## - ## To use this file, place one input to be tested per line. Each - ## test takes different inputs. Lines which are commented out with - ## a '#' are not used. - """ - self.file = file - self.eof = False - self.all = Storage() - - for key, value in input_dict: - self.all[key] = value - - try: - self.handler = open(self.file, 'r') - except IOError: - with open(self.file, 'w') as explain: - for line in self.__init__.__doc__: - explain.writeline(line) - self.handler = open(self.file, 'r') - try: - assert isinstance(self.handler, file), "That's not a file!" - except AssertionError, ae: - log.err(ae) - - # def __handler__(self): - # """ - # Attempt to open InputFile.file and check that it is actually a file. - # If it's not, create it and add an explaination for how InputFile files - # should be used. - - # :return: - # A :type:`file` which has been opened in read-only mode. - # """ - # try: - # handler = open(self.file, 'r') - # except IOError, ioerror: ## not the hacker <(A)3 - # log.err(ioerror) - # explanation = ( - # with open(self.file, 'w') as explain: - # for line in explanation: - # explain.writeline(line) - # handler = open(self.file, 'r') - # try: - # assert isinstance(handler, file), "That's not a file!" - # except AssertionError, ae: - # log.err(ae) - # else: - # return handler - - def __iter__(next, StopIteration): - """ - Returns the next input from the file. - """ - #return self.next() - return self - - def len(self): - """ - Returns the number of the lines in the InputFile. - """ - with open(self.file, 'r') as input_file: - lines = input_file.readlines() - for number, line in enumerate(lines): - self.input_dict[number] = line - return number + 1 - - def next(self): - try: - return self.next_input() - except: - raise StopIteration - - def next_input(self): - """ - Return the next input. - """ - line = self.handler.readline() - if line: - parsed_line = self.parse_line(line) - if parsed_line: - return parsed_line - else: - self.fh.seek(0) - raise StopIteration - - def default_parser(self, line): - """ - xxx fill me in - """ - if not line.startswith('#'): - return line.replace('\n', '') - else: - return False - - def parse_line(self, line): - """ - Override this method if you need line by line parsing of an Asset. - - The default parsing action is to ignore lines which are commented out - with a '#', and to strip the newline character from the end of the - line. - - If the line was commented out return an empty string instead. - - If a subclass Foo incorporates another class Bar, when Bar is not - also a subclass of InputFile, and Bar.parse_line() exists, then - do not overwrite Bar's parse_line method. - """ - assert not hasattr(super(InputFile, self), 'parse_line') - - if self.parser is None: - if not line.startswith('#'): - return line.replace('\n', '') - else: - return '' - else: - try: - assert isinstance(self.parser, FunctionType),"Not a function!" - except AssertionError, ae: - log.err(ae) - else: - return self.parser(line) - -class BridgeFile(InputFile): - """ - xxx fill me in - """ - def __init__(self, **kw): - super(BridgeFile, self).__init__(**kw) - -class MissingInputException(Exception): - """ - - Raised when an :class:`InputFile` necessary for running the Test is - missing. - - """ - def __init__(self, error_message): - print error_message - import sys - return sys.exit() diff --git a/ooni/bridget/utils/interface.py b/ooni/bridget/utils/interface.py deleted file mode 100644 index aa55436..0000000 --- a/ooni/bridget/utils/interface.py +++ /dev/null @@ -1,54 +0,0 @@ -from zope.interface import implements, Interface, Attribute - -class ITest(Interface): - """ - This interface represents an OONI test. It fires a deferred on completion. - """ - - shortName = Attribute("""A short user facing description for this test""") - description = Attribute("""A string containing a longer description for the test""") - - requirements = Attribute("""What is required to run this this test, for example raw socket access or UDP or TCP""") - - options = Attribute("""These are the arguments to be passed to the test for it's execution""") - - blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""") - - def control(experiment_result, args): - """ - @param experiment_result: The result returned by the experiment method. - - @param args: the keys of this dict are the names of the assets passed in - from load_assets. The value is one item of the asset. - - Must return a dict containing what should be written to the report. - Anything returned by control ends up inside of the YAMLOONI report. - """ - - def experiment(args): - """ - Perform all the operations that are necessary to running a test. - - @param args: the keys of this dict are the names of the assets passed in - from load_assets. The value is one item of the asset. - - Must return a dict containing the values to be passed to control. - """ - - def load_assets(): - """ - Load the assets that should be passed to the Test. These are the inputs - to the OONI test. - Must return a dict that has as keys the asset names and values the - asset contents. - If the test does not have any assets it should return an empty dict. - """ - - def end(): - """ - This can be called at any time to terminate the execution of all of - these test instances. - - What this means is that no more test instances with new parameters will - be created. A report will be written. - """ diff --git a/ooni/bridget/utils/log.py b/ooni/bridget/utils/log.py deleted file mode 100644 index eef50d8..0000000 --- a/ooni/bridget/utils/log.py +++ /dev/null @@ -1,98 +0,0 @@ -""" -OONI logging facility. -""" -from sys import stderr, stdout - -from twisted.python import log, util -from twisted.python.failure import Failure - -def _get_log_level(level): - english = ['debug', 'info', 'warn', 'err', 'crit'] - - levels = dict(zip(range(len(english)), english)) - number = dict(zip(english, range(len(english)))) - - if not level: - return number['info'] - else: - ve = "Unknown log level: %s\n" % level - ve += "Allowed levels: %s\n" % [word for word in english] - - if type(level) is int: - if 0 <= level <= 4: - return level - elif type(level) is str: - if number.has_key(level.lower()): - return number[level] - else: - raise ValueError, ve - else: - raise ValueError, ve - -class OONITestFailure(Failure): - """ - For handling Exceptions asynchronously. - - Can be given an Exception as an argument, else will use the - most recent Exception from the current stack frame. - """ - def __init__(self, exception=None, _type=None, - _traceback=None, _capture=False): - Failure.__init__(self, exc_type=_type, - exc_tb=_traceback, captureVars=_capture) - -class OONILogObserver(log.FileLogObserver): - """ - Supports logging level verbosity. - """ - def __init__(self, logfile, verb=None): - log.FileLogObserver.__init__(self, logfile) - self.level = _get_log_level(verb) if verb is not None else 1 - assert type(self.level) is int - - def emit(self, eventDict): - if 'logLevel' in eventDict: - msgLvl = _get_log_level(eventDict['logLevel']) - assert type(msgLvl) is int - ## only log our level and higher - if self.level <= msgLvl: - text = log.textFromEventDict(eventDict) - else: - text = None - else: - text = log.textFromEventDict(eventDict) - - if text is None: - return - - timeStr = self.formatTime(eventDict['time']) - fmtDict = {'system': eventDict['system'], - 'text': text.replace('\n','\n\t')} - msgStr = log._safeFormat("[%(system)s] %(text)s\n", fmtDict) - - util.untilConcludes(self.write, timeStr + " " + msgStr) - util.untilConcludes(self.flush) - -def start(logfile=None, verbosity=None): - if log.defaultObserver: - verbosity = _get_log_level(verbosity) - - ## Always log to file, keep level at info - file = open(logfile, 'a') if logfile else stderr - OONILogObserver(file, "info").start() - - log.msg("Starting OONI...") - -def debug(message, level="debug", **kw): - print "[%s] %s" % (level, message) - ## If we want debug messages in the logfile: - #log.msg(message, logLevel=level, **kw) - -def msg(message, level="info", **kw): - log.msg(message, logLevel=level, **kw) - -def err(message, level="err", **kw): - log.err(logLevel=level, **kw) - -def fail(message, exception, level="crit", **kw): - log.failure(message, OONITestFailure(exception, **kw), logLevel=level) diff --git a/ooni/bridget/utils/nodes.py b/ooni/bridget/utils/nodes.py deleted file mode 100644 index 155f183..0000000 --- a/ooni/bridget/utils/nodes.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -""" - nodes - ***** - - This contains all the code related to Nodes - both network and code execution. - - :copyright: (c) 2012 by Arturo Filastò, Isis Lovecruft - :license: see LICENSE for more details. - -""" - -import os -from binascii import hexlify - -try: - import paramiko -except: - print "Error: module paramiko is not installed." -from pprint import pprint -import sys -import socks -import xmlrpclib - -class Node(object): - def __init__(self, address, port): - self.address = address - self.port = port - -class LocalNode(object): - def __init__(self): - pass - -""" -[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5") -[]: node_socket = node.wrap_socket() -""" -class NetworkNode(Node): - def __init__(self, address, port, node_type="SOCKS5", auth_creds=None): - self.node = Node(address,port) - - # XXX support for multiple types - # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...) - self.node_type = node_type - # type-specific authentication credentials - self.auth_creds = auth_creds - - def _get_socksipy_socket(self, proxy_type, auth_creds): - import socks - s = socks.socksocket() - # auth_creds[0] -> username - # auth_creds[1] -> password - s.setproxy(proxy_type, self.node.address, self.node.port, - self.auth_creds[0], self.auth_creds[1]) - return s - - def _get_socket_wrapper(self): - if (self.node_type.startswith("SOCKS")): # SOCKS proxies - if (self.node_type != "SOCKS5"): - proxy_type = socks.PROXY_TYPE_SOCKS5 - elif (self.node_type != "SOCKS4"): - proxy_type = socks.PROXY_TYPE_SOCKS4 - else: - print "We don't know this proxy type." - sys.exit(1) - - return self._get_socksipy_socket(proxy_type) - elif (self.node_type == "HTTP"): # HTTP proxies - return self._get_socksipy_socket(PROXY_TYPE_HTTP) - else: # Unknown proxies - print "We don't know this proxy type." - sys.exit(1) - - def wrap_socket(self): - return self._get_socket_wrapper() - -class CodeExecNode(Node): - def __init__(self, address, port, node_type, auth_creds): - self.node = Node(address,port) - - # node type (SSH proxy, etc.) - self.node_type = node_type - # type-specific authentication credentials - self.auth_creds = auth_creds - - def add_unit(self): - pass - - def get_status(self): - pass - -class PlanetLab(CodeExecNode): - def __init__(self, address, auth_creds, ooni): - self.auth_creds = auth_creds - - self.config = ooni.utils.config - self.logger = ooni.logger - self.name = "PlanetLab" - - def _api_auth(self): - api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/') - auth = {} - ## should be changed to separate node.conf file - auth['Username'] = self.config.main.pl_username - auth['AuthString'] = self.config.main.pl_password - auth['AuthMethod'] = "password" - authorized = api_server.AuthCheck(auth) - - if authorized: - print 'We are authorized!' - return auth - else: - print 'Authorization failed. Please check your settings for pl_username and pl_password in the ooni-probe.conf file.' - - def _search_for_nodes(self, node_filter=None): - api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True) - node_filter = {'hostname': '*.cert.org.cn'} - return_fields = ['hostname', 'site_id'] - all_nodes = api_server.GetNodes(self.api_auth(), node_filter, boot_state_filter) - pprint(all_nodes) - return all_nodes - - def _add_nodes_to_slice(self): - api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True) - all_nodes = self.search_for_nodes() - for node in all_nodes: - api_server.AddNode(self.api_auth(), node['site_id'], all_nodes) - print 'Adding nodes %s' % node['hostname'] - - def _auth_login(slicename, machinename): - """Attempt to authenticate to the given PL node, slicename and - machinename, using any of the private keys in ~/.ssh/ """ - - agent = paramiko.Agent() - agent_keys = agent.get_keys() - if len(agent_keys) == 0: - return - - for key in agent_keys: - print 'Trying ssh-agent key %s' % hexlify(key.get_fingerprint()), - try: - paramiko.transport.auth_publickey(machinename, slicename) - print 'Public key authentication to PlanetLab node %s successful.' % machinename, - return - except paramiko.SSHException: - print 'Public key authentication to PlanetLab node %s failed.' % machinename, - - def _get_command(): - pass - - def ssh_and_run_(slicename, machinename, command): - """Attempt to make a standard OpenSSH client to PL node, and run - commands from a .conf file.""" - - ## needs a way to specify 'ssh -l <slicename> <machinename>' - ## with public key authentication. - - command = PlanetLab.get_command() - - client = paramiko.SSHClient() - client.load_system_host_keys() - client.connect(machinename) - - stdin, stdout, stderr = client.exec_command(command) - - def send_files_to_node(directory, files): - """Attempt to rsync a tree to the PL node.""" - pass - - def add_unit(): - pass - - def get_status(): - pass diff --git a/ooni/bridget/utils/onion.py b/ooni/bridget/utils/onion.py deleted file mode 100644 index 9d4cae7..0000000 --- a/ooni/bridget/utils/onion.py +++ /dev/null @@ -1,686 +0,0 @@ -# -# onion.py -# ---------- -# Utilities for working with Tor. -# -# This code is largely taken from txtorcon and its documentation, and as such -# any and all credit should go to Meejah. Minor adjustments have been made to -# use OONI's logging system, and to build custom circuits without actually -# attaching streams. -# -# :author: Meejah, Isis Lovecruft -# :license: see included LICENSE file -# :copyright: copyright (c) 2012 The Tor Project, Inc. -# :version: 0.1.0-alpha -# -# XXX TODO add report keys for onion methods - -import random -import sys - -from twisted.internet import defer -from zope.interface import implements - -from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher -from ooni.lib.txtorcon import TorState, TorConfig -from ooni.utils import log -from ooni.utils.timer import deferred_timeout, TimeoutError - -def parse_data_dir(data_dir): - """ - Parse a string that a has been given as a DataDirectory and determine - its absolute path on the filesystem. - - :param data_dir: - A directory for Tor's DataDirectory, to be parsed. - :return: - The absolute path of :param:data_dir. - """ - from os import path, getcwd - import sys - - try: - assert isinstance(data_dir, str), \ - "Parameter type(data_dir) must be str" - except AssertionError, ae: - log.err(ae) - - if data_dir.startswith('~'): - data_dir = path.expanduser(data_dir) - elif data_dir.startswith('/'): - data_dir = path.join(getcwd(), data_dir) - elif data_dir.startswith('./'): - data_dir = path.abspath(data_dir) - else: - data_dir = path.join(getcwd(), data_dir) - - try: - assert path.isdir(data_dir), "Could not find %s" % data_dir - except AssertionError, ae: - log.err(ae) - sys.exit() - else: - return data_dir - -def write_torrc(conf, data_dir=None): - """ - Create a torrc in our data_dir. If we don't yet have a data_dir, create a - temporary one. Any temporary files or folders are added to delete_list. - - :param conf: - A :class:`ooni.lib.txtorcon.TorConfig` object, with all configuration - values saved. - :param data_dir: - The Tor DataDirectory to use. - :return: torrc, data_dir, delete_list - """ - try: - from os import write, close - from tempfile import mkstemp, mkdtemp - except ImportError, ie: - log.err(ie) - - delete_list = [] - - if data_dir is None: - data_dir = mkdtemp(prefix='bridget-tordata') - delete_list.append(data_dir) - conf.DataDirectory = data_dir - - (fd, torrc) = mkstemp(dir=data_dir) - delete_list.append(torrc) - write(fd, conf.create_torrc()) - close(fd) - - return torrc, data_dir, delete_list - -def delete_files_or_dirs(delete_list): - """ - Given a list of files or directories to delete, delete all and suppress - all errors. - - :param delete_list: - A list of files or directories to delete. - """ - try: - from os import unlink - from shutil import rmtree - except ImportError, ie: - log.err(ie) - - for temp in delete_list: - try: - unlink(temp) - except OSError: - rmtree(temp, ignore_errors=True) - -def remove_node_from_list(node, list): - for item in list: ## bridges don't match completely - if item.startswith(node): ## due to the :<port>. - try: - log.msg("Removing %s because it is a public relay" % node) - list.remove(item) - except ValueError, ve: - log.err(ve) - -def remove_public_relays(state, bridges): - """ - Remove bridges from our bridge list which are also listed as public - relays. This must be called after Tor has fully bootstrapped and we have a - :class:`ooni.lib.txtorcon.TorState` with the - :attr:`ooni.lib.txtorcon.TorState.routers` attribute assigned. - - XXX Does state.router.values() have all of the relays in the consensus, or - just the ones we know about so far? - - XXX FIXME: There is a problem in that Tor needs a Bridge line to already be - configured in order to bootstrap. However, after bootstrapping, we grab the - microdescriptors of all the relays and check if any of our bridges are - listed as public relays. Because of this, the first bridge does not get - checked for being a relay. - """ - IPs = map(lambda addr: addr.split(':',1)[0], bridges['all']) - both = set(state.routers.values()).intersection(IPs) - - if len(both) > 0: - try: - updated = map(lambda node: remove_node_from_list(node), both) - log.debug("Bridges in both: %s" % both) - log.debug("Updated = %s" % updated) - #if not updated: - # defer.returnValue(state) - #else: - # defer.returnValue(state) - return state - except Exception, e: - log.err("Removing public relays %s from bridge list failed:\n%s" - % (both, e)) - -def setup_done(proto): - log.msg("Setup Complete") - state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallback(state_complete) - state.post_bootstrap.addErrback(setup_fail) - -def setup_fail(proto): - log.msg("Setup Failed:\n%s" % proto) - return proto - #reactor.stop() - -def state_complete(state): - """Called when we've got a TorState.""" - log.msg("We've completely booted up a Tor version %s at PID %d" - % (state.protocol.version, state.tor_pid)) - log.msg("This Tor has the following %d Circuits:" - % len(state.circuits)) - for circ in state.circuits.values(): - log.msg("%s" % circ) - return state - -def updates(_progress, _tag, _summary): - """Log updates on the Tor bootstrapping process.""" - log.msg("%d%%: %s" % (_progress, _summary)) - -def bootstrap(ctrl): - """ - Bootstrap Tor from an instance of - :class:`ooni.lib.txtorcon.TorControlProtocol`. - """ - conf = TorConfig(ctrl) - conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) - log.msg("Tor process connected, bootstrapping ...") - -def start_tor(reactor, config, control_port, tor_binary, data_dir, - report=None, progress=updates, - process_cb=None, process_eb=None): - """ - Use a txtorcon.TorConfig() instance, config, to write a torrc to a - tempfile in our DataDirectory, data_dir. If data_dir is None, a temp - directory will be created. Finally, create a TCP4ClientEndpoint at our - control_port, and connect it to our reactor and a spawned Tor - process. Compare with :meth:`txtorcon.launch_tor` for differences. - - :param reactor: - An instance of class:`twisted.internet.reactor`. - :param config: - An instance of class:`txtorcon.TorConfig` with all torrc options - already configured. ivar:`config.ControlPort`, - ivar:`config.SocksPort`, ivar:`config.CookieAuthentication`, should - already be set, as well as ivar:`config.UseBridges` and - ivar:`config.Bridge` if bridges are to be used. - ivar:`txtorcon.DataDirectory` does not need to be set. - :param control_port: - The port number to use for Tor's ControlPort. - :param tor_binary: - The full path to the Tor binary to use. - :param data_dir: - The directory to use as Tor's DataDirectory. - :param report: - The class:`ooni.plugoo.reports.Report` instance. - :param progress: - A non-blocking function to handle bootstrapping updates, which takes - three parameters: _progress, _tag, and _summary. - :param process_cb: - The function to callback to after - class:`ooni.lib.txtorcon.TorProcessProtocol` returns with the fully - bootstrapped Tor process. - :param process_eb: - The function to errback to if - class:`ooni.lib.txtorcon.TorProcessProtocol` fails. - :return: - The result of the callback of a - class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a - class:`txtorcon.TorControlProtocol` as .protocol. - """ - try: - from functools import partial - from twisted.internet.endpoints import TCP4ClientEndpoint - from ooni.lib.txtorcon import TorProtocolFactory - from ooni.lib.txtorcon import TorProcessProtocol - except ImportError, ie: - log.err(ie) - - ## TODO: add option to specify an already existing torrc, which - ## will require prior parsing to enforce necessary lines - (torrc, data_dir, to_delete) = write_torrc(config, data_dir) - - log.msg("Starting Tor ...") - log.msg("Using the following as our torrc:\n%s" % config.create_torrc()) - if report is None: - report = {'torrc': config.create_torrc()} - else: - report.update({'torrc': config.create_torrc()}) - - end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port) - connection_creator = partial(end_point.connect, TorProtocolFactory()) - process_protocol = TorProcessProtocol(connection_creator, progress) - process_protocol.to_delete = to_delete - - if process_cb is not None and process_eb is not None: - process_protocol.connected_cb.addCallbacks(process_cb, process_eb) - - reactor.addSystemEventTrigger('before', 'shutdown', - partial(delete_files_or_dirs, to_delete)) - try: - transport = reactor.spawnProcess(process_protocol, - tor_binary, - args=(tor_binary,'-f',torrc), - env={'HOME': data_dir}, - path=data_dir) - transport.closeStdin() - except RuntimeError, e: - log.err("Starting Tor failed:") - process_protocol.connected_cb.errback(e) - except NotImplementedError, e: - url = "http://starship.python.net/crew/mhammond/win32/Downloads.html" - log.msg("Running bridget on Windows requires pywin32: %s" % url) - process_protocol.connected_cb.errback(e) - - return process_protocol.connected_cb - -@defer.inlineCallbacks -def start_tor_filter_nodes(reactor, config, control_port, tor_binary, - data_dir, bridges): - """ - Bootstrap a Tor process and return a fully-setup - :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges - to test in the list of known public relays, - :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges - which are known public relays. - - :param reactor: - The :class:`twisted.internet.reactor`. - :param config: - An instance of :class:`ooni.lib.txtorcon.TorConfig`. - :param control_port: - The port to use for Tor's ControlPort. If already configured in - the TorConfig instance, this can be given as - TorConfig.config.ControlPort. - :param tor_binary: - The full path to the Tor binary to execute. - :param data_dir: - The full path to the directory to use as Tor's DataDirectory. - :param bridges: - A dictionary which has a key 'all' which is a list of bridges to - test connecting to, e.g.: - bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001'] - :return: - A fully initialized :class:`ooni.lib.txtorcon.TorState`. - """ - setup = yield start_tor(reactor, config, control_port, - tor_binary, data_dir, - process_cb=setup_done, process_eb=setup_fail) - filter_nodes = yield remove_public_relays(setup, bridges) - defer.returnValue(filter_nodes) - -@defer.inlineCallbacks -def start_tor_with_timer(reactor, config, control_port, tor_binary, data_dir, - bridges, timeout): - """ - Start bootstrapping a Tor process wrapped with an instance of the class - decorator :func:`ooni.utils.timer.deferred_timeout` and complete callbacks - to either :func:`setup_done` or :func:`setup_fail`. Return a fully-setup - :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges to test - in the list of known public relays, - :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges which - are listed as known public relays. - - :param reactor: - The :class:`twisted.internet.reactor`. - :param config: - An instance of :class:`ooni.lib.txtorcon.TorConfig`. - :param control_port: - The port to use for Tor's ControlPort. If already configured in - the TorConfig instance, this can be given as - TorConfig.config.ControlPort. - :param tor_binary: - The full path to the Tor binary to execute. - :param data_dir: - The full path to the directory to use as Tor's DataDirectory. - :param bridges: - A dictionary which has a key 'all' which is a list of bridges to - test connecting to, e.g.: - bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001'] - :param timeout: - The number of seconds to attempt to bootstrap the Tor process before - raising a :class:`ooni.utils.timer.TimeoutError`. - :return: - If the timeout limit is not exceeded, return a fully initialized - :class:`ooni.lib.txtorcon.TorState`, else return None. - """ - error_msg = "Bootstrapping has exceeded the timeout limit..." - with_timeout = deferred_timeout(timeout, e=error_msg)(start_tor) - try: - setup = yield with_timeout(reactor, config, control_port, tor_binary, - data_dir, process_cb=setup_done, - process_eb=setup_fail) - except TimeoutError, te: - log.err(te) - defer.returnValue(None) - #except Exception, e: - # log.err(e) - # defer.returnValue(None) - else: - state = yield remove_public_relays(setup, bridges) - defer.returnValue(state) - -@defer.inlineCallbacks -def start_tor_filter_nodes_with_timer(reactor, config, control_port, - tor_binary, data_dir, bridges, timeout): - """ - Start bootstrapping a Tor process wrapped with an instance of the class - decorator :func:`ooni.utils.timer.deferred_timeout` and complete callbacks - to either :func:`setup_done` or :func:`setup_fail`. Then, filter our list - of bridges to remove known public relays by calling back to - :func:`remove_public_relays`. Return a fully-setup - :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges to test - in the list of known public relays, - :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges which - are listed as known public relays. - - :param reactor: - The :class:`twisted.internet.reactor`. - :param config: - An instance of :class:`ooni.lib.txtorcon.TorConfig`. - :param control_port: - The port to use for Tor's ControlPort. If already configured in - the TorConfig instance, this can be given as - TorConfig.config.ControlPort. - :param tor_binary: - The full path to the Tor binary to execute. - :param data_dir: - The full path to the directory to use as Tor's DataDirectory. - :param bridges: - A dictionary which has a key 'all' which is a list of bridges to - test connecting to, e.g.: - bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001'] - :param timeout: - The number of seconds to attempt to bootstrap the Tor process before - raising a :class:`ooni.utils.timer.TimeoutError`. - :return: - If the timeout limit is not exceeded, return a fully initialized - :class:`ooni.lib.txtorcon.TorState`, else return None. - """ - error_msg = "Bootstrapping has exceeded the timeout limit..." - with_timeout = deferred_timeout(timeout, e=error_msg)(start_tor_filter_nodes) - try: - state = yield with_timeout(reactor, config, control_port, - tor_binary, data_dir, bridges) - except TimeoutError, te: - log.err(te) - defer.returnValue(None) - #except Exception, e: - # log.err(e) - # defer.returnValue(None) - else: - defer.returnValue(state) - -class CustomCircuit(CircuitListenerMixin): - """ - Utility class for controlling circuit building. See - 'attach_streams_by_country.py' in the txtorcon documentation. - - :param state: - A fully bootstrapped instance of :class:`ooni.lib.txtorcon.TorState`. - :param relays: - A dictionary containing a key 'all', which is a list of relays to - test connecting to. - :ivar waiting_circuits: - The list of circuits which we are waiting to attach to. You shouldn't - need to touch this. - """ - implements(IStreamAttacher) - - def __init__(self, state, relays=None): - self.state = state - self.waiting_circuits = [] - self.relays = relays - - def waiting_on(self, circuit): - """ - Whether or not we are waiting on the given circuit before attaching to - it. - - :param circuit: - An item from :ivar:`ooni.lib.txtorcon.TorState.circuits`. - :return: - True if we are waiting on the circuit, False if not waiting. - """ - for (circid, d) in self.waiting_circuits: - if circuit.id == circid: - return True - return False - - def circuit_extend(self, circuit, router): - "ICircuitListener" - if circuit.purpose != 'GENERAL': - return - if self.waiting_on(circuit): - log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) - - def circuit_built(self, circuit): - "ICircuitListener" - if circuit.purpose != 'GENERAL': - return - log.msg("Circuit %s built ..." % circuit.id) - log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) - for (circid, d) in self.waiting_circuits: - if circid == circuit.id: - self.waiting_circuits.remove((circid, d)) - d.callback(circuit) - - def circuit_failed(self, circuit, reason): - """ - If building a circuit has failed, try to remove it from our list of - :ivar:`waiting_circuits`, else request to build it. - - :param circuit: - An item from :ivar:`ooni.lib.txtorcon.TorState.circuits`. - :param reason: - A :class:`twisted.python.fail.Failure` instance. - :return: - None - """ - if self.waiting_on(circuit): - log.msg("Circuit %s failed for reason %s" % (circuit.id, reason)) - circid, d = None, None - for c in self.waiting_circuits: - if c[0] == circuit.id: - circid, d = c - if d is None: - raise Exception("Expected to find circuit.") - - self.waiting_circuits.remove((circid, d)) - log.msg("Trying to build a circuit for %s" % circid) - self.request_circuit_build(d) - - def check_circuit_route(self, router): - """ - Check if a relay is a hop in one of our already built circuits. - - :param router: - An item from the list - :func:`ooni.lib.txtorcon.TorState.routers.values()`. - """ - for circ in self.state.circuits.values(): - if router in circ.path: - #router.update() ## XXX can i use without args? no. - TorInfo.dump(self) - - def request_circuit_build(self, deferred, path=None): - """ - Request a custom circuit. - - :param deferred: - A :class:`twisted.internet.defer.Deferred` for this circuit. - :param path: - A list of router ids to build a circuit from. The length of this - list must be at least three. - """ - if path is None: - - pick = self.relays['all'].pop - n = self.state.entry_guards.values() - choose = random.choice - - first, middle, last = (None for i in range(3)) - - if self.relays['remaining']() >= 3: - first, middle, last = (pick() for i in range(3)) - elif self.relays['remaining']() < 3: - first = choose(n) - middle = pick() - if self.relays['remaining'] == 2: - middle, last = (pick() for i in range(2)) - elif self.relay['remaining'] == 1: - middle = pick() - last = choose(n) - else: - log.msg("Qu'est-que fuque?") - else: - middle, last = (random.choice(self.state.routers.values()) - for i in range(2)) - - path = [first, middle, last] - - else: - assert isinstance(path, list), \ - "Circuit path must be a list of relays!" - assert len(path) >= 3, \ - "Circuit path must be at least three hops!" - - log.msg("Requesting a circuit: %s" - % '->'.join(map(lambda node: node, path))) - - class AppendWaiting: - def __init__(self, attacher, deferred): - self.attacher = attacher - self.d = deferred - def __call__(self, circ): - """ - Return from build_circuit is a Circuit, however, - we want to wait until it is built before we can - issue an attach on it and callback to the Deferred - we issue here. - """ - log.msg("Circuit %s is in progress ..." % circ.id) - self.attacher.waiting_circuits.append((circ.id, self.d)) - - return self.state.build_circuit(path).addCallback( - AppendWaiting(self, deferred)).addErrback( - log.err) - -class TxtorconImportError(ImportError): - """ - Raised when ooni.lib.txtorcon cannot be imported from. Checks our current - working directory and the path given to see if txtorcon has been - initialized via /ooni/lib/Makefile. - """ - from os import getcwd, path - - cwd, tx = getcwd(), 'lib/txtorcon/torconfig.py' - try: - log.msg("Unable to import from ooni.lib.txtorcon") - if cwd.endswith('ooni'): - check = path.join(cwd, tx) - elif cwd.endswith('utils'): - check = path.join(cwd, '../'+tx) - else: - check = path.join(cwd, 'ooni/'+tx) - assert path.isfile(check) - except: - log.msg("Error: Some OONI libraries are missing!") - log.msg("Please go to /ooni/lib/ and do "make all"") - -class PTNoBridgesException(Exception): - """Raised when a pluggable transport is specified, but not bridges.""" - def __init__(self): - log.msg("Pluggable transport requires the bridges option") - return sys.exit() - -class PTNotFoundException(Exception): - def __init__(self, transport_type): - m = "Pluggable Transport type %s was unaccounted " % transport_type - m += "for, please contact isis(at)torproject(dot)org and it will " - m += "get included." - log.msg("%s" % m) - return sys.exit() - -@defer.inlineCallbacks -def __start_tor_with_timer__(reactor, config, control_port, tor_binary, - data_dir, bridges=None, relays=None, timeout=None, - retry=None): - """ - A wrapper for :func:`start_tor` which wraps the bootstrapping of a Tor - process and its connection to a reactor with a - :class:`twisted.internet.defer.Deferred` class decorator utility, - :func:`ooni.utils.timer.deferred_timeout`, and a mechanism for resets. - - ## XXX fill me in - """ - raise NotImplementedError - - class RetryException(Exception): - pass - - import sys - from ooni.utils.timer import deferred_timeout, TimeoutError - - def __make_var__(old, default, _type): - if old is not None: - assert isinstance(old, _type) - new = old - else: - new = default - return new - - reactor = reactor - timeout = __make_var__(timeout, 120, int) - retry = __make_var__(retry, 1, int) - - with_timeout = deferred_timeout(timeout)(start_tor) - - @defer.inlineCallbacks - def __start_tor__(rc=reactor, cf=config, cp=control_port, tb=tor_binary, - dd=data_dir, br=bridges, rl=relays, cb=setup_done, - eb=setup_fail, af=remove_public_relays, retry=retry): - try: - setup = yield with_timeout(rc,cf,cp,tb,dd) - except TimeoutError: - retry -= 1 - defer.returnValue(retry) - else: - if setup.callback: - setup = yield cb(setup) - elif setup.errback: - setup = yield eb(setup) - else: - setup = setup - - if br is not None: - state = af(setup,br) - else: - state = setup - defer.returnValue(state) - - @defer.inlineCallbacks - def __try_until__(tries): - result = yield __start_tor__() - try: - assert isinstance(result, int) - except AssertionError: - defer.returnValue(result) - else: - if result >= 0: - tried = yield __try_until__(result) - defer.returnValue(tried) - else: - raise RetryException - try: - tried = yield __try_until__(retry) - except RetryException: - log.msg("All retry attempts to bootstrap Tor have timed out.") - log.msg("Exiting ...") - defer.returnValue(sys.exit()) - else: - defer.returnValue(tried) diff --git a/ooni/bridget/utils/reports.py b/ooni/bridget/utils/reports.py deleted file mode 100644 index ae67b13..0000000 --- a/ooni/bridget/utils/reports.py +++ /dev/null @@ -1,144 +0,0 @@ -from __future__ import with_statement - -import os -import yaml - -import itertools -from ooni.utils import log, date, net - -class Report: - """This is the ooni-probe reporting mechanism. It allows - reporting to multiple destinations and file formats. - - :scp the string of <host>:<port> of an ssh server - - :yaml the filename of a the yaml file to write - - :file the filename of a simple txt file to write - - :tcp the <host>:<port> of a TCP server that will just listen for - inbound connection and accept a stream of data (think of it - as a `nc -l -p <port> > filename.txt`) - """ - def __init__(self, testname=None, file="report.log", - scp=None, - tcp=None): - - self.testname = testname - self.file = file - self.tcp = tcp - self.scp = scp - #self.config = ooni.config.report - - #if self.config.timestamp: - # tmp = self.file.split('.') - # self.file = '.'.join(tmp[:-1]) + "-" + \ - # datetime.now().isoformat('-') + '.' + \ - # tmp[-1] - # print self.file - - self.scp = None - self.write_header() - - def write_header(self): - pretty_date = date.pretty_date() - header = "# OONI Probe Report for Test %s\n" % self.testname - header += "# %s\n\n" % pretty_date - self._write_to_report(header) - # XXX replace this with something proper - address = net.getClientAddress() - test_details = {'start_time': str(date.now()), - 'asn': address['asn'], - 'test_name': self.testname, - 'addr': address['ip']} - self(test_details) - - def _write_to_report(self, dump): - reports = [] - - if self.file: - reports.append("file") - - if self.tcp: - reports.append("tcp") - - if self.scp: - reports.append("scp") - - #XXX make this non blocking - for report in reports: - self.send_report(dump, report) - - def __call__(self, data): - """ - This should be invoked every time you wish to write some - data to the reporting system - """ - dump = yaml.dump([data]) - self._write_to_report(dump) - - def file_report(self, data): - """ - This reports to a file in YAML format - """ - with open(self.file, 'a+') as f: - f.write(data) - - def send_report(self, data, type): - """ - This sends the report using the - specified type. - """ - #print "Reporting %s to %s" % (data, type) - log.msg("Reporting to %s" % type) - getattr(self, type+"_report").__call__(data) - -class NewReport(object): - filename = 'report.log' - startTime = None - endTime = None - testName = None - ipAddr = None - asnAddr = None - - def _open(): - self.fp = open(self.filename, 'a+') - - @property - def header(): - pretty_date = date.pretty_date() - report_header = "# OONI Probe Report for Test %s\n" % self.testName - report_header += "# %s\n\n" % pretty_date - test_details = {'start_time': self.startTime, - 'asn': asnAddr, - 'test_name': self.testName, - 'addr': ipAddr} - report_header += yaml.dump([test_details]) - return report_header - - def create(): - """ - Create a new report by writing it's header. - """ - self.fp = open(self.filename, 'w+') - self.fp.write(self.header) - - def exists(): - """ - Returns False if the file does not exists. - """ - return os.path.exists(self.filename) - - def write(data): - """ - Write a report to the file. - - :data: python data structure to be written to report. - """ - if not self.exists(): - self.create() - else: - self._open() - yaml_encoded_data = yaml.dump([data]) - self.fp.write(yaml_encoded_data) - self.fp.close() diff --git a/ooni/bridget/utils/tests.py b/ooni/bridget/utils/tests.py deleted file mode 100644 index ea4be0b..0000000 --- a/ooni/bridget/utils/tests.py +++ /dev/null @@ -1,141 +0,0 @@ -import os -import yaml -from zope.interface import Interface, Attribute - -import logging -import itertools -from twisted.internet import reactor, defer, threads -## XXX why is this imported and not used? -from twisted.python import failure - -from ooni.utils import log, date -from ooni.plugoo import assets, work -from ooni.plugoo.reports import Report -from ooni.plugoo.interface import ITest - -class OONITest(object): - """ - This is the base class for writing OONI Tests. - - It should be used in conjunction with the ITest Interface. It allows the - developer to benefit from OONIs reporting system and command line argument - parsing system. - """ - name = "oonitest" - # By default we set this to False, meaning that we don't block - blocking = False - reactor = reactor - tool = False - ended = False - - def __init__(self, local_options, global_options, report, ooninet=None, - reactor=reactor): - # These are the options that are read through the tests suboptions - self.local_options = local_options - # These are the options global to all of OONI - self.global_options = global_options - self.report = report - #self.ooninet = ooninet - self.reactor = reactor - self.result = {} - self.initialize() - self.assets = self.load_assets() - - def initialize(self): - """ - Override this method if you are interested in having some extra - behavior when your test class is instantiated. - """ - pass - - def load_assets(self): - """ - This method should be overriden by the test writer to provide the - logic for loading their assets. - """ - return {} - - def __repr__(self): - return "<OONITest %s %s %s>" % (self.local_options, - self.global_options, - self.assets) - - def end(self): - """ - State that the current test should finish. - """ - self.ended = True - - def finished(self, return_value): - """ - The Test has finished running, we must now calculate the test runtime - and add all time data to the report. - """ - #self.ooninet.report(result) - self.end_time = date.now() - result = self.result - result['start_time'] = str(self.start_time) - result['end_time'] = str(self.end_time) - result['run_time'] = str(self.end_time - self.start_time) - result['return_value'] = return_value - log.msg("FINISHED %s" % result) - self.report(result) - return result - - def _do_experiment(self, args): - """ - A wrapper around the launch of experiment. - If we are running a blocking test experiment will be run in a thread if - not we expect it to return a Deferred. - - @param args: the asset line(s) that we are working on. - - returns a deferred. - """ - if self.blocking: - self.d = threads.deferToThread(self.experiment, args) - else: - self.d = self.experiment(args) - - self.d.addCallback(self.control, args) - self.d.addCallback(self.finished) - self.d.addErrback(self.finished) - return self.d - - def control(self, result, args): - """ - Run the control. - - @param result: what was returned by experiment. - - @param args: the asset(s) lines that we are working on. - """ - log.msg("Doing control") - return result - - def experiment(self, args): - """ - Run the experiment. This sample implementation returns a deferred, - making it a non-blocking test. - - @param args: the asset(s) lines that we are working on. - """ - log.msg("Doing experiment") - d = defer.Deferred() - return d - - def startTest(self, args): - """ - This method is invoked by the worker to start the test with one line of - the asset file. - - @param args: the asset(s) lines that we are working on. - """ - self.start_time = date.now() - - if self.shortName: - log.msg("Starting test %s" % self.shortName) - else: - log.msg("Starting test %s" % self.__class__) - - return self._do_experiment(args) diff --git a/ooni/bridget/utils/work.py b/ooni/bridget/utils/work.py deleted file mode 100644 index c329c20..0000000 --- a/ooni/bridget/utils/work.py +++ /dev/null @@ -1,147 +0,0 @@ -# -*- coding: UTF-8 -""" - work.py - ********** - - This contains all code related to generating - Units of Work and processing it. - - :copyright: (c) 2012 by Arturo Filastò. - :license: see LICENSE for more details. - -""" -import itertools -import yaml -from datetime import datetime - -from zope.interface import Interface, Attribute - -from twisted.python import failure -from twisted.internet import reactor, defer - -class Worker(object): - """ - This is the core of OONI. It takes as input Work Units and - runs them concurrently. - """ - def __init__(self, maxconcurrent=10, reactor=reactor): - """ - @param maxconcurrent: how many test instances should be run - concurrently. - """ - self.reactor = reactor - self.maxconcurrent = maxconcurrent - self._running = 0 - self._queued = [] - - def _run(self, r): - """ - Check if we should start another test because we are below maximum - concurrency. - - This function is called every time a test finishes running. - - @param r: the return value of a previous test. - """ - if self._running > 0: - self._running -= 1 - - if self._running < self.maxconcurrent and self._queued: - workunit, d = self._queued.pop(0) - asset, test, idx = workunit - while test.ended and workunit: - try: - workunit, d = self._queued.pop(0) - asset, test, idx = workunit - except: - workunit = None - - if not test.ended: - self._running += 1 - actuald = test.startTest(asset).addBoth(self._run) - - if isinstance(r, failure.Failure): - # XXX probably we should be doing something to retry test running - r.trap() - - if self._running == 0 and not self._queued: - self.reactor.stop() - - return r - - def push(self, workunit): - """ - Add a test to the test queue and run it if we are not maxed out on - concurrency. - - @param workunit: a tuple containing the (asset, test, idx), where asset - is the line of the asset(s) we are working on, test - is an instantiated test and idx is the index we are - currently at. - """ - if self._running < self.maxconcurrent: - asset, test, idx = workunit - if not test.ended: - self._running += 1 - return test.startTest(asset).addBoth(self._run) - - d = defer.Deferred() - self._queued.append((workunit, d)) - return d - -class WorkGenerator(object): - """ - Factory responsible for creating units of work. - - This shall be run on the machine running OONI-cli. The returned WorkUnits - can either be run locally or on a remote OONI Node or Network Node. - """ - size = 10 - - def __init__(self, test, arguments=None, start=None): - self.Test = test - - if self.Test.assets and self.Test.assets.values()[0]: - self.assetGenerator = itertools.product(*self.Test.assets.values()) - else: - self.assetGenerator = None - - self.assetNames = self.Test.assets.keys() - - self.idx = 0 - self.end = False - if start: - self.skip(start) - - def __iter__(self): - return self - - def skip(self, start): - """ - Skip the first x number of lines of the asset. - - @param start: int how many items we should skip. - """ - for j in xrange(0, start-1): - for i in xrange(0, self.size): - self.assetGenerator.next() - self.idx += 1 - - def next(self): - if self.end: - raise StopIteration - - if not self.assetGenerator: - self.end = True - return ({}, self.Test, self.idx) - - try: - asset = self.assetGenerator.next() - ret = {} - for i, v in enumerate(asset): - ret[self.assetNames[i]] = v - except StopIteration: - raise StopIteration - - self.idx += 1 - return (ret, self.Test, self.idx)