[tor-commits] [ooni-probe/master] * Moving bridget around to be better integrated with the new ooni structure.
isis at torproject.org
isis at torproject.org
Sun Nov 4 13:50:41 UTC 2012
commit 276aeea3a524018df7a4535add3e2ad83f755e33
Author: Isis Lovecruft <isis at torproject.org>
Date: Sun Nov 4 12:23:09 2012 +0000
* Moving bridget around to be better integrated with the new ooni structure.
---
ooni/bridget/__init__.py | 14 -
ooni/bridget/custodiet.py | 421 ---------------------
ooni/bridget/tests/__init__.py | 14 -
ooni/bridget/tests/bridget.py | 499 -------------------------
ooni/bridget/tests/echo.py | 205 -----------
ooni/bridget/tests/tls-handshake.py | 32 --
ooni/bridget/utils/__init__.py | 1 -
ooni/bridget/utils/inputs.py | 174 ---------
ooni/bridget/utils/interface.py | 54 ---
ooni/bridget/utils/log.py | 98 -----
ooni/bridget/utils/nodes.py | 176 ---------
ooni/bridget/utils/onion.py | 686 -----------------------------------
ooni/bridget/utils/reports.py | 144 --------
ooni/bridget/utils/tests.py | 141 -------
ooni/bridget/utils/work.py | 147 --------
15 files changed, 0 insertions(+), 2806 deletions(-)
diff --git a/ooni/bridget/__init__.py b/ooni/bridget/__init__.py
deleted file mode 100644
index 4648d77..0000000
--- a/ooni/bridget/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-#-*- coding: utf-8 -*-
-
-#import os, sys
-#import copy_reg
-
-## Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain.
-#sys.path[:] = map(os.path.abspath, sys.path)
-#sys.path.insert(0, os.path.abspath(os.getcwd()))
-
-## This is a hack to overcome a bug in python
-#from ooni.utils.hacks import patched_reduce_ex
-#copy_reg._reduce_ex = patched_reduce_ex
-
-__all__ = ['custodiet']
diff --git a/ooni/bridget/custodiet.py b/ooni/bridget/custodiet.py
deleted file mode 100755
index 8cbcfce..0000000
--- a/ooni/bridget/custodiet.py
+++ /dev/null
@@ -1,421 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8
-#
-# custodiet
-# *********
-#
-# "...quis custodiet ipsos custodes?"
-# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.)
-#
-# "'Hand me the Custodian,' Goodchild demands, inserting the waiflike
-# robot into Bambara's opened navel. 'Providing conscience for those who
-# have none.' Goodchild and the other Breen government agents disappear
-# into the surrounding desert in a vehicle, kicking up cloud of white dust.
-# Bambara awakens, and, patting the dust from his clothing, turns to
-# greet a one-armed child. 'Hi, my name's Bambara; I'm a
-# thirty-six-year-old Virgo and a former killer, who's hobbies include
-# performing recreational autopsies, defecating, and drinking rum. I've
-# recently been given a conscience, and would very much like to help you.'
-# Cut to Bambara and the child, now with one of Bambara's arms, leaving
-# a surgical clinic."
-# - AeonFlux, "The Purge" (sometime in the late 90s)
-#
-# :copyright: (c) 2012 Isis Lovecruft
-# :license: see LICENSE for more details.
-# :version: 0.1.0-beta
-#
-
-# ooniprobe.py imports
-import sys
-from signal import SIGTERM, signal
-from pprint import pprint
-
-from twisted.python import usage
-from twisted.internet import reactor
-from twisted.plugin import getPlugins
-
-from zope.interface.verify import verifyObject
-from zope.interface.exceptions import BrokenImplementation
-from zope.interface.exceptions import BrokenMethodImplementation
-
-from ooni.bridget.tests import bridget
-from ooni.bridget.utils import log, tests, work, reports
-from ooni.bridget.utils.interface import ITest
-from ooni.utils.logo import getlogo
-
-# runner.py imports
-import os
-import types
-import time
-import inspect
-import yaml
-
-from twisted.internet import defer, reactor
-from twisted.python import reflect, failure, usage
-from twisted.python import log as tlog
-
-from twisted.trial import unittest
-from twisted.trial.runner import TrialRunner, TestLoader
-from twisted.trial.runner import isPackage, isTestCase, ErrorHolder
-from twisted.trial.runner import filenameToModule, _importFromFile
-
-from ooni import nettest
-from ooni.inputunit import InputUnitFactory
-from ooni.nettest import InputTestSuite
-from ooni.plugoo import tests as oonitests
-from ooni.reporter import ReporterFactory
-from ooni.utils import log, geodata, date
-from ooni.utils.legacy import LegacyOONITest
-from ooni.utils.legacy import start_legacy_test, adapt_legacy_test
-
-
-__version__ = "0.1.0-beta"
-
-
-#def retrieve_plugoo():
-# """
-# Get all the plugins that implement the ITest interface and get the data
-# associated to them into a dict.
-# """
-# interface = ITest
-# d = {}
-# error = False
-# for p in getPlugins(interface, plugins):
-# try:
-# verifyObject(interface, p)
-# d[p.shortName] = p
-# except BrokenImplementation, bi:
-# print "Plugin Broken"
-# print bi
-# error = True
-# if error != False:
-# print "Plugin Loaded!"
-# return d
-#
-#plugoo = retrieve_plugoo()
-
-"""
-
-ai to watch over which tests to run - custodiet
-
- * runTest() or getPrefixMethodNames() to run the tests in order for each
- test (esp. the tcp and icmp parts) to be oonicompat we should use the
- test_icmp_ping API framework for those.
-
- * should handle calling
-
-tests to run:
- echo
- syn
- fin
- conn
- tls
- tor
-need fakebridge - canary
-
-"""
-
-def runTest(test, options, global_options, reactor=reactor):
- """
- Run an OONI probe test by name.
-
- @param test: a string specifying the test name as specified inside of
- shortName.
-
- @param options: the local options to be passed to the test.
-
- @param global_options: the global options for OONI
- """
- parallelism = int(global_options['parallelism'])
- worker = work.Worker(parallelism, reactor=reactor)
- test_class = plugoo[test].__class__
- report = reports.Report(test, global_options['output'])
-
- log_to_stdout = True
- if global_options['quiet']:
- log_to_stdout = False
-
- log.start(log_to_stdout,
- global_options['log'],
- global_options['verbosity'])
-
- resume = 0
- if not options:
- options = {}
- if 'resume' in options:
- resume = options['resume']
-
- test = test_class(options, global_options, report, reactor=reactor)
- if test.tool:
- test.runTool()
- return True
-
- if test.ended:
- print "Ending test"
- return None
-
- wgen = work.WorkGenerator(test,
- dict(options),
- start=resume)
- for x in wgen:
- worker.push(x)
-
-class MainOptions(usage.Options):
- tests = [bridget, ]
- subCommands = []
- for test in tests:
- print test
- testopt = getattr(test, 'options')
- subCommands.append([test, None, testopt, "Run the %s test" % test])
-
- optFlags = [
- ['quiet', 'q', "Don't log to stdout"]
- ]
-
- optParameters = [
- ['parallelism', 'n', 10, "Specify the number of parallel tests to run"],
- #['target-node', 't', 'localhost:31415', 'Select target node'],
- ['output', 'o', 'bridge.log', "Specify output report file"],
- ['reportfile', 'o', 'bridge.log', "Specify output log file"],
- ['verbosity', 'v', 1, "Specify the logging level"],
- ]
-
- def opt_version(self):
- """
- Display OONI version and exit.
- """
- print "OONI version:", __version__
- sys.exit(0)
-
- def __str__(self):
- """
- Hack to get the sweet ascii art into the help output and replace the
- strings "Commands" with "Tests".
- """
- return getlogo() + '\n' + self.getSynopsis() + '\n' + \
- self.getUsage(width=None).replace("Commands:", "Tests:")
-
-
-
-def isTestCase(thing):
- try:
- return issubclass(thing, unittest.TestCase)
- except TypeError:
- return False
-
-def isLegacyTest(obj):
- """
- Returns True if the test in question is written using the OONITest legacy
- class.
- We do this for backward compatibility of the OONIProbe API.
- """
- try:
- if issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest:
- return True
- else:
- return False
- except TypeError:
- return False
-
-def processTest(obj, config):
- """
- Process the parameters and :class:`twisted.python.usage.Options` of a
- :class:`ooni.nettest.Nettest`.
-
- :param obj:
- An uninstantiated old test, which should be a subclass of
- :class:`ooni.plugoo.tests.OONITest`.
- :param config:
- A configured and instantiated :class:`twisted.python.usage.Options`
- class.
- """
-
- inputFile = obj.inputFile
-
- if obj.optParameters or inputFile:
- if not obj.optParameters:
- obj.optParameters = []
-
- if inputFile:
- obj.optParameters.append(inputFile)
-
- class Options(usage.Options):
- optParameters = obj.optParameters
-
- options = Options()
- options.parseOptions(config['subArgs'])
- obj.localOptions = options
-
- if inputFile:
- obj.inputFile = options[inputFile[0]]
- try:
- tmp_obj = obj()
- tmp_obj.getOptions()
- except usage.UsageError:
- options.opt_help()
-
- return obj
-
-def findTestClassesFromConfig(config):
- """
- Takes as input the command line config parameters and returns the test
- case classes.
- If it detects that a certain test class is using the old OONIProbe format,
- then it will adapt it to the new testing system.
-
- :param config:
- A configured and instantiated :class:`twisted.python.usage.Options`
- class.
- :return:
- A list of class objects found in a file or module given on the
- commandline.
- """
-
- filename = config['test']
- classes = []
-
- module = filenameToModule(filename)
- for name, val in inspect.getmembers(module):
- if isTestCase(val):
- classes.append(processTest(val, config))
- elif isLegacyTest(val):
- classes.append(adapt_legacy_test(val, config))
- return classes
-
-def makeTestCases(klass, tests, methodPrefix):
- """
- Takes a class some tests and returns the test cases. methodPrefix is how
- the test case functions should be prefixed with.
- """
-
- cases = []
- for test in tests:
- cases.append(klass(methodPrefix+test))
- return cases
-
-def loadTestsAndOptions(classes, config):
- """
- Takes a list of classes and returns their testcases and options.
- Legacy tests will be adapted.
- """
-
- methodPrefix = 'test'
- suiteFactory = InputTestSuite
- options = []
- testCases = []
- names = []
-
- _old_klass_type = LegacyOONITest
-
- for klass in classes:
- if isinstance(klass, _old_klass_type):
- try:
- cases = start_legacy_test(klass)
- #cases.callback()
- if cases:
- print cases
- return [], []
- testCases.append(cases)
- except Exception, e:
- log.err(e)
- else:
- try:
- opts = klass.local_options
- options.append(opts)
- except AttributeError, ae:
- options.append([])
- log.err(ae)
- elif not isinstance(klass, _old_klass_type):
- tests = reflect.prefixedMethodNames(klass, methodPrefix)
- if tests:
- cases = makeTestCases(klass, tests, methodPrefix)
- testCases.append(cases)
- try:
- k = klass()
- opts = k.getOptions()
- options.append(opts)
- except AttributeError, ae:
- options.append([])
- log.err(ae)
- else:
- try:
- raise RuntimeError, "Class is some strange type!"
- except RuntimeError, re:
- log.err(re)
-
- return testCases, options
-
-class ORunner(object):
- """
- This is a specialized runner used by the ooniprobe command line tool.
- I am responsible for reading the inputs from the test files and splitting
- them in input units. I also create all the report instances required to run
- the tests.
- """
- def __init__(self, cases, options=None, config=None, *arg, **kw):
- self.baseSuite = InputTestSuite
- self.cases = cases
- self.options = options
-
- try:
- assert len(options) != 0, "Length of options is zero!"
- except AssertionError, ae:
- self.inputs = []
- log.err(ae)
- else:
- try:
- first = options.pop(0)
- except:
- first = {}
- if 'inputs' in first:
- self.inputs = options['inputs']
- else:
- log.msg("Could not find inputs!")
- log.msg("options[0] = %s" % first)
- self.inputs = [None]
-
- try:
- reportFile = open(config['reportfile'], 'a+')
- except:
- filename = 'report_'+date.timestamp()+'.yaml'
- reportFile = open(filename, 'a+')
- self.reporterFactory = ReporterFactory(reportFile,
- testSuite=self.baseSuite(self.cases))
-
- def runWithInputUnit(self, inputUnit):
- idx = 0
- result = self.reporterFactory.create()
-
- for inputs in inputUnit:
- result.reporterFactory = self.reporterFactory
-
- suite = self.baseSuite(self.cases)
- suite.input = inputs
- suite(result, idx)
-
- # XXX refactor all of this index bullshit to avoid having to pass
- # this index around. Probably what I want to do is go and make
- # changes to report to support the concept of having multiple runs
- # of the same test.
- # We currently need to do this addition in order to get the number
- # of times the test cases that have run inside of the test suite.
- idx += (suite._idx - idx)
-
- result.done()
-
- def run(self):
- self.reporterFactory.options = self.options
- for inputUnit in InputUnitFactory(self.inputs):
- self.runWithInputUnit(inputUnit)
-
-if __name__ == "__main__":
- config = Options()
- config.parseOptions()
-
- if not config.subCommand:
- config.opt_help()
- signal(SIGTERM)
- #sys.exit(1)
-
- runTest(config.subCommand, config.subOptions, config)
- reactor.run()
diff --git a/ooni/bridget/tests/__init__.py b/ooni/bridget/tests/__init__.py
deleted file mode 100644
index 9ecc88d..0000000
--- a/ooni/bridget/tests/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# -*- coding: UTF-8
-#
-# bridget/tests/__init__.py
-# *************************
-#
-# "...quis custodiet ipsos custodes?"
-# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.)
-#
-# :copyright: (c) 2012 Isis Lovecruft
-# :license: see LICENSE for more details.
-# :version: 0.1.0-beta
-#
-
-all = ['bridget']
diff --git a/ooni/bridget/tests/bridget.py b/ooni/bridget/tests/bridget.py
deleted file mode 100644
index a334747..0000000
--- a/ooni/bridget/tests/bridget.py
+++ /dev/null
@@ -1,499 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-#
-# +-----------+
-# | BRIDGET |
-# | +--------------------------------------------+
-# +--------| Use a Tor process to test making a Tor |
-# | connection to a list of bridges or relays. |
-# +--------------------------------------------+
-#
-# :authors: Isis Lovecruft, Arturo Filasto
-# :licence: see included LICENSE
-# :version: 0.1.0-alpha
-
-from __future__ import with_statement
-from functools import partial
-from random import randint
-
-import os
-import sys
-
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from twisted.internet import defer, error, reactor
-from zope.interface import implements
-
-from ooni.utils import log, date
-from ooni.utils.config import ValueChecker
-
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.plugoo.assets import Asset, MissingAssetException
-from ooni.utils.onion import TxtorconImportError
-from ooni.utils.onion import PTNoBridgesException, PTNotFoundException
-
-try:
- from ooni.utils.onion import parse_data_dir
-except:
- log.msg("Please go to /ooni/lib and do 'make txtorcon' to run this test!")
-
-class RandomPortException(Exception):
- """Raised when using a random port conflicts with configured ports."""
- def __init__(self):
- log.msg("Unable to use random and specific ports simultaneously")
- return sys.exit()
-
-class BridgetArgs(usage.Options):
- """Commandline options."""
- allowed = "Port to use for Tor's %s, must be between 1024 and 65535."
- sock_check = ValueChecker(allowed % "SocksPort").port_check
- ctrl_check = ValueChecker(allowed % "ControlPort").port_check
-
- optParameters = [
- ['bridges', 'b', None,
- 'File listing bridge IP:ORPorts to test'],
- ['relays', 'f', None,
- 'File listing relay IPs to test'],
- ['socks', 's', 9049, None, sock_check],
- ['control', 'c', 9052, None, ctrl_check],
- ['torpath', 'p', None,
- 'Path to the Tor binary to use'],
- ['datadir', 'd', None,
- 'Tor DataDirectory to use'],
- ['transport', 't', None,
- 'Tor ClientTransportPlugin'],
- ['resume', 'r', 0,
- 'Resume at this index']]
- optFlags = [['random', 'x', 'Use random ControlPort and SocksPort']]
-
- def postOptions(self):
- if not self['bridges'] and not self['relays']:
- raise MissingAssetException(
- "Bridget can't run without bridges or relays to test!")
- if self['transport']:
- ValueChecker.uid_check(
- "Can't run bridget as root with pluggable transports!")
- if not self['bridges']:
- raise PTNoBridgesException
- if self['socks'] or self['control']:
- if self['random']:
- raise RandomPortException
- if self['datadir']:
- ValueChecker.dir_check(self['datadir'])
- if self['torpath']:
- ValueChecker.file_check(self['torpath'])
-
-class BridgetAsset(Asset):
- """Class for parsing bridget Assets ignoring commented out lines."""
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
- def parse_line(self, line):
- if line.startswith('#'):
- return
- else:
- return line.replace('\n','')
-
-class BridgetTest(OONITest):
- """
- XXX fill me in
-
- :ivar config:
- An :class:`ooni.lib.txtorcon.TorConfig` instance.
- :ivar relays:
- A list of all provided relays to test.
- :ivar bridges:
- A list of all provided bridges to test.
- :ivar socks_port:
- Integer for Tor's SocksPort.
- :ivar control_port:
- Integer for Tor's ControlPort.
- :ivar transport:
- String defining the Tor's ClientTransportPlugin, for testing
- a bridge's pluggable transport functionality.
- :ivar tor_binary:
- Path to the Tor binary to use, e.g. \'/usr/sbin/tor\'
- """
- implements(IPlugin, ITest)
-
- shortName = "bridget"
- description = "Use a Tor process to test connecting to bridges or relays"
- requirements = None
- options = BridgetArgs
- blocking = False
-
- def initialize(self):
- """
- Extra initialization steps. We only want one child Tor process
- running, so we need to deal with most of the TorConfig() only once,
- before the experiment runs.
- """
- self.socks_port = 9049
- self.control_port = 9052
- self.circuit_timeout = 90
- self.tor_binary = '/usr/sbin/tor'
- self.data_directory = None
-
- def __make_asset_list__(opt, lst):
- log.msg("Loading information from %s ..." % opt)
- with open(opt) as opt_file:
- for line in opt_file.readlines():
- if line.startswith('#'):
- continue
- else:
- lst.append(line.replace('\n',''))
-
- def __count_remaining__(which):
- total, reach, unreach = map(lambda x: which[x],
- ['all', 'reachable', 'unreachable'])
- count = len(total) - reach() - unreach()
- return count
-
- ## XXX should we do report['bridges_up'].append(self.bridges['current'])
- self.bridges = {}
- self.bridges['all'], self.bridges['up'], self.bridges['down'] = \
- ([] for i in range(3))
- self.bridges['reachable'] = lambda: len(self.bridges['up'])
- self.bridges['unreachable'] = lambda: len(self.bridges['down'])
- self.bridges['remaining'] = lambda: __count_remaining__(self.bridges)
- self.bridges['current'] = None
- self.bridges['pt_type'] = None
- self.bridges['use_pt'] = False
-
- self.relays = {}
- self.relays['all'], self.relays['up'], self.relays['down'] = \
- ([] for i in range(3))
- self.relays['reachable'] = lambda: len(self.relays['up'])
- self.relays['unreachable'] = lambda: len(self.relays['down'])
- self.relays['remaining'] = lambda: __count_remaining__(self.relays)
- self.relays['current'] = None
-
- if self.local_options:
- try:
- from ooni.lib.txtorcon import TorConfig
- except ImportError:
- raise TxtorconImportError
- else:
- self.config = TorConfig()
- finally:
- options = self.local_options
-
- if options['bridges']:
- self.config.UseBridges = 1
- __make_asset_list__(options['bridges'], self.bridges['all'])
- if options['relays']:
- ## first hop must be in TorState().guards
- self.config.EntryNodes = ','.join(relay_list)
- __make_asset_list__(options['relays'], self.relays['all'])
- if options['socks']:
- self.socks_port = options['socks']
- if options['control']:
- self.control_port = options['control']
- if options['random']:
- log.msg("Using randomized ControlPort and SocksPort ...")
- self.socks_port = randint(1024, 2**16)
- self.control_port = randint(1024, 2**16)
- if options['torpath']:
- self.tor_binary = options['torpath']
- if options['datadir']:
- self.data_directory = parse_data_dir(options['datadir'])
- if options['transport']:
- ## ClientTransportPlugin transport exec pathtobinary [options]
- ## XXX we need a better way to deal with all PTs
- log.msg("Using ClientTransportPlugin %s" % options['transport'])
- self.bridges['use_pt'] = True
- [self.bridges['pt_type'], pt_exec] = \
- options['transport'].split(' ', 1)
-
- if self.bridges['pt_type'] == "obfs2":
- self.config.ClientTransportPlugin = \
- self.bridges['pt_type'] + " " + pt_exec
- else:
- raise PTNotFoundException
-
- self.config.SocksPort = self.socks_port
- self.config.ControlPort = self.control_port
- self.config.CookieAuthentication = 1
-
- def __load_assets__(self):
- """
- Load bridges and/or relays from files given in user options. Bridges
- should be given in the form IP:ORport. We don't want to load these as
- assets, because it's inefficient to start a Tor process for each one.
-
- We cannot use the Asset model, because that model calls
- self.experiment() with the current Assets, which would be one relay
- and one bridge, then it gives the defer.Deferred returned from
- self.experiment() to self.control(), which means that, for each
- (bridge, relay) pair, experiment gets called again, which instantiates
- an additional Tor process that attempts to bind to the same
- ports. Thus, additionally instantiated Tor processes return with
- RuntimeErrors, which break the final defer.chainDeferred.callback(),
- sending it into the errback chain.
- """
- assets = {}
- if self.local_options:
- if self.local_options['bridges']:
- assets.update({'bridge':
- BridgetAsset(self.local_options['bridges'])})
- if self.local_options['relays']:
- assets.update({'relay':
- BridgetAsset(self.local_options['relays'])})
- return assets
-
- def experiment(self, args):
- """
- if bridges:
- 1. configure first bridge line
- 2a. configure data_dir, if it doesn't exist
- 2b. write torrc to a tempfile in data_dir
- 3. start tor } if any of these
- 4. remove bridges which are public relays } fail, add current
- 5. SIGHUP for each bridge } bridge to unreach-
- } able bridges.
- if relays:
- 1a. configure the data_dir, if it doesn't exist
- 1b. write torrc to a tempfile in data_dir
- 2. start tor
- 3. remove any of our relays which are already part of current
- circuits
- 4a. attach CustomCircuit() to self.state
- 4b. RELAY_EXTEND for each relay } if this fails, add
- } current relay to list
- } of unreachable relays
- 5.
- if bridges and relays:
- 1. configure first bridge line
- 2a. configure data_dir if it doesn't exist
- 2b. write torrc to a tempfile in data_dir
- 3. start tor
- 4. remove bridges which are public relays
- 5. remove any of our relays which are already part of current
- circuits
- 6a. attach CustomCircuit() to self.state
- 6b. for each bridge, build three circuits, with three
- relays each
- 6c. RELAY_EXTEND for each relay } if this fails, add
- } current relay to list
- } of unreachable relays
-
- :param args:
- The :class:`BridgetAsset` line currently being used. Except that it
- in Bridget it doesn't, so it should be ignored and avoided.
- """
- try:
- from ooni.utils import process
- from ooni.utils.onion import remove_public_relays, start_tor
- from ooni.utils.onion import start_tor_filter_nodes
- from ooni.utils.onion import setup_fail, setup_done
- from ooni.utils.onion import CustomCircuit
- from ooni.utils.timer import deferred_timeout, TimeoutError
- from ooni.lib.txtorcon import TorConfig, TorState
- except ImportError:
- raise TxtorconImportError
- except TxtorconImportError, tie:
- log.err(tie)
- sys.exit()
-
- def reconfigure_done(state, bridges):
- """
- Append :ivar:`bridges['current']` to the list
- :ivar:`bridges['up'].
- """
- log.msg("Reconfiguring with 'Bridge %s' successful"
- % bridges['current'])
- bridges['up'].append(bridges['current'])
- return state
-
- def reconfigure_fail(state, bridges):
- """
- Append :ivar:`bridges['current']` to the list
- :ivar:`bridges['down'].
- """
- log.msg("Reconfiguring TorConfig with parameters %s failed"
- % state)
- bridges['down'].append(bridges['current'])
- return state
-
- @defer.inlineCallbacks
- def reconfigure_bridge(state, bridges):
- """
- Rewrite the Bridge line in our torrc. If use of pluggable
- transports was specified, rewrite the line as:
- Bridge <transport_type> <IP>:<ORPort>
- Otherwise, rewrite in the standard form:
- Bridge <IP>:<ORPort>
-
- :param state:
- A fully bootstrapped instance of
- :class:`ooni.lib.txtorcon.TorState`.
- :param bridges:
- A dictionary of bridges containing the following keys:
-
- bridges['remaining'] :: A function returning and int for the
- number of remaining bridges to test.
- bridges['current'] :: A string containing the <IP>:<ORPort>
- of the current bridge.
- bridges['use_pt'] :: A boolean, True if we're testing
- bridges with a pluggable transport;
- False otherwise.
- bridges['pt_type'] :: If :ivar:`bridges['use_pt'] is True,
- this is a string containing the type
- of pluggable transport to test.
- :return:
- :param:`state`
- """
- log.msg("Current Bridge: %s" % bridges['current'])
- log.msg("We now have %d bridges remaining to test..."
- % bridges['remaining']())
- try:
- if bridges['use_pt'] is False:
- controller_response = yield state.protocol.set_conf(
- 'Bridge', bridges['current'])
- elif bridges['use_pt'] and bridges['pt_type'] is not None:
- controller_reponse = yield state.protocol.set_conf(
- 'Bridge', bridges['pt_type'] +' '+ bridges['current'])
- else:
- raise PTNotFoundException
-
- if controller_response == 'OK':
- finish = yield reconfigure_done(state, bridges)
- else:
- log.err("SETCONF for %s responded with error:\n %s"
- % (bridges['current'], controller_response))
- finish = yield reconfigure_fail(state, bridges)
-
- defer.returnValue(finish)
-
- except Exception, e:
- log.err("Reconfiguring torrc with Bridge line %s failed:\n%s"
- % (bridges['current'], e))
- defer.returnValue(None)
-
- def attacher_extend_circuit(attacher, deferred, router):
- ## XXX todo write me
- ## state.attacher.extend_circuit
- raise NotImplemented
- #attacher.extend_circuit
-
- def state_attach(state, path):
- log.msg("Setting up custom circuit builder...")
- attacher = CustomCircuit(state)
- state.set_attacher(attacher, reactor)
- state.add_circuit_listener(attacher)
- return state
-
- ## OLD
- #for circ in state.circuits.values():
- # for relay in circ.path:
- # try:
- # relay_list.remove(relay)
- # except KeyError:
- # continue
- ## XXX how do we attach to circuits with bridges?
- d = defer.Deferred()
- attacher.request_circuit_build(d)
- return d
-
- def state_attach_fail(state):
- log.err("Attaching custom circuit builder failed: %s" % state)
-
- log.msg("Bridget: initiating test ... ") ## Start the experiment
-
- ## if we've at least one bridge, and our config has no 'Bridge' line
- if self.bridges['remaining']() >= 1 \
- and not 'Bridge' in self.config.config:
-
- ## configure our first bridge line
- self.bridges['current'] = self.bridges['all'][0]
- self.config.Bridge = self.bridges['current']
- ## avoid starting several
- self.config.save() ## processes
- assert self.config.config.has_key('Bridge'), "No Bridge Line"
-
- ## start tor and remove bridges which are public relays
- from ooni.utils.onion import start_tor_filter_nodes
- state = start_tor_filter_nodes(reactor, self.config,
- self.control_port, self.tor_binary,
- self.data_directory, self.bridges)
- #controller = defer.Deferred()
- #controller.addCallback(singleton_semaphore, tor)
- #controller.addErrback(setup_fail)
- #bootstrap = defer.gatherResults([controller, filter_bridges],
- # consumeErrors=True)
-
- if state is not None:
- log.debug("state:\n%s" % state)
- log.debug("Current callbacks on TorState():\n%s"
- % state.callbacks)
-
- ## if we've got more bridges
- if self.bridges['remaining']() >= 2:
- #all = []
- for bridge in self.bridges['all'][1:]:
- self.bridges['current'] = bridge
- #new = defer.Deferred()
- #new.addCallback(reconfigure_bridge, state, self.bridges)
- #all.append(new)
- #check_remaining = defer.DeferredList(all, consumeErrors=True)
- #state.chainDeferred(check_remaining)
- state.addCallback(reconfigure_bridge, self.bridges)
-
- if self.relays['remaining']() > 0:
- while self.relays['remaining']() >= 3:
- #path = list(self.relays.pop() for i in range(3))
- #log.msg("Trying path %s" % '->'.join(map(lambda node:
- # node, path)))
- self.relays['current'] = self.relays['all'].pop()
- for circ in state.circuits.values():
- for node in circ.path:
- if node == self.relays['current']:
- self.relays['up'].append(self.relays['current'])
- if len(circ.path) < 3:
- try:
- ext = attacher_extend_circuit(state.attacher, circ,
- self.relays['current'])
- ext.addCallback(attacher_extend_circuit_done,
- state.attacher, circ,
- self.relays['current'])
- except Exception, e:
- log.err("Extend circuit failed: %s" % e)
- else:
- continue
-
- #state.callback(all)
- #self.reactor.run()
- return state
-
- def startTest(self, args):
- """
- Local override of :meth:`OONITest.startTest` to bypass calling
- self.control.
-
- :param args:
- The current line of :class:`Asset`, not used but kept for
- compatibility reasons.
- :return:
- A fired deferred which callbacks :meth:`experiment` and
- :meth:`OONITest.finished`.
- """
- self.start_time = date.now()
- self.d = self.experiment(args)
- self.d.addErrback(log.err)
- self.d.addCallbacks(self.finished, log.err)
- return self.d
-
-## So that getPlugins() can register the Test:
-#bridget = BridgetTest(None, None, None)
-
-## ISIS' NOTES
-## -----------
-## TODO:
-## x cleanup documentation
-## x add DataDirectory option
-## x check if bridges are public relays
-## o take bridge_desc file as input, also be able to give same
-## format as output
-## x Add asynchronous timeout for deferred, so that we don't wait
-## o Add assychronous timout for deferred, so that we don't wait
-## forever for bridges that don't work.
diff --git a/ooni/bridget/tests/echo.py b/ooni/bridget/tests/echo.py
deleted file mode 100644
index 7f3217a..0000000
--- a/ooni/bridget/tests/echo.py
+++ /dev/null
@@ -1,205 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# +---------+
-# | echo.py |
-# +---------+
-# A simple ICMP-8 ping test.
-#
-# :author: Isis Lovecruft
-# :version: 0.0.1-pre-alpha
-# :license: (c) 2012 Isis Lovecruft
-# see attached LICENCE file
-#
-
-import os
-import sys
-
-from pprint import pprint
-
-from twisted.internet import reactor
-from twisted.plugin import IPlugin
-from twisted.python import usage
-from ooni.nettest import NetTestCase
-from ooni.utils import log, Storage
-from ooni.utils.net import PermissionsError, IfaceError
-
-try:
- from scapy.all import sr1, IP, ICMP ## XXX v4/v6?
- from ooni.lib import txscapy
- from ooni.lib.txscapy import txsr, txsend
- from ooni.templates.scapyt import ScapyTest
-except:
- log.msg("This test requires scapy, see www.secdev.org/projects/scapy")
-
-## xxx TODO: move these to a utility function for determining OSes
-LINUX=sys.platform.startswith("linux")
-OPENBSD=sys.platform.startswith("openbsd")
-FREEBSD=sys.platform.startswith("freebsd")
-NETBSD=sys.platform.startswith("netbsd")
-DARWIN=sys.platform.startswith("darwin")
-SOLARIS=sys.platform.startswith("sunos")
-WINDOWS=sys.platform.startswith("win32")
-
-class EchoTest(ScapyTest):
- """
- xxx fill me in
- """
- name = 'echo'
- author = 'Isis Lovecruft <isis at torproject.org>'
- description = 'A simple ICMP-8 test to see if a host is reachable.'
- version = '0.0.1'
- inputFile = ['file', 'f', None, 'File of list of IPs to ping']
- requirements = None
- report = Storage()
-
- optParameters = [
- ['interface', 'i', None, 'Network interface to use'],
- ['count', 'c', 5, 'Number of packets to send', int],
- ['size', 's', 56, 'Number of bytes to send in ICMP data field', int],
- ['ttl', 'l', 25, 'Set the IP Time to Live', int],
- ['timeout', 't', 2, 'Seconds until timeout if no response', int],
- ['pcap', 'p', None, 'Save pcap to this file'],
- ['receive', 'r', True, 'Receive response packets']
- ]
-
- def setUpClass(self, *a, **kw):
- '''
- :ivar ifaces:
- Struct returned from getifaddrs(3) and turned into a tuple in the
- form (*ifa_name, AF_FAMILY, *ifa_addr)
- '''
- super(EchoTest, self).__init__(*a, **kw)
-
- ## allow subclasses which register/implement external classes
- ## to define their own reactor without overrides:
- if not hasattr(super(EchoTest, self), 'reactor'):
- log.debug("%s test: Didn't find reactor!" % self.name)
- self.reactor = reactor
-
- if self.localOptions:
- log.debug("%s localOptions found" % self.name)
- log.debug("%s test options: %s" % (self.name, self.subOptions))
- self.local_options = self.localOptions.parseOptions(self.subOptions)
- for key, value in self.local_options:
- log.debug("Set attribute %s[%s] = %s" % (self.name, key, value))
- setattr(self, key, value)
-
- ## xxx is this now .subOptions?
- #self.inputFile = self.localOptions['file']
- self.timeout *= 1000 ## convert to milliseconds
-
- if not self.interface:
- log.msg("No network interface specified!")
- log.debug("OS detected: %s" % sys.platform)
- if LINUX or OPENBSD or NETBSD or FREEBSD or DARWIN or SOLARIS:
- from twisted.internet.test import _posixifaces
- log.msg("Attempting to discover network interfaces...")
- ifaces = _posixifaces._interfaces()
- elif WINDOWS:
- from twisted.internet.test import _win32ifaces
- log.msg("Attempting to discover network interfaces...")
- ifaces = _win32ifaces._interfaces()
- else:
- log.debug("Client OS %s not accounted for!" % sys.platform)
- log.debug("Unable to discover network interfaces...")
- ifaces = [('lo', '')]
-
- ## found = {'eth0': '1.1.1.1'}
- found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo']
- log.info("Found interfaces:\n%s" % pprint(found))
- self.interfaces = self.tryInterfaces(found)
- else:
- ## xxx need a way to check that iface exists, is up, and
- ## we have permissions on it
- log.debug("Our interface has been set to %s" % self.interface)
-
- if self.pcap:
- try:
- self.pcapfile = open(self.pcap, 'a+')
- except:
- log.msg("Unable to write to pcap file %s" % self.pcap)
- self.pcapfile = None
-
- try:
- assert os.path.isfile(self.file)
- fp = open(self.file, 'r')
- except Exception, e:
- hosts = ['8.8.8.8', '38.229.72.14']
- log.err(e)
- else:
- self.inputs = self.inputProcessor(fp)
- self.removePorts(hosts)
-
- log.debug("Initialization of %s test completed with:\n%s"
- % (self.name, ''.join(self.__dict__)))
-
- @staticmethod
- def inputParser(inputs):
- log.debug("Removing possible ports from host addresses...")
- log.debug("Initial inputs:\n%s" % pprint(inputs))
-
- assert isinstance(inputs, list)
- hosts = [h.rsplit(':', 1)[0] for h in inputs]
- log.debug("Inputs converted to:\n%s" % hosts)
-
- return hosts
-
- def tryInterfaces(self, ifaces):
- try:
- from scapy.all import sr1 ## we want this check to be blocking
- except:
- log.msg("This test requires scapy: www.secdev.org/projects/scapy")
- raise SystemExit
-
- ifup = {}
- while ifaces:
- for ifname, ifaddr in ifaces:
- log.debug("Currently testing network capabilities of interface"
- + "%s by sending a packet to our address %s"
- % (ifname, ifaddr))
- try:
- pkt = IP(dst=ifaddr)/ICMP()
- ans, unans = sr(pkt, iface=ifname, timeout=self.timeout)
- except Exception, e:
- raise PermissionsError if e.find("Errno 1") else log.err(e)
- else:
- ## xxx i think this logic might be wrong
- log.debug("Interface test packet\n%s\n\n%s"
- % (pkt.summary(), pkt.show2()))
- if ans.summary():
- log.info("Received answer for test packet on interface"
- +"%s :\n%s" % (ifname, ans.summary()))
- ifup.update(ifname, ifaddr)
- else:
- log.info("Our interface test packet was unanswered:\n%s"
- % unans.summary())
-
- if len(ifup) > 0:
- log.msg("Discovered the following working network interfaces: %s"
- % ifup)
- return ifup
- else:
- raise IfaceError("Could not find a working network interface.")
-
- def buildPackets(self):
- log.debug("self.input is %s" % self.input)
- log.debug("self.hosts is %s" % self.hosts)
- for addr in self.input:
- packet = IP(dst=self.input)/ICMP()
- self.request.append(packet)
- return packet
-
- def test_icmp(self):
- if self.recieve:
- self.buildPackets()
- all = []
- for packet in self.request:
- d = self.sendReceivePackets(packets=packet)
- all.append(d)
- self.response.update({packet: d})
- d_list = defer.DeferredList(all)
- return d_list
- else:
- d = self.sendPackets()
- return d
diff --git a/ooni/bridget/tests/tls-handshake.py b/ooni/bridget/tests/tls-handshake.py
deleted file mode 100644
index eba950e..0000000
--- a/ooni/bridget/tests/tls-handshake.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-
-import subprocess
-from subprocess import PIPE
-serverport = "129.21.124.215:443"
-# a subset of those from firefox
-ciphers = [
- "ECDHE-ECDSA-AES256-SHA",
- "ECDHE-RSA-AES256-SHA",
- "DHE-RSA-CAMELLIA256-SHA",
- "DHE-DSS-CAMELLIA256-SHA",
- "DHE-RSA-AES256-SHA",
- "DHE-DSS-AES256-SHA",
- "ECDH-ECDSA-AES256-CBC-SHA",
- "ECDH-RSA-AES256-CBC-SHA",
- "CAMELLIA256-SHA",
- "AES256-SHA",
- "ECDHE-ECDSA-RC4-SHA",
- "ECDHE-ECDSA-AES128-SHA",
- "ECDHE-RSA-RC4-SHA",
- "ECDHE-RSA-AES128-SHA",
- "DHE-RSA-CAMELLIA128-SHA",
- "DHE-DSS-CAMELLIA128-SHA"
-]
-def checkBridgeConnection(host, port)
- cipher_arg = ":".join(ciphers)
- cmd = ["openssl", "s_client", "-connect", "%s:%s" % (host,port)]
- cmd += ["-cipher", cipher_arg]
- proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE,stdin=PIPE)
- out, error = proc.communicate()
- success = "Cipher is DHE-RSA-AES256-SHA" in out
- return success
diff --git a/ooni/bridget/utils/__init__.py b/ooni/bridget/utils/__init__.py
deleted file mode 100644
index 92893d6..0000000
--- a/ooni/bridget/utils/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-all = ['inputs', 'log', 'onion', 'tests', 'interface', 'nodes', 'reports', 'work']
diff --git a/ooni/bridget/utils/inputs.py b/ooni/bridget/utils/inputs.py
deleted file mode 100644
index fe058cc..0000000
--- a/ooni/bridget/utils/inputs.py
+++ /dev/null
@@ -1,174 +0,0 @@
-#-*- coding: utf-8 -*-
-#
-# inputs.py
-# *********
-#
-# "...quis custodiet ipsos custodes?"
-# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.)
-#
-# :copyright: (c) 2012 Isis Lovecruft
-# :license: see LICENSE for more details.
-# :version: 0.1.0-beta
-#
-
-#from types import FunctionType, FileType
-import types
-
-from ooni.bridget import log
-from ooni.utils import date, Storage
-
-class InputFile:
- """
- This is a class describing a file used to store Tor bridge or relays
- inputs. It is a python iterator object, allowing it to be efficiently
- looped.
-
- This class should not be used directly, but rather its subclasses,
- BridgeFile and RelayFile should be used instead.
- """
-
- def __init__(self, file, **kw):
- """
- ## This is an InputAsset file, created because you tried to pass a
- ## non-existent filename to a test.
- ##
- ## To use this file, place one input to be tested per line. Each
- ## test takes different inputs. Lines which are commented out with
- ## a '#' are not used.
- """
- self.file = file
- self.eof = False
- self.all = Storage()
-
- for key, value in input_dict:
- self.all[key] = value
-
- try:
- self.handler = open(self.file, 'r')
- except IOError:
- with open(self.file, 'w') as explain:
- for line in self.__init__.__doc__:
- explain.writeline(line)
- self.handler = open(self.file, 'r')
- try:
- assert isinstance(self.handler, file), "That's not a file!"
- except AssertionError, ae:
- log.err(ae)
-
- # def __handler__(self):
- # """
- # Attempt to open InputFile.file and check that it is actually a file.
- # If it's not, create it and add an explaination for how InputFile files
- # should be used.
-
- # :return:
- # A :type:`file` which has been opened in read-only mode.
- # """
- # try:
- # handler = open(self.file, 'r')
- # except IOError, ioerror: ## not the hacker <(A)3
- # log.err(ioerror)
- # explanation = (
- # with open(self.file, 'w') as explain:
- # for line in explanation:
- # explain.writeline(line)
- # handler = open(self.file, 'r')
- # try:
- # assert isinstance(handler, file), "That's not a file!"
- # except AssertionError, ae:
- # log.err(ae)
- # else:
- # return handler
-
- def __iter__(next, StopIteration):
- """
- Returns the next input from the file.
- """
- #return self.next()
- return self
-
- def len(self):
- """
- Returns the number of the lines in the InputFile.
- """
- with open(self.file, 'r') as input_file:
- lines = input_file.readlines()
- for number, line in enumerate(lines):
- self.input_dict[number] = line
- return number + 1
-
- def next(self):
- try:
- return self.next_input()
- except:
- raise StopIteration
-
- def next_input(self):
- """
- Return the next input.
- """
- line = self.handler.readline()
- if line:
- parsed_line = self.parse_line(line)
- if parsed_line:
- return parsed_line
- else:
- self.fh.seek(0)
- raise StopIteration
-
- def default_parser(self, line):
- """
- xxx fill me in
- """
- if not line.startswith('#'):
- return line.replace('\n', '')
- else:
- return False
-
- def parse_line(self, line):
- """
- Override this method if you need line by line parsing of an Asset.
-
- The default parsing action is to ignore lines which are commented out
- with a '#', and to strip the newline character from the end of the
- line.
-
- If the line was commented out return an empty string instead.
-
- If a subclass Foo incorporates another class Bar, when Bar is not
- also a subclass of InputFile, and Bar.parse_line() exists, then
- do not overwrite Bar's parse_line method.
- """
- assert not hasattr(super(InputFile, self), 'parse_line')
-
- if self.parser is None:
- if not line.startswith('#'):
- return line.replace('\n', '')
- else:
- return ''
- else:
- try:
- assert isinstance(self.parser, FunctionType),"Not a function!"
- except AssertionError, ae:
- log.err(ae)
- else:
- return self.parser(line)
-
-class BridgeFile(InputFile):
- """
- xxx fill me in
- """
- def __init__(self, **kw):
- super(BridgeFile, self).__init__(**kw)
-
-class MissingInputException(Exception):
- """
-
- Raised when an :class:`InputFile` necessary for running the Test is
- missing.
-
- """
- def __init__(self, error_message):
- print error_message
- import sys
- return sys.exit()
diff --git a/ooni/bridget/utils/interface.py b/ooni/bridget/utils/interface.py
deleted file mode 100644
index aa55436..0000000
--- a/ooni/bridget/utils/interface.py
+++ /dev/null
@@ -1,54 +0,0 @@
-from zope.interface import implements, Interface, Attribute
-
-class ITest(Interface):
- """
- This interface represents an OONI test. It fires a deferred on completion.
- """
-
- shortName = Attribute("""A short user facing description for this test""")
- description = Attribute("""A string containing a longer description for the test""")
-
- requirements = Attribute("""What is required to run this this test, for example raw socket access or UDP or TCP""")
-
- options = Attribute("""These are the arguments to be passed to the test for it's execution""")
-
- blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""")
-
- def control(experiment_result, args):
- """
- @param experiment_result: The result returned by the experiment method.
-
- @param args: the keys of this dict are the names of the assets passed in
- from load_assets. The value is one item of the asset.
-
- Must return a dict containing what should be written to the report.
- Anything returned by control ends up inside of the YAMLOONI report.
- """
-
- def experiment(args):
- """
- Perform all the operations that are necessary to running a test.
-
- @param args: the keys of this dict are the names of the assets passed in
- from load_assets. The value is one item of the asset.
-
- Must return a dict containing the values to be passed to control.
- """
-
- def load_assets():
- """
- Load the assets that should be passed to the Test. These are the inputs
- to the OONI test.
- Must return a dict that has as keys the asset names and values the
- asset contents.
- If the test does not have any assets it should return an empty dict.
- """
-
- def end():
- """
- This can be called at any time to terminate the execution of all of
- these test instances.
-
- What this means is that no more test instances with new parameters will
- be created. A report will be written.
- """
diff --git a/ooni/bridget/utils/log.py b/ooni/bridget/utils/log.py
deleted file mode 100644
index eef50d8..0000000
--- a/ooni/bridget/utils/log.py
+++ /dev/null
@@ -1,98 +0,0 @@
-"""
-OONI logging facility.
-"""
-from sys import stderr, stdout
-
-from twisted.python import log, util
-from twisted.python.failure import Failure
-
-def _get_log_level(level):
- english = ['debug', 'info', 'warn', 'err', 'crit']
-
- levels = dict(zip(range(len(english)), english))
- number = dict(zip(english, range(len(english))))
-
- if not level:
- return number['info']
- else:
- ve = "Unknown log level: %s\n" % level
- ve += "Allowed levels: %s\n" % [word for word in english]
-
- if type(level) is int:
- if 0 <= level <= 4:
- return level
- elif type(level) is str:
- if number.has_key(level.lower()):
- return number[level]
- else:
- raise ValueError, ve
- else:
- raise ValueError, ve
-
-class OONITestFailure(Failure):
- """
- For handling Exceptions asynchronously.
-
- Can be given an Exception as an argument, else will use the
- most recent Exception from the current stack frame.
- """
- def __init__(self, exception=None, _type=None,
- _traceback=None, _capture=False):
- Failure.__init__(self, exc_type=_type,
- exc_tb=_traceback, captureVars=_capture)
-
-class OONILogObserver(log.FileLogObserver):
- """
- Supports logging level verbosity.
- """
- def __init__(self, logfile, verb=None):
- log.FileLogObserver.__init__(self, logfile)
- self.level = _get_log_level(verb) if verb is not None else 1
- assert type(self.level) is int
-
- def emit(self, eventDict):
- if 'logLevel' in eventDict:
- msgLvl = _get_log_level(eventDict['logLevel'])
- assert type(msgLvl) is int
- ## only log our level and higher
- if self.level <= msgLvl:
- text = log.textFromEventDict(eventDict)
- else:
- text = None
- else:
- text = log.textFromEventDict(eventDict)
-
- if text is None:
- return
-
- timeStr = self.formatTime(eventDict['time'])
- fmtDict = {'system': eventDict['system'],
- 'text': text.replace('\n','\n\t')}
- msgStr = log._safeFormat("[%(system)s] %(text)s\n", fmtDict)
-
- util.untilConcludes(self.write, timeStr + " " + msgStr)
- util.untilConcludes(self.flush)
-
-def start(logfile=None, verbosity=None):
- if log.defaultObserver:
- verbosity = _get_log_level(verbosity)
-
- ## Always log to file, keep level at info
- file = open(logfile, 'a') if logfile else stderr
- OONILogObserver(file, "info").start()
-
- log.msg("Starting OONI...")
-
-def debug(message, level="debug", **kw):
- print "[%s] %s" % (level, message)
- ## If we want debug messages in the logfile:
- #log.msg(message, logLevel=level, **kw)
-
-def msg(message, level="info", **kw):
- log.msg(message, logLevel=level, **kw)
-
-def err(message, level="err", **kw):
- log.err(logLevel=level, **kw)
-
-def fail(message, exception, level="crit", **kw):
- log.failure(message, OONITestFailure(exception, **kw), logLevel=level)
diff --git a/ooni/bridget/utils/nodes.py b/ooni/bridget/utils/nodes.py
deleted file mode 100644
index 155f183..0000000
--- a/ooni/bridget/utils/nodes.py
+++ /dev/null
@@ -1,176 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8
-"""
- nodes
- *****
-
- This contains all the code related to Nodes
- both network and code execution.
-
- :copyright: (c) 2012 by Arturo Filastò, Isis Lovecruft
- :license: see LICENSE for more details.
-
-"""
-
-import os
-from binascii import hexlify
-
-try:
- import paramiko
-except:
- print "Error: module paramiko is not installed."
-from pprint import pprint
-import sys
-import socks
-import xmlrpclib
-
-class Node(object):
- def __init__(self, address, port):
- self.address = address
- self.port = port
-
-class LocalNode(object):
- def __init__(self):
- pass
-
-"""
-[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5")
-[]: node_socket = node.wrap_socket()
-"""
-class NetworkNode(Node):
- def __init__(self, address, port, node_type="SOCKS5", auth_creds=None):
- self.node = Node(address,port)
-
- # XXX support for multiple types
- # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...)
- self.node_type = node_type
- # type-specific authentication credentials
- self.auth_creds = auth_creds
-
- def _get_socksipy_socket(self, proxy_type, auth_creds):
- import socks
- s = socks.socksocket()
- # auth_creds[0] -> username
- # auth_creds[1] -> password
- s.setproxy(proxy_type, self.node.address, self.node.port,
- self.auth_creds[0], self.auth_creds[1])
- return s
-
- def _get_socket_wrapper(self):
- if (self.node_type.startswith("SOCKS")): # SOCKS proxies
- if (self.node_type != "SOCKS5"):
- proxy_type = socks.PROXY_TYPE_SOCKS5
- elif (self.node_type != "SOCKS4"):
- proxy_type = socks.PROXY_TYPE_SOCKS4
- else:
- print "We don't know this proxy type."
- sys.exit(1)
-
- return self._get_socksipy_socket(proxy_type)
- elif (self.node_type == "HTTP"): # HTTP proxies
- return self._get_socksipy_socket(PROXY_TYPE_HTTP)
- else: # Unknown proxies
- print "We don't know this proxy type."
- sys.exit(1)
-
- def wrap_socket(self):
- return self._get_socket_wrapper()
-
-class CodeExecNode(Node):
- def __init__(self, address, port, node_type, auth_creds):
- self.node = Node(address,port)
-
- # node type (SSH proxy, etc.)
- self.node_type = node_type
- # type-specific authentication credentials
- self.auth_creds = auth_creds
-
- def add_unit(self):
- pass
-
- def get_status(self):
- pass
-
-class PlanetLab(CodeExecNode):
- def __init__(self, address, auth_creds, ooni):
- self.auth_creds = auth_creds
-
- self.config = ooni.utils.config
- self.logger = ooni.logger
- self.name = "PlanetLab"
-
- def _api_auth(self):
- api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/')
- auth = {}
- ## should be changed to separate node.conf file
- auth['Username'] = self.config.main.pl_username
- auth['AuthString'] = self.config.main.pl_password
- auth['AuthMethod'] = "password"
- authorized = api_server.AuthCheck(auth)
-
- if authorized:
- print 'We are authorized!'
- return auth
- else:
- print 'Authorization failed. Please check your settings for pl_username and pl_password in the ooni-probe.conf file.'
-
- def _search_for_nodes(self, node_filter=None):
- api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True)
- node_filter = {'hostname': '*.cert.org.cn'}
- return_fields = ['hostname', 'site_id']
- all_nodes = api_server.GetNodes(self.api_auth(), node_filter, boot_state_filter)
- pprint(all_nodes)
- return all_nodes
-
- def _add_nodes_to_slice(self):
- api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True)
- all_nodes = self.search_for_nodes()
- for node in all_nodes:
- api_server.AddNode(self.api_auth(), node['site_id'], all_nodes)
- print 'Adding nodes %s' % node['hostname']
-
- def _auth_login(slicename, machinename):
- """Attempt to authenticate to the given PL node, slicename and
- machinename, using any of the private keys in ~/.ssh/ """
-
- agent = paramiko.Agent()
- agent_keys = agent.get_keys()
- if len(agent_keys) == 0:
- return
-
- for key in agent_keys:
- print 'Trying ssh-agent key %s' % hexlify(key.get_fingerprint()),
- try:
- paramiko.transport.auth_publickey(machinename, slicename)
- print 'Public key authentication to PlanetLab node %s successful.' % machinename,
- return
- except paramiko.SSHException:
- print 'Public key authentication to PlanetLab node %s failed.' % machinename,
-
- def _get_command():
- pass
-
- def ssh_and_run_(slicename, machinename, command):
- """Attempt to make a standard OpenSSH client to PL node, and run
- commands from a .conf file."""
-
- ## needs a way to specify 'ssh -l <slicename> <machinename>'
- ## with public key authentication.
-
- command = PlanetLab.get_command()
-
- client = paramiko.SSHClient()
- client.load_system_host_keys()
- client.connect(machinename)
-
- stdin, stdout, stderr = client.exec_command(command)
-
- def send_files_to_node(directory, files):
- """Attempt to rsync a tree to the PL node."""
- pass
-
- def add_unit():
- pass
-
- def get_status():
- pass
diff --git a/ooni/bridget/utils/onion.py b/ooni/bridget/utils/onion.py
deleted file mode 100644
index 9d4cae7..0000000
--- a/ooni/bridget/utils/onion.py
+++ /dev/null
@@ -1,686 +0,0 @@
-#
-# onion.py
-# ----------
-# Utilities for working with Tor.
-#
-# This code is largely taken from txtorcon and its documentation, and as such
-# any and all credit should go to Meejah. Minor adjustments have been made to
-# use OONI's logging system, and to build custom circuits without actually
-# attaching streams.
-#
-# :author: Meejah, Isis Lovecruft
-# :license: see included LICENSE file
-# :copyright: copyright (c) 2012 The Tor Project, Inc.
-# :version: 0.1.0-alpha
-#
-# XXX TODO add report keys for onion methods
-
-import random
-import sys
-
-from twisted.internet import defer
-from zope.interface import implements
-
-from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher
-from ooni.lib.txtorcon import TorState, TorConfig
-from ooni.utils import log
-from ooni.utils.timer import deferred_timeout, TimeoutError
-
-def parse_data_dir(data_dir):
- """
- Parse a string that a has been given as a DataDirectory and determine
- its absolute path on the filesystem.
-
- :param data_dir:
- A directory for Tor's DataDirectory, to be parsed.
- :return:
- The absolute path of :param:data_dir.
- """
- from os import path, getcwd
- import sys
-
- try:
- assert isinstance(data_dir, str), \
- "Parameter type(data_dir) must be str"
- except AssertionError, ae:
- log.err(ae)
-
- if data_dir.startswith('~'):
- data_dir = path.expanduser(data_dir)
- elif data_dir.startswith('/'):
- data_dir = path.join(getcwd(), data_dir)
- elif data_dir.startswith('./'):
- data_dir = path.abspath(data_dir)
- else:
- data_dir = path.join(getcwd(), data_dir)
-
- try:
- assert path.isdir(data_dir), "Could not find %s" % data_dir
- except AssertionError, ae:
- log.err(ae)
- sys.exit()
- else:
- return data_dir
-
-def write_torrc(conf, data_dir=None):
- """
- Create a torrc in our data_dir. If we don't yet have a data_dir, create a
- temporary one. Any temporary files or folders are added to delete_list.
-
- :param conf:
- A :class:`ooni.lib.txtorcon.TorConfig` object, with all configuration
- values saved.
- :param data_dir:
- The Tor DataDirectory to use.
- :return: torrc, data_dir, delete_list
- """
- try:
- from os import write, close
- from tempfile import mkstemp, mkdtemp
- except ImportError, ie:
- log.err(ie)
-
- delete_list = []
-
- if data_dir is None:
- data_dir = mkdtemp(prefix='bridget-tordata')
- delete_list.append(data_dir)
- conf.DataDirectory = data_dir
-
- (fd, torrc) = mkstemp(dir=data_dir)
- delete_list.append(torrc)
- write(fd, conf.create_torrc())
- close(fd)
-
- return torrc, data_dir, delete_list
-
-def delete_files_or_dirs(delete_list):
- """
- Given a list of files or directories to delete, delete all and suppress
- all errors.
-
- :param delete_list:
- A list of files or directories to delete.
- """
- try:
- from os import unlink
- from shutil import rmtree
- except ImportError, ie:
- log.err(ie)
-
- for temp in delete_list:
- try:
- unlink(temp)
- except OSError:
- rmtree(temp, ignore_errors=True)
-
-def remove_node_from_list(node, list):
- for item in list: ## bridges don't match completely
- if item.startswith(node): ## due to the :<port>.
- try:
- log.msg("Removing %s because it is a public relay" % node)
- list.remove(item)
- except ValueError, ve:
- log.err(ve)
-
-def remove_public_relays(state, bridges):
- """
- Remove bridges from our bridge list which are also listed as public
- relays. This must be called after Tor has fully bootstrapped and we have a
- :class:`ooni.lib.txtorcon.TorState` with the
- :attr:`ooni.lib.txtorcon.TorState.routers` attribute assigned.
-
- XXX Does state.router.values() have all of the relays in the consensus, or
- just the ones we know about so far?
-
- XXX FIXME: There is a problem in that Tor needs a Bridge line to already be
- configured in order to bootstrap. However, after bootstrapping, we grab the
- microdescriptors of all the relays and check if any of our bridges are
- listed as public relays. Because of this, the first bridge does not get
- checked for being a relay.
- """
- IPs = map(lambda addr: addr.split(':',1)[0], bridges['all'])
- both = set(state.routers.values()).intersection(IPs)
-
- if len(both) > 0:
- try:
- updated = map(lambda node: remove_node_from_list(node), both)
- log.debug("Bridges in both: %s" % both)
- log.debug("Updated = %s" % updated)
- #if not updated:
- # defer.returnValue(state)
- #else:
- # defer.returnValue(state)
- return state
- except Exception, e:
- log.err("Removing public relays %s from bridge list failed:\n%s"
- % (both, e))
-
-def setup_done(proto):
- log.msg("Setup Complete")
- state = TorState(proto.tor_protocol)
- state.post_bootstrap.addCallback(state_complete)
- state.post_bootstrap.addErrback(setup_fail)
-
-def setup_fail(proto):
- log.msg("Setup Failed:\n%s" % proto)
- return proto
- #reactor.stop()
-
-def state_complete(state):
- """Called when we've got a TorState."""
- log.msg("We've completely booted up a Tor version %s at PID %d"
- % (state.protocol.version, state.tor_pid))
- log.msg("This Tor has the following %d Circuits:"
- % len(state.circuits))
- for circ in state.circuits.values():
- log.msg("%s" % circ)
- return state
-
-def updates(_progress, _tag, _summary):
- """Log updates on the Tor bootstrapping process."""
- log.msg("%d%%: %s" % (_progress, _summary))
-
-def bootstrap(ctrl):
- """
- Bootstrap Tor from an instance of
- :class:`ooni.lib.txtorcon.TorControlProtocol`.
- """
- conf = TorConfig(ctrl)
- conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail)
- log.msg("Tor process connected, bootstrapping ...")
-
-def start_tor(reactor, config, control_port, tor_binary, data_dir,
- report=None, progress=updates,
- process_cb=None, process_eb=None):
- """
- Use a txtorcon.TorConfig() instance, config, to write a torrc to a
- tempfile in our DataDirectory, data_dir. If data_dir is None, a temp
- directory will be created. Finally, create a TCP4ClientEndpoint at our
- control_port, and connect it to our reactor and a spawned Tor
- process. Compare with :meth:`txtorcon.launch_tor` for differences.
-
- :param reactor:
- An instance of class:`twisted.internet.reactor`.
- :param config:
- An instance of class:`txtorcon.TorConfig` with all torrc options
- already configured. ivar:`config.ControlPort`,
- ivar:`config.SocksPort`, ivar:`config.CookieAuthentication`, should
- already be set, as well as ivar:`config.UseBridges` and
- ivar:`config.Bridge` if bridges are to be used.
- ivar:`txtorcon.DataDirectory` does not need to be set.
- :param control_port:
- The port number to use for Tor's ControlPort.
- :param tor_binary:
- The full path to the Tor binary to use.
- :param data_dir:
- The directory to use as Tor's DataDirectory.
- :param report:
- The class:`ooni.plugoo.reports.Report` instance.
- :param progress:
- A non-blocking function to handle bootstrapping updates, which takes
- three parameters: _progress, _tag, and _summary.
- :param process_cb:
- The function to callback to after
- class:`ooni.lib.txtorcon.TorProcessProtocol` returns with the fully
- bootstrapped Tor process.
- :param process_eb:
- The function to errback to if
- class:`ooni.lib.txtorcon.TorProcessProtocol` fails.
- :return:
- The result of the callback of a
- class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a
- class:`txtorcon.TorControlProtocol` as .protocol.
- """
- try:
- from functools import partial
- from twisted.internet.endpoints import TCP4ClientEndpoint
- from ooni.lib.txtorcon import TorProtocolFactory
- from ooni.lib.txtorcon import TorProcessProtocol
- except ImportError, ie:
- log.err(ie)
-
- ## TODO: add option to specify an already existing torrc, which
- ## will require prior parsing to enforce necessary lines
- (torrc, data_dir, to_delete) = write_torrc(config, data_dir)
-
- log.msg("Starting Tor ...")
- log.msg("Using the following as our torrc:\n%s" % config.create_torrc())
- if report is None:
- report = {'torrc': config.create_torrc()}
- else:
- report.update({'torrc': config.create_torrc()})
-
- end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port)
- connection_creator = partial(end_point.connect, TorProtocolFactory())
- process_protocol = TorProcessProtocol(connection_creator, progress)
- process_protocol.to_delete = to_delete
-
- if process_cb is not None and process_eb is not None:
- process_protocol.connected_cb.addCallbacks(process_cb, process_eb)
-
- reactor.addSystemEventTrigger('before', 'shutdown',
- partial(delete_files_or_dirs, to_delete))
- try:
- transport = reactor.spawnProcess(process_protocol,
- tor_binary,
- args=(tor_binary,'-f',torrc),
- env={'HOME': data_dir},
- path=data_dir)
- transport.closeStdin()
- except RuntimeError, e:
- log.err("Starting Tor failed:")
- process_protocol.connected_cb.errback(e)
- except NotImplementedError, e:
- url = "http://starship.python.net/crew/mhammond/win32/Downloads.html"
- log.msg("Running bridget on Windows requires pywin32: %s" % url)
- process_protocol.connected_cb.errback(e)
-
- return process_protocol.connected_cb
-
- at defer.inlineCallbacks
-def start_tor_filter_nodes(reactor, config, control_port, tor_binary,
- data_dir, bridges):
- """
- Bootstrap a Tor process and return a fully-setup
- :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges
- to test in the list of known public relays,
- :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges
- which are known public relays.
-
- :param reactor:
- The :class:`twisted.internet.reactor`.
- :param config:
- An instance of :class:`ooni.lib.txtorcon.TorConfig`.
- :param control_port:
- The port to use for Tor's ControlPort. If already configured in
- the TorConfig instance, this can be given as
- TorConfig.config.ControlPort.
- :param tor_binary:
- The full path to the Tor binary to execute.
- :param data_dir:
- The full path to the directory to use as Tor's DataDirectory.
- :param bridges:
- A dictionary which has a key 'all' which is a list of bridges to
- test connecting to, e.g.:
- bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001']
- :return:
- A fully initialized :class:`ooni.lib.txtorcon.TorState`.
- """
- setup = yield start_tor(reactor, config, control_port,
- tor_binary, data_dir,
- process_cb=setup_done, process_eb=setup_fail)
- filter_nodes = yield remove_public_relays(setup, bridges)
- defer.returnValue(filter_nodes)
-
- at defer.inlineCallbacks
-def start_tor_with_timer(reactor, config, control_port, tor_binary, data_dir,
- bridges, timeout):
- """
- Start bootstrapping a Tor process wrapped with an instance of the class
- decorator :func:`ooni.utils.timer.deferred_timeout` and complete callbacks
- to either :func:`setup_done` or :func:`setup_fail`. Return a fully-setup
- :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges to test
- in the list of known public relays,
- :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges which
- are listed as known public relays.
-
- :param reactor:
- The :class:`twisted.internet.reactor`.
- :param config:
- An instance of :class:`ooni.lib.txtorcon.TorConfig`.
- :param control_port:
- The port to use for Tor's ControlPort. If already configured in
- the TorConfig instance, this can be given as
- TorConfig.config.ControlPort.
- :param tor_binary:
- The full path to the Tor binary to execute.
- :param data_dir:
- The full path to the directory to use as Tor's DataDirectory.
- :param bridges:
- A dictionary which has a key 'all' which is a list of bridges to
- test connecting to, e.g.:
- bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001']
- :param timeout:
- The number of seconds to attempt to bootstrap the Tor process before
- raising a :class:`ooni.utils.timer.TimeoutError`.
- :return:
- If the timeout limit is not exceeded, return a fully initialized
- :class:`ooni.lib.txtorcon.TorState`, else return None.
- """
- error_msg = "Bootstrapping has exceeded the timeout limit..."
- with_timeout = deferred_timeout(timeout, e=error_msg)(start_tor)
- try:
- setup = yield with_timeout(reactor, config, control_port, tor_binary,
- data_dir, process_cb=setup_done,
- process_eb=setup_fail)
- except TimeoutError, te:
- log.err(te)
- defer.returnValue(None)
- #except Exception, e:
- # log.err(e)
- # defer.returnValue(None)
- else:
- state = yield remove_public_relays(setup, bridges)
- defer.returnValue(state)
-
- at defer.inlineCallbacks
-def start_tor_filter_nodes_with_timer(reactor, config, control_port,
- tor_binary, data_dir, bridges, timeout):
- """
- Start bootstrapping a Tor process wrapped with an instance of the class
- decorator :func:`ooni.utils.timer.deferred_timeout` and complete callbacks
- to either :func:`setup_done` or :func:`setup_fail`. Then, filter our list
- of bridges to remove known public relays by calling back to
- :func:`remove_public_relays`. Return a fully-setup
- :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges to test
- in the list of known public relays,
- :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges which
- are listed as known public relays.
-
- :param reactor:
- The :class:`twisted.internet.reactor`.
- :param config:
- An instance of :class:`ooni.lib.txtorcon.TorConfig`.
- :param control_port:
- The port to use for Tor's ControlPort. If already configured in
- the TorConfig instance, this can be given as
- TorConfig.config.ControlPort.
- :param tor_binary:
- The full path to the Tor binary to execute.
- :param data_dir:
- The full path to the directory to use as Tor's DataDirectory.
- :param bridges:
- A dictionary which has a key 'all' which is a list of bridges to
- test connecting to, e.g.:
- bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001']
- :param timeout:
- The number of seconds to attempt to bootstrap the Tor process before
- raising a :class:`ooni.utils.timer.TimeoutError`.
- :return:
- If the timeout limit is not exceeded, return a fully initialized
- :class:`ooni.lib.txtorcon.TorState`, else return None.
- """
- error_msg = "Bootstrapping has exceeded the timeout limit..."
- with_timeout = deferred_timeout(timeout, e=error_msg)(start_tor_filter_nodes)
- try:
- state = yield with_timeout(reactor, config, control_port,
- tor_binary, data_dir, bridges)
- except TimeoutError, te:
- log.err(te)
- defer.returnValue(None)
- #except Exception, e:
- # log.err(e)
- # defer.returnValue(None)
- else:
- defer.returnValue(state)
-
-class CustomCircuit(CircuitListenerMixin):
- """
- Utility class for controlling circuit building. See
- 'attach_streams_by_country.py' in the txtorcon documentation.
-
- :param state:
- A fully bootstrapped instance of :class:`ooni.lib.txtorcon.TorState`.
- :param relays:
- A dictionary containing a key 'all', which is a list of relays to
- test connecting to.
- :ivar waiting_circuits:
- The list of circuits which we are waiting to attach to. You shouldn't
- need to touch this.
- """
- implements(IStreamAttacher)
-
- def __init__(self, state, relays=None):
- self.state = state
- self.waiting_circuits = []
- self.relays = relays
-
- def waiting_on(self, circuit):
- """
- Whether or not we are waiting on the given circuit before attaching to
- it.
-
- :param circuit:
- An item from :ivar:`ooni.lib.txtorcon.TorState.circuits`.
- :return:
- True if we are waiting on the circuit, False if not waiting.
- """
- for (circid, d) in self.waiting_circuits:
- if circuit.id == circid:
- return True
- return False
-
- def circuit_extend(self, circuit, router):
- "ICircuitListener"
- if circuit.purpose != 'GENERAL':
- return
- if self.waiting_on(circuit):
- log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex))
-
- def circuit_built(self, circuit):
- "ICircuitListener"
- if circuit.purpose != 'GENERAL':
- return
- log.msg("Circuit %s built ..." % circuit.id)
- log.msg("Full path of %s: %s" % (circuit.id, circuit.path))
- for (circid, d) in self.waiting_circuits:
- if circid == circuit.id:
- self.waiting_circuits.remove((circid, d))
- d.callback(circuit)
-
- def circuit_failed(self, circuit, reason):
- """
- If building a circuit has failed, try to remove it from our list of
- :ivar:`waiting_circuits`, else request to build it.
-
- :param circuit:
- An item from :ivar:`ooni.lib.txtorcon.TorState.circuits`.
- :param reason:
- A :class:`twisted.python.fail.Failure` instance.
- :return:
- None
- """
- if self.waiting_on(circuit):
- log.msg("Circuit %s failed for reason %s" % (circuit.id, reason))
- circid, d = None, None
- for c in self.waiting_circuits:
- if c[0] == circuit.id:
- circid, d = c
- if d is None:
- raise Exception("Expected to find circuit.")
-
- self.waiting_circuits.remove((circid, d))
- log.msg("Trying to build a circuit for %s" % circid)
- self.request_circuit_build(d)
-
- def check_circuit_route(self, router):
- """
- Check if a relay is a hop in one of our already built circuits.
-
- :param router:
- An item from the list
- :func:`ooni.lib.txtorcon.TorState.routers.values()`.
- """
- for circ in self.state.circuits.values():
- if router in circ.path:
- #router.update() ## XXX can i use without args? no.
- TorInfo.dump(self)
-
- def request_circuit_build(self, deferred, path=None):
- """
- Request a custom circuit.
-
- :param deferred:
- A :class:`twisted.internet.defer.Deferred` for this circuit.
- :param path:
- A list of router ids to build a circuit from. The length of this
- list must be at least three.
- """
- if path is None:
-
- pick = self.relays['all'].pop
- n = self.state.entry_guards.values()
- choose = random.choice
-
- first, middle, last = (None for i in range(3))
-
- if self.relays['remaining']() >= 3:
- first, middle, last = (pick() for i in range(3))
- elif self.relays['remaining']() < 3:
- first = choose(n)
- middle = pick()
- if self.relays['remaining'] == 2:
- middle, last = (pick() for i in range(2))
- elif self.relay['remaining'] == 1:
- middle = pick()
- last = choose(n)
- else:
- log.msg("Qu'est-que fuque?")
- else:
- middle, last = (random.choice(self.state.routers.values())
- for i in range(2))
-
- path = [first, middle, last]
-
- else:
- assert isinstance(path, list), \
- "Circuit path must be a list of relays!"
- assert len(path) >= 3, \
- "Circuit path must be at least three hops!"
-
- log.msg("Requesting a circuit: %s"
- % '->'.join(map(lambda node: node, path)))
-
- class AppendWaiting:
- def __init__(self, attacher, deferred):
- self.attacher = attacher
- self.d = deferred
- def __call__(self, circ):
- """
- Return from build_circuit is a Circuit, however,
- we want to wait until it is built before we can
- issue an attach on it and callback to the Deferred
- we issue here.
- """
- log.msg("Circuit %s is in progress ..." % circ.id)
- self.attacher.waiting_circuits.append((circ.id, self.d))
-
- return self.state.build_circuit(path).addCallback(
- AppendWaiting(self, deferred)).addErrback(
- log.err)
-
-class TxtorconImportError(ImportError):
- """
- Raised when ooni.lib.txtorcon cannot be imported from. Checks our current
- working directory and the path given to see if txtorcon has been
- initialized via /ooni/lib/Makefile.
- """
- from os import getcwd, path
-
- cwd, tx = getcwd(), 'lib/txtorcon/torconfig.py'
- try:
- log.msg("Unable to import from ooni.lib.txtorcon")
- if cwd.endswith('ooni'):
- check = path.join(cwd, tx)
- elif cwd.endswith('utils'):
- check = path.join(cwd, '../'+tx)
- else:
- check = path.join(cwd, 'ooni/'+tx)
- assert path.isfile(check)
- except:
- log.msg("Error: Some OONI libraries are missing!")
- log.msg("Please go to /ooni/lib/ and do \"make all\"")
-
-class PTNoBridgesException(Exception):
- """Raised when a pluggable transport is specified, but not bridges."""
- def __init__(self):
- log.msg("Pluggable transport requires the bridges option")
- return sys.exit()
-
-class PTNotFoundException(Exception):
- def __init__(self, transport_type):
- m = "Pluggable Transport type %s was unaccounted " % transport_type
- m += "for, please contact isis(at)torproject(dot)org and it will "
- m += "get included."
- log.msg("%s" % m)
- return sys.exit()
-
- at defer.inlineCallbacks
-def __start_tor_with_timer__(reactor, config, control_port, tor_binary,
- data_dir, bridges=None, relays=None, timeout=None,
- retry=None):
- """
- A wrapper for :func:`start_tor` which wraps the bootstrapping of a Tor
- process and its connection to a reactor with a
- :class:`twisted.internet.defer.Deferred` class decorator utility,
- :func:`ooni.utils.timer.deferred_timeout`, and a mechanism for resets.
-
- ## XXX fill me in
- """
- raise NotImplementedError
-
- class RetryException(Exception):
- pass
-
- import sys
- from ooni.utils.timer import deferred_timeout, TimeoutError
-
- def __make_var__(old, default, _type):
- if old is not None:
- assert isinstance(old, _type)
- new = old
- else:
- new = default
- return new
-
- reactor = reactor
- timeout = __make_var__(timeout, 120, int)
- retry = __make_var__(retry, 1, int)
-
- with_timeout = deferred_timeout(timeout)(start_tor)
-
- @defer.inlineCallbacks
- def __start_tor__(rc=reactor, cf=config, cp=control_port, tb=tor_binary,
- dd=data_dir, br=bridges, rl=relays, cb=setup_done,
- eb=setup_fail, af=remove_public_relays, retry=retry):
- try:
- setup = yield with_timeout(rc,cf,cp,tb,dd)
- except TimeoutError:
- retry -= 1
- defer.returnValue(retry)
- else:
- if setup.callback:
- setup = yield cb(setup)
- elif setup.errback:
- setup = yield eb(setup)
- else:
- setup = setup
-
- if br is not None:
- state = af(setup,br)
- else:
- state = setup
- defer.returnValue(state)
-
- @defer.inlineCallbacks
- def __try_until__(tries):
- result = yield __start_tor__()
- try:
- assert isinstance(result, int)
- except AssertionError:
- defer.returnValue(result)
- else:
- if result >= 0:
- tried = yield __try_until__(result)
- defer.returnValue(tried)
- else:
- raise RetryException
- try:
- tried = yield __try_until__(retry)
- except RetryException:
- log.msg("All retry attempts to bootstrap Tor have timed out.")
- log.msg("Exiting ...")
- defer.returnValue(sys.exit())
- else:
- defer.returnValue(tried)
diff --git a/ooni/bridget/utils/reports.py b/ooni/bridget/utils/reports.py
deleted file mode 100644
index ae67b13..0000000
--- a/ooni/bridget/utils/reports.py
+++ /dev/null
@@ -1,144 +0,0 @@
-from __future__ import with_statement
-
-import os
-import yaml
-
-import itertools
-from ooni.utils import log, date, net
-
-class Report:
- """This is the ooni-probe reporting mechanism. It allows
- reporting to multiple destinations and file formats.
-
- :scp the string of <host>:<port> of an ssh server
-
- :yaml the filename of a the yaml file to write
-
- :file the filename of a simple txt file to write
-
- :tcp the <host>:<port> of a TCP server that will just listen for
- inbound connection and accept a stream of data (think of it
- as a `nc -l -p <port> > filename.txt`)
- """
- def __init__(self, testname=None, file="report.log",
- scp=None,
- tcp=None):
-
- self.testname = testname
- self.file = file
- self.tcp = tcp
- self.scp = scp
- #self.config = ooni.config.report
-
- #if self.config.timestamp:
- # tmp = self.file.split('.')
- # self.file = '.'.join(tmp[:-1]) + "-" + \
- # datetime.now().isoformat('-') + '.' + \
- # tmp[-1]
- # print self.file
-
- self.scp = None
- self.write_header()
-
- def write_header(self):
- pretty_date = date.pretty_date()
- header = "# OONI Probe Report for Test %s\n" % self.testname
- header += "# %s\n\n" % pretty_date
- self._write_to_report(header)
- # XXX replace this with something proper
- address = net.getClientAddress()
- test_details = {'start_time': str(date.now()),
- 'asn': address['asn'],
- 'test_name': self.testname,
- 'addr': address['ip']}
- self(test_details)
-
- def _write_to_report(self, dump):
- reports = []
-
- if self.file:
- reports.append("file")
-
- if self.tcp:
- reports.append("tcp")
-
- if self.scp:
- reports.append("scp")
-
- #XXX make this non blocking
- for report in reports:
- self.send_report(dump, report)
-
- def __call__(self, data):
- """
- This should be invoked every time you wish to write some
- data to the reporting system
- """
- dump = yaml.dump([data])
- self._write_to_report(dump)
-
- def file_report(self, data):
- """
- This reports to a file in YAML format
- """
- with open(self.file, 'a+') as f:
- f.write(data)
-
- def send_report(self, data, type):
- """
- This sends the report using the
- specified type.
- """
- #print "Reporting %s to %s" % (data, type)
- log.msg("Reporting to %s" % type)
- getattr(self, type+"_report").__call__(data)
-
-class NewReport(object):
- filename = 'report.log'
- startTime = None
- endTime = None
- testName = None
- ipAddr = None
- asnAddr = None
-
- def _open():
- self.fp = open(self.filename, 'a+')
-
- @property
- def header():
- pretty_date = date.pretty_date()
- report_header = "# OONI Probe Report for Test %s\n" % self.testName
- report_header += "# %s\n\n" % pretty_date
- test_details = {'start_time': self.startTime,
- 'asn': asnAddr,
- 'test_name': self.testName,
- 'addr': ipAddr}
- report_header += yaml.dump([test_details])
- return report_header
-
- def create():
- """
- Create a new report by writing it's header.
- """
- self.fp = open(self.filename, 'w+')
- self.fp.write(self.header)
-
- def exists():
- """
- Returns False if the file does not exists.
- """
- return os.path.exists(self.filename)
-
- def write(data):
- """
- Write a report to the file.
-
- :data: python data structure to be written to report.
- """
- if not self.exists():
- self.create()
- else:
- self._open()
- yaml_encoded_data = yaml.dump([data])
- self.fp.write(yaml_encoded_data)
- self.fp.close()
diff --git a/ooni/bridget/utils/tests.py b/ooni/bridget/utils/tests.py
deleted file mode 100644
index ea4be0b..0000000
--- a/ooni/bridget/utils/tests.py
+++ /dev/null
@@ -1,141 +0,0 @@
-import os
-import yaml
-from zope.interface import Interface, Attribute
-
-import logging
-import itertools
-from twisted.internet import reactor, defer, threads
-## XXX why is this imported and not used?
-from twisted.python import failure
-
-from ooni.utils import log, date
-from ooni.plugoo import assets, work
-from ooni.plugoo.reports import Report
-from ooni.plugoo.interface import ITest
-
-class OONITest(object):
- """
- This is the base class for writing OONI Tests.
-
- It should be used in conjunction with the ITest Interface. It allows the
- developer to benefit from OONIs reporting system and command line argument
- parsing system.
- """
- name = "oonitest"
- # By default we set this to False, meaning that we don't block
- blocking = False
- reactor = reactor
- tool = False
- ended = False
-
- def __init__(self, local_options, global_options, report, ooninet=None,
- reactor=reactor):
- # These are the options that are read through the tests suboptions
- self.local_options = local_options
- # These are the options global to all of OONI
- self.global_options = global_options
- self.report = report
- #self.ooninet = ooninet
- self.reactor = reactor
- self.result = {}
- self.initialize()
- self.assets = self.load_assets()
-
- def initialize(self):
- """
- Override this method if you are interested in having some extra
- behavior when your test class is instantiated.
- """
- pass
-
- def load_assets(self):
- """
- This method should be overriden by the test writer to provide the
- logic for loading their assets.
- """
- return {}
-
- def __repr__(self):
- return "<OONITest %s %s %s>" % (self.local_options,
- self.global_options,
- self.assets)
-
- def end(self):
- """
- State that the current test should finish.
- """
- self.ended = True
-
- def finished(self, return_value):
- """
- The Test has finished running, we must now calculate the test runtime
- and add all time data to the report.
- """
- #self.ooninet.report(result)
- self.end_time = date.now()
- result = self.result
- result['start_time'] = str(self.start_time)
- result['end_time'] = str(self.end_time)
- result['run_time'] = str(self.end_time - self.start_time)
- result['return_value'] = return_value
- log.msg("FINISHED %s" % result)
- self.report(result)
- return result
-
- def _do_experiment(self, args):
- """
- A wrapper around the launch of experiment.
- If we are running a blocking test experiment will be run in a thread if
- not we expect it to return a Deferred.
-
- @param args: the asset line(s) that we are working on.
-
- returns a deferred.
- """
- if self.blocking:
- self.d = threads.deferToThread(self.experiment, args)
- else:
- self.d = self.experiment(args)
-
- self.d.addCallback(self.control, args)
- self.d.addCallback(self.finished)
- self.d.addErrback(self.finished)
- return self.d
-
- def control(self, result, args):
- """
- Run the control.
-
- @param result: what was returned by experiment.
-
- @param args: the asset(s) lines that we are working on.
- """
- log.msg("Doing control")
- return result
-
- def experiment(self, args):
- """
- Run the experiment. This sample implementation returns a deferred,
- making it a non-blocking test.
-
- @param args: the asset(s) lines that we are working on.
- """
- log.msg("Doing experiment")
- d = defer.Deferred()
- return d
-
- def startTest(self, args):
- """
- This method is invoked by the worker to start the test with one line of
- the asset file.
-
- @param args: the asset(s) lines that we are working on.
- """
- self.start_time = date.now()
-
- if self.shortName:
- log.msg("Starting test %s" % self.shortName)
- else:
- log.msg("Starting test %s" % self.__class__)
-
- return self._do_experiment(args)
diff --git a/ooni/bridget/utils/work.py b/ooni/bridget/utils/work.py
deleted file mode 100644
index c329c20..0000000
--- a/ooni/bridget/utils/work.py
+++ /dev/null
@@ -1,147 +0,0 @@
-# -*- coding: UTF-8
-"""
- work.py
- **********
-
- This contains all code related to generating
- Units of Work and processing it.
-
- :copyright: (c) 2012 by Arturo Filastò.
- :license: see LICENSE for more details.
-
-"""
-import itertools
-import yaml
-from datetime import datetime
-
-from zope.interface import Interface, Attribute
-
-from twisted.python import failure
-from twisted.internet import reactor, defer
-
-class Worker(object):
- """
- This is the core of OONI. It takes as input Work Units and
- runs them concurrently.
- """
- def __init__(self, maxconcurrent=10, reactor=reactor):
- """
- @param maxconcurrent: how many test instances should be run
- concurrently.
- """
- self.reactor = reactor
- self.maxconcurrent = maxconcurrent
- self._running = 0
- self._queued = []
-
- def _run(self, r):
- """
- Check if we should start another test because we are below maximum
- concurrency.
-
- This function is called every time a test finishes running.
-
- @param r: the return value of a previous test.
- """
- if self._running > 0:
- self._running -= 1
-
- if self._running < self.maxconcurrent and self._queued:
- workunit, d = self._queued.pop(0)
- asset, test, idx = workunit
- while test.ended and workunit:
- try:
- workunit, d = self._queued.pop(0)
- asset, test, idx = workunit
- except:
- workunit = None
-
- if not test.ended:
- self._running += 1
- actuald = test.startTest(asset).addBoth(self._run)
-
- if isinstance(r, failure.Failure):
- # XXX probably we should be doing something to retry test running
- r.trap()
-
- if self._running == 0 and not self._queued:
- self.reactor.stop()
-
- return r
-
- def push(self, workunit):
- """
- Add a test to the test queue and run it if we are not maxed out on
- concurrency.
-
- @param workunit: a tuple containing the (asset, test, idx), where asset
- is the line of the asset(s) we are working on, test
- is an instantiated test and idx is the index we are
- currently at.
- """
- if self._running < self.maxconcurrent:
- asset, test, idx = workunit
- if not test.ended:
- self._running += 1
- return test.startTest(asset).addBoth(self._run)
-
- d = defer.Deferred()
- self._queued.append((workunit, d))
- return d
-
-class WorkGenerator(object):
- """
- Factory responsible for creating units of work.
-
- This shall be run on the machine running OONI-cli. The returned WorkUnits
- can either be run locally or on a remote OONI Node or Network Node.
- """
- size = 10
-
- def __init__(self, test, arguments=None, start=None):
- self.Test = test
-
- if self.Test.assets and self.Test.assets.values()[0]:
- self.assetGenerator = itertools.product(*self.Test.assets.values())
- else:
- self.assetGenerator = None
-
- self.assetNames = self.Test.assets.keys()
-
- self.idx = 0
- self.end = False
- if start:
- self.skip(start)
-
- def __iter__(self):
- return self
-
- def skip(self, start):
- """
- Skip the first x number of lines of the asset.
-
- @param start: int how many items we should skip.
- """
- for j in xrange(0, start-1):
- for i in xrange(0, self.size):
- self.assetGenerator.next()
- self.idx += 1
-
- def next(self):
- if self.end:
- raise StopIteration
-
- if not self.assetGenerator:
- self.end = True
- return ({}, self.Test, self.idx)
-
- try:
- asset = self.assetGenerator.next()
- ret = {}
- for i, v in enumerate(asset):
- ret[self.assetNames[i]] = v
- except StopIteration:
- raise StopIteration
-
- self.idx += 1
- return (ret, self.Test, self.idx)
More information about the tor-commits
mailing list