commit de0c47f7152b3a0a9dd2baf27ce2b028819e3f63 Author: Arturo Filastò arturo@filasto.net Date: Mon Nov 5 20:23:27 2012 +0100
Port Bridget to the new API * Unbreak the new API and make sure tests work as they should * Add more advanced parameters for the test input as requested by Isis --- nettests/bridget.py | 89 +++-------- ooni/__init__.py | 18 +-- ooni/inputunit.py | 133 +--------------- ooni/nettest.py | 399 ++++++++--------------------------------------- ooni/oonicli.py | 2 + ooni/reporter.py | 10 +- ooni/runner.py | 237 +++++++++++----------------- ooni/templates/httpt.py | 19 +++ ooni/utils/onion.py | 4 +- 9 files changed, 221 insertions(+), 690 deletions(-)
diff --git a/nettests/bridget.py b/nettests/bridget.py index a334747..74d1d47 100644 --- a/nettests/bridget.py +++ b/nettests/bridget.py @@ -20,23 +20,25 @@ import os import sys
from twisted.python import usage -from twisted.plugin import IPlugin from twisted.internet import defer, error, reactor -from zope.interface import implements + +from ooni import nettest
from ooni.utils import log, date from ooni.utils.config import ValueChecker
-from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset, MissingAssetException from ooni.utils.onion import TxtorconImportError from ooni.utils.onion import PTNoBridgesException, PTNotFoundException
+ try: from ooni.utils.onion import parse_data_dir except: log.msg("Please go to /ooni/lib and do 'make txtorcon' to run this test!")
+class MissingAssetException(Exception): + pass + class RandomPortException(Exception): """Raised when using a random port conflicts with configured ports.""" def __init__(self): @@ -83,18 +85,7 @@ class BridgetArgs(usage.Options): if self['torpath']: ValueChecker.file_check(self['torpath'])
-class BridgetAsset(Asset): - """Class for parsing bridget Assets ignoring commented out lines.""" - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - if line.startswith('#'): - return - else: - return line.replace('\n','') - -class BridgetTest(OONITest): +class BridgetTest(nettest.TestCase): """ XXX fill me in
@@ -114,15 +105,13 @@ class BridgetTest(OONITest): :ivar tor_binary: Path to the Tor binary to use, e.g. '/usr/sbin/tor' """ - implements(IPlugin, ITest) + name = "bridget" + author = "Isis Lovecruft isis@torproject.org" + version = "0.1" + description = "Use a Tor process to test connecting to bridges or relays" + advancedOptParameters = BridgetArgs
- shortName = "bridget" - description = "Use a Tor process to test connecting to bridges or relays" - requirements = None - options = BridgetArgs - blocking = False - - def initialize(self): + def setUp(self): """ Extra initialization steps. We only want one child Tor process running, so we need to deal with most of the TorConfig() only once, @@ -134,14 +123,16 @@ class BridgetTest(OONITest): self.tor_binary = '/usr/sbin/tor' self.data_directory = None
- def __make_asset_list__(opt, lst): + def read_from_file(filename): log.msg("Loading information from %s ..." % opt) - with open(opt) as opt_file: - for line in opt_file.readlines(): + with open(filename) as fp: + lst = [] + for line in fp.readlines(): if line.startswith('#'): continue else: lst.append(line.replace('\n','')) + return lst
def __count_remaining__(which): total, reach, unreach = map(lambda x: which[x], @@ -168,23 +159,24 @@ class BridgetTest(OONITest): self.relays['remaining'] = lambda: __count_remaining__(self.relays) self.relays['current'] = None
- if self.local_options: + if self.localOptions: try: - from ooni.lib.txtorcon import TorConfig + from txtorcon import TorConfig except ImportError: raise TxtorconImportError else: self.config = TorConfig() finally: - options = self.local_options + options = self.localOptions
if options['bridges']: self.config.UseBridges = 1 - __make_asset_list__(options['bridges'], self.bridges['all']) + self.bridges['all'] = read_from_file(options['bridges']) if options['relays']: ## first hop must be in TorState().guards + # XXX where is this defined? self.config.EntryNodes = ','.join(relay_list) - __make_asset_list__(options['relays'], self.relays['all']) + self.relays['all'] = read_from_file(options['relays']) if options['socks']: self.socks_port = options['socks'] if options['control']: @@ -215,33 +207,7 @@ class BridgetTest(OONITest): self.config.ControlPort = self.control_port self.config.CookieAuthentication = 1
- def __load_assets__(self): - """ - Load bridges and/or relays from files given in user options. Bridges - should be given in the form IP:ORport. We don't want to load these as - assets, because it's inefficient to start a Tor process for each one. - - We cannot use the Asset model, because that model calls - self.experiment() with the current Assets, which would be one relay - and one bridge, then it gives the defer.Deferred returned from - self.experiment() to self.control(), which means that, for each - (bridge, relay) pair, experiment gets called again, which instantiates - an additional Tor process that attempts to bind to the same - ports. Thus, additionally instantiated Tor processes return with - RuntimeErrors, which break the final defer.chainDeferred.callback(), - sending it into the errback chain. - """ - assets = {} - if self.local_options: - if self.local_options['bridges']: - assets.update({'bridge': - BridgetAsset(self.local_options['bridges'])}) - if self.local_options['relays']: - assets.update({'relay': - BridgetAsset(self.local_options['relays'])}) - return assets - - def experiment(self, args): + def test_bridget(self): """ if bridges: 1. configure first bridge line @@ -465,7 +431,7 @@ class BridgetTest(OONITest): #self.reactor.run() return state
- def startTest(self, args): + def disabled_startTest(self, args): """ Local override of :meth:`OONITest.startTest` to bypass calling self.control. @@ -483,9 +449,6 @@ class BridgetTest(OONITest): self.d.addCallbacks(self.finished, log.err) return self.d
-## So that getPlugins() can register the Test: -#bridget = BridgetTest(None, None, None) - ## ISIS' NOTES ## ----------- ## TODO: diff --git a/ooni/__init__.py b/ooni/__init__.py index 2f7c19f..f025d30 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,11 +1,3 @@ -__all__ = [ - 'config', 'inputunit', 'kit', - 'lib', 'nettest', 'oonicli', 'reporter', - 'runner', 'templates', 'utils', - ## XXX below are legacy related modules - 'ooniprobe', 'plugoo', 'plugins' - ] - from . import config from . import inputunit from . import kit @@ -17,7 +9,13 @@ from . import runner from . import templates from . import utils
-## XXX below are legacy related modules +# XXX below are legacy related modules #from . import ooniprobe -from . import plugoo +#from . import plugoo #from . import plugins + +__all__ = ['config', 'inputunit', 'kit', + 'lib', 'nettest', 'oonicli', 'reporter', + 'runner', 'templates', 'utils'] + # XXX below are legacy related modules + #'ooniprobe', 'plugoo', 'plugins'] diff --git a/ooni/inputunit.py b/ooni/inputunit.py index 054b3a9..e5b6187 100644 --- a/ooni/inputunit.py +++ b/ooni/inputunit.py @@ -1,31 +1,5 @@ -# -*- coding: utf-8 -*- -# -# inputunit.py -# ------------ -# Classes and function for working with OONI TestCase inputs. -# -# @authors: Arturo Filasto, Isis Lovecruft -# @version: 0.1.0-alpha -# @license: see included LICENSE file -# @copyright: 2012 Arturo Filasto, Isis Lovecruft, The Tor Project Inc. -# - from twisted.trial import unittest
-from zope.interface.exceptions import BrokenImplementation - - -def simpleInputUnitProcessor(input_unit): - """A simple InputUnit generator without parsing.""" - try: - assert hasattr(input_unit, '__iter__'), "inputs must be iterable!" - except AssertionError, ae: - raise BrokenImplementation, ae - else: - for i in input_unit: - yield i - - class PatchedPyUnitResultAdapter(unittest.PyUnitResultAdapter): def __init__(self, original): """ @@ -39,8 +13,8 @@ unittest.PyUnitResultAdapter = PatchedPyUnitResultAdapter
class InputUnitFactory(object): """ - This is a factory that takes the size of input units to be generated and a - set of units that is a python iterable item, and outputs InputUnit objects + This is a factory that takes the size of input units to be generated a set + of units that is a python iterable item and outputs InputUnit objects containing inputUnitSize elements.
This object is a python iterable, this means that it does not need to keep @@ -74,6 +48,7 @@ class InputUnitFactory(object):
return InputUnit(input_unit_elements)
+ class InputUnit(object): """ This is a python iterable object that contains the input elements to be @@ -101,107 +76,5 @@ class InputUnit(object): def append(self, input): self._inputs.append(input)
-class IUProcessorExit(GeneratorExit): - """InputUnitProcessor has exited.""" - -class IUProcessorAborted(Exception): - """The InputUnitProcessor was aborted with Inputs remaining.""" - -class InputUnitProcessor(InputUnit): - """ - Create a generator for returning inputs one-by-one from a - :class:`InputUnit` (or any other iterable defined within an instance of - :class:`ooni.nettest.NetTestCase`), and a generator function. - - The :ivar:generator can be a custom generator, or chain of generators, for - customized parsing of an InputUnit, or it can be an imported - function. There are useful imported functions in the builtin - :mod:`itertools`. If no :ivar:`generator` is given, the default one strips - whitespace characters, then returns the Input if the input does not begin - with a crunch (#) or bang (!). :) - - If :ivar:catchStopIter is True, then catch any StopIterations and return a - 2-tuple containing (boolean, list(unprocessed)): - - (True, None) when :ivar:iterable is empty, or - (False, [unprocessed]) if :ivar:iterable was not empty. - - If :ivar:catchStopIter is False (default), then we catch the StopIteration - exception, mark :attr:`empty` as 'True', and reraise the StopIteration.
- XXX fill me in with parameter details - XXX I'm not sure if we need this class anymore - """ - empty = False - - def __init__(self, iterable, input_filter=None, catch_err=False): - """ - Create an InputUnitProcessor. - - XXX fill me in - """ - from itertools import takewhile - from types import GeneratorType - - assert hasattr(iterable, "__iter__"), "That's not an iterable!" - - self._iterable = iterable - self._infilter = input_filter - self._noerr = catch_err - self._empty = self.empty
- def __len__(self): - return len(self.iterable) - - def __unprocessed__(self): - if not self.isdone(): - unprocessed = \ - [u for u in self._unit if self._unit is not StopIteration] - return (False, unprocessed) - else: - return (True, None) - - def isdone(self): - return self._empty - - def throw(self, exception, message="", traceback=None): - try: - raise exception, message, traceback - except: - yield self._iterable.next() - - @staticmethod - def __strip__(x): - return x.strip() - - @staticmethod - def _default_input_filter(x): - if not x.startswith('#') and not x.startswith('!'): - return True - else: - return False - - def __make_filter__(self): - if self.input_filter and hasattr(self.input_filter, "__call__"): - return self.input_filter - else: - return self._default_input_filter - - def finish(self): - if self._noerr: - return self.__unprocessed__() - else: - if not self.isdone(): - raise IUProcessorAborted - else: - raise IUProcessorExit - - def process(self): - carbon = self.__make_filter__() - try: - yield takewhile(carbon, - [i for i in self.__strip__(self._iterable)] - ).next() - except StopIteration, si: - self._empty = True - yield self.finish() diff --git a/ooni/nettest.py b/ooni/nettest.py index 94c8c56..c2a8575 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -1,17 +1,8 @@ # -*- encoding: utf-8 -*- # -# nettest.py -# -------------------> -# -# :authors: Arturo "hellais" Filastò art@fuffa.org, -# Isis Lovecruft isis@torproject.org +# :authors: Arturo "hellais" Filastò art@fuffa.org # :licence: see LICENSE -# :copyright: 2012 Arturo Filasto, Isis Lovecruft -# :version: 0.1.0-alpha -# -# <-------------------
-from functools import partial import sys import os import itertools @@ -25,7 +16,6 @@ from ooni.utils import log
pyunit = __import__('unittest')
- class InputTestSuite(pyunit.TestSuite): """ This in an extension of a unittest test suite. It adds support for inputs @@ -66,306 +56,26 @@ class InputTestSuite(pyunit.TestSuite): self._idx += 1 return result
-class NetTestAdaptor(unittest.TestCase): - """ - XXX fill me in - """ - - # @classmethod - # def __new__(cls, *args, **kwargs): - # try: - # setUpClass() - # except Exception, e: - # log.debug("NetTestAdaptor: constructor could not find setUpClass") - # log.err(e) - # return super( NetTestAdaptor, cls ).__new__(cls, *args, **kwargs) - - def __init__(self, *args, **kwargs): - """ - If you override me, you must call - - ``super(NetTestCase, self).__init__(*args, **kwargs)`` - - at the beginning of your __init__ method. Keyword arguments passed to - the above statement become attributes of the adaptor, and can be used - to alter the logic of input handling and parent class instantiation. - Therefore, You probably do not need to pass me any keyword arguments - when calling me, i.e. using ``(*args, **kwargs)`` will work just fine. - """ - log.debug("NetTestAdaptor: created") - if kwargs: - if 'methodName' in kwargs: - log.debug("NetTestAdaptor: found 'methodName' in kwargs") - log.debug("NetTestAdaptor: calling unittest.TestCase.__init()") - super( NetTestAdaptor, self ).__init__( - methodName=kwargs['methodName'] ) - else: - log.debug("NetTestAdaptor: calling unittest.TestCase.__init()") - super( NetTestAdaptor, self ).__init__( ) - - for key, value in kwargs.items(): ## Let subclasses define their - if key != 'methodName': ## instantiation without - if not hasattr(self, key): ## overriding parent classes - log.debug("NetTestAdaptor: calling setattr(self,%s,%s)" - % (key, value) ) - setattr(self, key, value) - - #setattr(self, "copyattr", __copy_attr__) - - ## Internal attribute copies: - #self._input_parser = copyattr("inputParser", alt=__input_parser__) - #self._nettest_name = copyattr("name", alt="NetTestAdaptor")) - - #self.setUpClass(self.__class__)
- #if hasattr(self, parsed_inputs): - # self.inputs = self.parsed_inputs - #else: - # log.debug("Unable to find parsed inputs") - - @staticmethod - def __copyattr__(obj, old, new=None, alt=None): - """ - Assign me to a new attribute name to have a copy of the old - attribute, if it exists. - - Example: - >>> self.sun = "black" - >>> self._watermelon = __copyattr__(self, "sun") - >>> self._clocknoise = __copyattr__(self, "sound") - >>> print self._watermelon - black - >>> print self._clocknoise - - :param old: - A string representing the name of the old attribute - :param new: - (Optional) A string to set as the new attribute name. - :param alt: - (Optional) An alternate value to return if the old - attribute is not found. - :return: - If :param:`old` is found, I return it's value. - - If :param:`old` is not found: - If :param:`alt` is given, I return :param:`alt`. - Else, I return None. - - If :param:`new` is set, I do not return anything, and - instead I set the new attribute's name to :param:`name` - and it's value to the value which I would have otherwise - returned. - """ - if not new: - if not alt: - if hasattr(obj, old): - return getattr(obj, old) - return - else: - if hasattr(obj, old): - return getattr(obj, old) - return alt - else: - if not alt: - if hasattr(obj, old): - _copy = getattr(obj, old) - else: - copy = None - setattr(obj, new, _copy) - else: - if hasattr(obj, old): - _copy = getattr(obj, old) - else: - _copy = alt - setattr(obj, new, _copy) - - def copyattr(self, *args, **kwargs): - if len(args) >= 1: - _copy = partial(__copyattr__, args[0]) - if len(args) == 2: - return _copy(new=args[1]) - elif len(args) == 3: - return _copy(new=args[1], alt=args[2]) - elif kwargs: - return _copy(kwargs) - else: - return - - @staticmethod - def __input_parser__(one_input): return one_input - - @classmethod - def __get_inputs__(cls): - """ - I am called during class setup and you probably should not override - me. I gather the internal inputs from :class:`NetTestCase` attributes - and pass them through :meth:`NetTestCase.inputParser`. If you are - looking for a way to parse inputs from inputFile, see - :meth:`inputParser`. If :class:`NetTestCase` has an attribute - :attr:`inputFile`, I also handle opening that file, striping each line - of whitespace, and then sending the line to - :meth:`NetTestCase.inputParser`. - - All inputs which I find, both statically set inputs and those returned - from processing an inputFile, I add to a list :ivar:`parsed`, after - parsing them. I return :ivar:`parsed`: - - :ivar parsed: - A list of parsed inputs. - :return: - :ivar:`parsed`. - """ - parsed = [] - - if cls._raw_inputs: - if isinstance(cls._raw_inputs, (list, tuple,)): - if len(cls._raw_inputs) > 0: - if len(cls._raw_inputs) == 1 and cls._raw_inputs[0] is None: - pass ## don't burn cycles on testing null inputs - else: - log.msg("Received direct inputs:\n%s" % cls._raw_inputs) - parsed.extend( - [cls._input_parser(x) for x in cls._raw_inputs]) - elif isinstance(cls._raw_inputs, str): - separated = cls._raw_inputs.translate(None, ',') ## space delineates - inputlist = separated.split(' ') - parsed.extend([cls._input_parser(x) for x in inputlist]) - else: - log.debug("inputs not string or list; type: %s" - % type(cls._raw_inputs)) - - if cls.subarg_inputs: - log.debug("NetTestAdaptor: __get_inputs__ found subarg_inputs=%s" - % cls.subarg_inputs) - parsed.extend([cls._input_parser(x) for x in cls.subarg_inputs]) - - if cls._input_file: - try: - log.debug("NetTestAdaptor: __get_inputs__ Opening input file") - fp = open(cls._input_file) - except: - log.debug("NetTestAdaptor: __get_inputs__ Couldn't open input file") - else: - log.debug("NetTestAdaptor: __get_inputs__ Running input file processor") - lines = [line.strip() for line in fp.readlines()] - fp.close() - - ## add to what we've already parsed, if any: - log.debug("NetTestAdaptor: __get_inputs__ Parsing lines from input file") - parsed.extend([cls._input_parser(ln) for ln in lines]) - else: - log.debug("NetTestAdaptor: %s specified that it doesn't need inputFile." - % cls._nettest_name) - - return parsed - - @classmethod - def __getopt__(cls, parseArgs=None): - """ - Constuctor for a custom t.p.usage.Options class, per NetTestCase. - - old code from runner.py: - opts = Options() - opts.parseOptions(config['subArgs']) - cls.localOptions = opts - """ - if cls._testopt_params or cls._input_file: - if not cls._testopt_params: - cls._testopt_params = [] - - if cls._input_file: - cls._testopt_params.append(cls.input_file) - - class NetTestOptions(usage.Options): - """Per NetTestCase Options class.""" - optParameters = cls._testopt_params - optFlags = cls._testopt_flags - subOptions = cls._sub_options - subCommands = cls._sub_commands - defaultSubCommand = cls._default_subcmd - ## XXX i'm not sure if this part will work: - parseArgs = lambda a: cls.subarg_inputs.append(a) - - def opt_version(self): - """Display test version and exit.""" - print "Test version: ", cls._nettest_version - sys.exit(0) - - options = NetTestOptions() - return options - - #if cls._input_file: - # cls._input_file = cls.options[cls._input_file[0]] - - @classmethod - def addSubArgToInputs(cls, subarg): - cls.subarg_inputs.append(subarg) - - @classmethod - def buildOptions(cls, from_global): - log.debug("NetTestAdaptor: getTestOptions called") - options = cls.__getopt__() - log.debug("NetTestAdaptor: getTestOptions: cls.options = %s" - % options) - options.parseOptions(from_global) - setattr(cls, "local_options", options) - log.debug("NetTestAdaptor: getTestOptions: cls.local_options = %s" - % cls.local_options) - - @classmethod - def setUpClass(cls): - """ - Create a NetTestCase. To add futher setup steps before a set of tests - in a TestCase instance run, create a function called 'setUp'. - - Class attributes, such as `report`, `optParameters`, `name`, and - `author` should be overriden statically as class attributes in any - subclass of :class:`ooni.nettest.NetTestCase`, so that the calling - functions during NetTestCase class setup can handle them correctly. - """ - - log.debug("NetTestAdaptor: setUpClass called") - - ## These internal inputs are for handling inputs and inputFile - cls._raw_inputs = cls.__copyattr__(cls, "inputs") - cls._input_file = cls.__copyattr__(cls, "inputFile") - cls._input_parser = cls.__copyattr__(cls, "inputParser", - alt=cls.__input_parser__) - cls._nettest_name = cls.__copyattr__(cls, "name", alt="NetTestAdaptor") - - ## This creates a class attribute with all of the parsed inputs, - ## which the instance will later set to be `self.inputs`. - cls.parsed_inputs = cls.__get_inputs__() - cls.subarg_inputs = cls.__copyattr__(cls, "subarg_inputs", - alt=[]) - - ## XXX we should handle options generation here - cls._testopt_params = cls.__copyattr__(cls, "optParameters") - cls._testopt_flags = cls.__copyattr__(cls, "optFlags") - cls._sub_options = cls.__copyattr__(cls, "subOptions") - cls._sub_commands = cls.__copyattr__(cls, "subCommands") - cls._default_subcmd = cls.__copyattr__(cls, "defaultSubCommand") - cls._nettest_version = cls.__copyattr__(cls, "version") - -class NetTestCase(NetTestAdaptor): +class NetTestCase(unittest.TestCase): """ This is the monad of the OONI nettest universe. When you write a nettest you will subclass this object.
* inputs: can be set to a static set of inputs. All the tests (the methods - starting with the "test_" prefix) will be run once per input. At every - run the _input_ attribute of the TestCase instance will be set to the - value of the current iteration over inputs. Any python iterable object - can be set to inputs. + starting with the "test_" prefix) will be run once per input. At every run + the _input_ attribute of the TestCase instance will be set to the value of + the current iteration over inputs. Any python iterable object can be set + to inputs.
- * inputFile: attribute should be set to an array containing the command - line argument that should be used as the input file. Such array looks - like this: + * inputFile: attribute should be set to an array containing the command line + argument that should be used as the input file. Such array looks like + this:
- ``["commandlinearg", "c", "The description"]`` + ``["commandlinearg", "c", "default value" "The description"]``
- The second value of such arrray is the shorthand for the command line - arg. The user will then be able to specify inputs to the test via: + The second value of such arrray is the shorthand for the command line arg. + The user will then be able to specify inputs to the test via:
``ooniprobe mytest.py --commandlinearg path/to/file.txt``
@@ -374,9 +84,9 @@ class NetTestCase(NetTestAdaptor): ``ooniprobe mytest.py -c path/to/file.txt``
- * inputParser: should be set to a function that takes as argument an - iterable containing test inputs and it simply parses those inputs and - returns them back to test instance. + * inputProcessor: should be set to a function that takes as argument an + open file descriptor and it will return the input to be passed to the test + instance.
* name: should be set to the name of the test.
@@ -388,26 +98,44 @@ class NetTestCase(NetTestAdaptor): * version: is the version string of the test.
* requiresRoot: set to True if the test must be run as root. + + * optFlags: is assigned a list of lists. Each list represents a flag parameter, as so: + + optFlags = [['verbose', 'v', 'Makes it tell you what it doing.'], | ['quiet', 'q', 'Be vewy vewy quiet.']] + + As you can see, the first item is the long option name (prefixed with + '--' on the command line), followed by the short option name (prefixed with + '-'), and the description. The description is used for the built-in handling of + the --help switch, which prints a usage summary. + + + * optParameters: is much the same, except the list also contains a default value: + + | optParameters = [['outfile', 'O', 'outfile.log', 'Description...']] + + * advancedOptParameters: a subclass of twisted.python.usage.Options for more advanced command line arguments fun. + """ name = "I Did Not Change The Name" author = "Jane Doe foo@example.com" - version = "0.0.0" + version = "0"
+ inputs = [None] inputFile = None - inputs = [None]
report = {} report['errors'] = []
- optParameters = None optFlags = None - subCommands = None + optParameters = None + advancedOptParameters = None + requiresRoot = False
def deferSetUp(self, ignored, result): """ - If we have the reporterFactory set we need to write the header. If - such method is not present we will only run the test skipping header + If we have the reporterFactory set we need to write the header. If such + method is not present we will only run the test skipping header writing. """ if result.reporterFactory.firstrun: @@ -420,35 +148,34 @@ class NetTestCase(NetTestAdaptor): log.debug("Not first run. Running test setup directly") return unittest.TestCase.deferSetUp(self, ignored, result)
- def inputParser(self, inputs): - """Replace me with a custom function for parsing inputs.""" - log.debug("Running custom input processor") - return inputs + def inputProcessor(self, fp): + log.debug("Running default input processor") + for x in fp.readlines(): + yield x.strip() + fp.close()
def getOptions(self): log.debug("Getting options for test") - - if self.local_options: - log.debug("NetTestCase: getOptions: self.local_options=%s" - % str(self.local_options)) - else: - log.debug("could not find cls.localOptions!") - - return {'inputs': self.parsed_inputs, + if self.inputFile: + try: + assert isinstance(self.inputFile, str) + except AssertionError, ae: + log.err(ae) + else: + if os.path.isfile(self.inputFile): + print self.inputFile + fp = open(self.inputFile) + self.inputs = self.inputProcessor(fp) + elif not self.inputs[0]: + pass + elif self.inputFile: + raise usage.UsageError("No input file specified!") + # XXX perhaps we may want to name and version to be inside of a + # different object that is not called options. + return {'inputs': self.inputs, 'name': self.name, 'version': self.version} - # if options: - # return options - # else: - # ## is this safe to do? it might turn Hofstaeder... - # return self.__dict__ - #################### - ## original return - #################### - #return {'inputs': self.inputs, - # 'name': self.name, - # 'version': self.version} - ''' + def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) - ''' + diff --git a/ooni/oonicli.py b/ooni/oonicli.py index b9f1dee..aa3cdc0 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -88,6 +88,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
def run(): log.start() + log.debug("Started logging")
if len(sys.argv) == 1: sys.argv.append("--help") @@ -106,3 +107,4 @@ def run(): for idx, cases in enumerate(casesList): orunner = runner.ORunner(cases, options[idx], config) orunner.run() + diff --git a/ooni/reporter.py b/ooni/reporter.py index b856948..14327ef 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -16,7 +16,7 @@ from twisted.python.util import untilConcludes from twisted.trial import reporter from twisted.internet import defer
-from ooni.templates.httpt import BodyReceiver +from ooni.templates.httpt import BodyReceiver, StringProducer from ooni.utils import date, log, geodata
try: @@ -178,9 +178,9 @@ class ReporterFactory(OReporter): @defer.inlineCallbacks def writeHeader(self): self.firstrun = False - (klass, options) = self.options + options = self.options self._writeln("###########################################") - self._writeln("# OONI Probe Report for %s test" % klass.name) + self._writeln("# OONI Probe Report for %s test" % options['name']) self._writeln("# %s" % date.pretty_date()) self._writeln("###########################################")
@@ -207,8 +207,8 @@ class ReporterFactory(OReporter): 'probeLocation': {'city': client_geodata['city'], 'countrycode': client_geodata['countrycode']}, - 'testName': klass.name, - 'testVersion': klass.version, + 'testName': options['name'], + 'testVersion': options['version'], } self.writeYamlLine(test_details) self._writeln('') diff --git a/ooni/runner.py b/ooni/runner.py index d711153..acf5519 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -18,20 +18,13 @@ from twisted.trial.runner import isTestCase from twisted.trial.runner import filenameToModule
from ooni.inputunit import InputUnitFactory -from ooni.nettest import InputTestSuite, NetTestCase +from ooni.nettest import InputTestSuite from ooni.plugoo import tests as oonitests from ooni.reporter import ReporterFactory from ooni.utils import log, date from ooni.utils.legacy import LegacyOONITest from ooni.utils.legacy import start_legacy_test, adapt_legacy_test
- -def isTemplate(obj): - origin = obj.__module__ - if origin.find('templates') >= 0: - return True - return False - def isLegacyTest(obj): """ Returns True if the test in question is written using the OONITest legacy @@ -43,6 +36,58 @@ def isLegacyTest(obj): except TypeError: return False
+def processTest(obj, config): + """ + Process the parameters and :class:`twisted.python.usage.Options` of a + :class:`ooni.nettest.Nettest`. + + :param obj: + An uninstantiated old test, which should be a subclass of + :class:`ooni.plugoo.tests.OONITest`. + :param config: + A configured and instantiated :class:`twisted.python.usage.Options` + class. + """ + + input_file = obj.inputFile + if obj.requiresRoot: + if os.getuid() != 0: + raise Exception("This test requires root to run") + + if obj.optParameters or input_file \ + or obj.advancedOptParameters or obj.optFlags: + + if not obj.optParameters: + obj.optParameters = [] + + if input_file: + obj.optParameters.append(input_file) + + if obj.advancedOptParameters: + log.debug("Got advanced options") + options = obj.advancedOptParameters() + else: + log.debug("Got optParameters") + class Options(usage.Options): + optParameters = obj.optParameters + if obj.optFlags: + log.debug("Got optFlags") + optFlags = obj.optFlags + options = Options() + + options.parseOptions(config['subArgs']) + obj.localOptions = options + + if input_file: + obj.inputFile = options[input_file[0]] + try: + tmp_obj = obj() + tmp_obj.getOptions() + except usage.UsageError: + options.opt_help() + + return obj + def findTestClassesFromConfig(config): """ Takes as input the command line config parameters and returns the test @@ -57,174 +102,76 @@ def findTestClassesFromConfig(config): A list of class objects found in a file or module given on the commandline. """ + filename = config['test'] classes = []
module = filenameToModule(filename) for name, val in inspect.getmembers(module): if isTestCase(val): - if val != NetTestCase and not isTemplate(val): - log.debug("findTestClassesFromConfig: detected %s" - % val.__name__) - classes.append(val) + log.debug("Detected TestCase %s" % val) + classes.append(processTest(val, config)) elif isLegacyTest(val): log.debug("Detected Legacy Test %s" % val) classes.append(adapt_legacy_test(val, config)) return classes
-def makeTestCases(klass, tests, method_prefix=None): +def makeTestCases(klass, tests, method_prefix): """ Takes a class some tests and returns the test cases. method_prefix is how the test case functions should be prefixed with. """ - if not method_prefix: - method_prefix = 'test'
cases = [] for test in tests: - log.debug("makeTestCases: making test case for %s" % test) - method_name = str(method_prefix)+str(test) - log.debug("makeTestCases: using methodName=%s" % method_name) - cases.append(klass(methodName=method_name)) + cases.append(klass(method_prefix+test)) return cases
-def getTestOptions(cls, subargs): - """ - Process the parameters and :class:`twisted.python.usage.Options` of a - :class:`ooni.nettest.Nettest`. - - :param cls: - An subclass of :class:`ooni.nettest.NetTestCase`. - :param config: - A configured and instantiated :class:`twisted.python.usage.Options` - class. - """ - if cls.requiresRoot: - if os.getuid() != 0: - raise Exception("This test requires root to run") - - try: - cls.buildOptions(subargs) - except Exception, e: - log.err(e) - - return cls.local_options - def loadTestsAndOptions(classes, config): """ Takes a list of test classes and returns their testcases and options. Legacy tests will be adapted. """ - from inspect import isclass
method_prefix = 'test' options = [] test_cases = []
- DEPRECATED = LegacyOONITest + _old_klass_type = LegacyOONITest
for klass in classes: - if isinstance(klass, DEPRECATED): + if isinstance(klass, _old_klass_type): try: - cases, opts = processLegacyTest(klass, config) + cases = start_legacy_test(klass) if cases: - log.debug("loadTestsAndOptions: processing cases %s" - % str(cases)) + log.debug("Processing cases") + log.debug(str(cases)) return [], [] test_cases.append(cases) - except Exception, e: log.err(e) + except Exception, e: + log.err(e) else: try: opts = klass.local_options - option.append(opts) + options.append(opts) except AttributeError, ae: options.append([]) log.err(ae) - - elif issubclass(klass, NetTestCase): - try: - cases, opts = processNetTest(klass, config, method_prefix) - except Exception, e: - log.err(e) - else: + else: + tests = reflect.prefixedMethodNames(klass, method_prefix) + if tests: + cases = makeTestCases(klass, tests, method_prefix) test_cases.append(cases) + try: + k = klass() + opts = k.getOptions() options.append(opts) + except AttributeError, ae: + options.append([]) + log.err(ae)
return test_cases, options
-def processNetTest(klass, config, method_prefix): - try: - klass.setUpClass() - except Exception, e: - log.err(e) - - subargs_from_config = config['subArgs'] - log.debug("processNetTest: received subargs from config: %s" - % str(subargs_from_config)) - try: - opts = getTestOptions(klass, subargs_from_config) - except Exception, e: - opts = [] - log.err(e) - - try: - log.debug("processNetTest: processing cases for %s" - % (klass.name if hasattr(klass, 'name') else 'Network Test')) - tests = reflect.prefixedMethodNames(klass, method_prefix) - except Exception, e: - cases = [] - opts = [] - log.err(e) - else: - if tests: - cases = makeTestCases(klass, tests, method_prefix) - log.debug("processNetTest: test %s found cases %s" - % (tests, cases)) - else: - cases = [] - - return cases, opts - -def processLegacyTest(klass, config): - log.msg("Processing cases and options for legacy test %s" - % ( klass.shortName if hasattr(klass, shortName) else 'oonitest' )) - if hasattr(klass, description): - log.msg("%s" % klass.description) - - subcmds = [] - if hasattr(klass, options): ## an unitiated Legacy test - log.debug("%s.options found: %s " % (klass, klass.options)) - try: - assert isclass(klass.options), "legacytest.options is not class" - except AssertionError, ae: - log.debug(ae) - else: - ok = klass.options - ok.parseArgs = lambda x: subcmds.append(x) - try: - opts = ok() - opts.parseOptions(config['subArgs']) - except Exception, e: - log.err(e) - opts = {} - - elif hasattr(klass, local_options): ## we've been initialized already - log.debug("processLegacyTest: %s.local_options found" % str(klass)) - try: - opts = klass.local_options - except AttributeError, ae: opts = {}; log.err(ae) - log.debug("processLegacyTest: opts set to %s" % str(opts)) - - try: - cases = start_legacy_test(klass) - ## XXX we need to get these results into the reporter - if cases: - log.debug("processLegacyTest: found cases: %s" % str(cases)) - return [], [] - except Exception, e: cases = []; log.err(e) - - return cases, opts - class ORunner(object): """ This is a specialized runner used by the ooniprobe command line tool. @@ -237,29 +184,31 @@ class ORunner(object): self.cases = cases self.options = options
- log.debug("ORunner: cases=%s" % type(cases)) - log.debug("ORunner: options=%s" % options) - - try: - first = options.pop(0) - except: - first = options - - if 'inputs' in first: - self.inputs = self.options['inputs'] + assert len(options) != 0, "Length of options is zero!" + except AssertionError, ae: + self.inputs = [] + log.err(ae) else: - log.msg("Could not find inputs!") - self.inputs = [None] + try: + first = options.pop(0) + except: + first = options + + if 'inputs' in first: + self.inputs = options['inputs'] + else: + log.msg("Could not find inputs!") + log.msg("options[0] = %s" % first) + self.inputs = [None]
try: reportFile = open(config['reportfile'], 'a+') except: filename = 'report_'+date.timestamp()+'.yaml' reportFile = open(filename, 'a+') - self.reporterFactory = ReporterFactory( - reportFile, testSuite=self.baseSuite(self.cases) - ) + self.reporterFactory = ReporterFactory(reportFile, + testSuite=self.baseSuite(self.cases))
def runWithInputUnit(self, input_unit): idx = 0 diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index acad538..2e2d9e1 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -6,12 +6,14 @@ import random
from zope.interface import implements + from twisted.python import usage from twisted.plugin import IPlugin from twisted.internet import protocol, defer from twisted.internet.ssl import ClientContextFactory
from twisted.web.http_headers import Headers +from twisted.web.iweb import IBodyProducer
from ooni.nettest import NetTestCase from ooni.utils import log @@ -28,6 +30,23 @@ useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Geck ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")]
+class StringProducer(object): + implements(IBodyProducer) + + def __init__(self, body): + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + consumer.write(self.body) + return defer.succeed(None) + + def pauseProducing(self): + pass + + def stopProducing(self): + pass + class BodyReceiver(protocol.Protocol): def __init__(self, finished): self.finished = finished diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 9c80f62..fe97ca3 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -21,8 +21,8 @@ import sys from twisted.internet import defer from zope.interface import implements
-from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher -from ooni.lib.txtorcon import TorState, TorConfig +from txtorcon import CircuitListenerMixin, IStreamAttacher +from txtorcon import TorState, TorConfig from ooni.utils import log from ooni.utils.timer import deferred_timeout, TimeoutError