[tor-commits] [ooni-probe/master] Port Bridget to the new API
art at torproject.org
art at torproject.org
Mon Nov 5 19:46:50 UTC 2012
commit de0c47f7152b3a0a9dd2baf27ce2b028819e3f63
Author: Arturo Filastò <arturo at filasto.net>
Date: Mon Nov 5 20:23:27 2012 +0100
Port Bridget to the new API
* Unbreak the new API and make sure tests work as they should
* Add more advanced parameters for the test input as requested by Isis
---
nettests/bridget.py | 89 +++--------
ooni/__init__.py | 18 +--
ooni/inputunit.py | 133 +---------------
ooni/nettest.py | 399 ++++++++---------------------------------------
ooni/oonicli.py | 2 +
ooni/reporter.py | 10 +-
ooni/runner.py | 237 +++++++++++-----------------
ooni/templates/httpt.py | 19 +++
ooni/utils/onion.py | 4 +-
9 files changed, 221 insertions(+), 690 deletions(-)
diff --git a/nettests/bridget.py b/nettests/bridget.py
index a334747..74d1d47 100644
--- a/nettests/bridget.py
+++ b/nettests/bridget.py
@@ -20,23 +20,25 @@ import os
import sys
from twisted.python import usage
-from twisted.plugin import IPlugin
from twisted.internet import defer, error, reactor
-from zope.interface import implements
+
+from ooni import nettest
from ooni.utils import log, date
from ooni.utils.config import ValueChecker
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.plugoo.assets import Asset, MissingAssetException
from ooni.utils.onion import TxtorconImportError
from ooni.utils.onion import PTNoBridgesException, PTNotFoundException
+
try:
from ooni.utils.onion import parse_data_dir
except:
log.msg("Please go to /ooni/lib and do 'make txtorcon' to run this test!")
+class MissingAssetException(Exception):
+ pass
+
class RandomPortException(Exception):
"""Raised when using a random port conflicts with configured ports."""
def __init__(self):
@@ -83,18 +85,7 @@ class BridgetArgs(usage.Options):
if self['torpath']:
ValueChecker.file_check(self['torpath'])
-class BridgetAsset(Asset):
- """Class for parsing bridget Assets ignoring commented out lines."""
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
- def parse_line(self, line):
- if line.startswith('#'):
- return
- else:
- return line.replace('\n','')
-
-class BridgetTest(OONITest):
+class BridgetTest(nettest.TestCase):
"""
XXX fill me in
@@ -114,15 +105,13 @@ class BridgetTest(OONITest):
:ivar tor_binary:
Path to the Tor binary to use, e.g. \'/usr/sbin/tor\'
"""
- implements(IPlugin, ITest)
+ name = "bridget"
+ author = "Isis Lovecruft <isis at torproject.org>"
+ version = "0.1"
+ description = "Use a Tor process to test connecting to bridges or relays"
+ advancedOptParameters = BridgetArgs
- shortName = "bridget"
- description = "Use a Tor process to test connecting to bridges or relays"
- requirements = None
- options = BridgetArgs
- blocking = False
-
- def initialize(self):
+ def setUp(self):
"""
Extra initialization steps. We only want one child Tor process
running, so we need to deal with most of the TorConfig() only once,
@@ -134,14 +123,16 @@ class BridgetTest(OONITest):
self.tor_binary = '/usr/sbin/tor'
self.data_directory = None
- def __make_asset_list__(opt, lst):
+ def read_from_file(filename):
log.msg("Loading information from %s ..." % opt)
- with open(opt) as opt_file:
- for line in opt_file.readlines():
+ with open(filename) as fp:
+ lst = []
+ for line in fp.readlines():
if line.startswith('#'):
continue
else:
lst.append(line.replace('\n',''))
+ return lst
def __count_remaining__(which):
total, reach, unreach = map(lambda x: which[x],
@@ -168,23 +159,24 @@ class BridgetTest(OONITest):
self.relays['remaining'] = lambda: __count_remaining__(self.relays)
self.relays['current'] = None
- if self.local_options:
+ if self.localOptions:
try:
- from ooni.lib.txtorcon import TorConfig
+ from txtorcon import TorConfig
except ImportError:
raise TxtorconImportError
else:
self.config = TorConfig()
finally:
- options = self.local_options
+ options = self.localOptions
if options['bridges']:
self.config.UseBridges = 1
- __make_asset_list__(options['bridges'], self.bridges['all'])
+ self.bridges['all'] = read_from_file(options['bridges'])
if options['relays']:
## first hop must be in TorState().guards
+ # XXX where is this defined?
self.config.EntryNodes = ','.join(relay_list)
- __make_asset_list__(options['relays'], self.relays['all'])
+ self.relays['all'] = read_from_file(options['relays'])
if options['socks']:
self.socks_port = options['socks']
if options['control']:
@@ -215,33 +207,7 @@ class BridgetTest(OONITest):
self.config.ControlPort = self.control_port
self.config.CookieAuthentication = 1
- def __load_assets__(self):
- """
- Load bridges and/or relays from files given in user options. Bridges
- should be given in the form IP:ORport. We don't want to load these as
- assets, because it's inefficient to start a Tor process for each one.
-
- We cannot use the Asset model, because that model calls
- self.experiment() with the current Assets, which would be one relay
- and one bridge, then it gives the defer.Deferred returned from
- self.experiment() to self.control(), which means that, for each
- (bridge, relay) pair, experiment gets called again, which instantiates
- an additional Tor process that attempts to bind to the same
- ports. Thus, additionally instantiated Tor processes return with
- RuntimeErrors, which break the final defer.chainDeferred.callback(),
- sending it into the errback chain.
- """
- assets = {}
- if self.local_options:
- if self.local_options['bridges']:
- assets.update({'bridge':
- BridgetAsset(self.local_options['bridges'])})
- if self.local_options['relays']:
- assets.update({'relay':
- BridgetAsset(self.local_options['relays'])})
- return assets
-
- def experiment(self, args):
+ def test_bridget(self):
"""
if bridges:
1. configure first bridge line
@@ -465,7 +431,7 @@ class BridgetTest(OONITest):
#self.reactor.run()
return state
- def startTest(self, args):
+ def disabled_startTest(self, args):
"""
Local override of :meth:`OONITest.startTest` to bypass calling
self.control.
@@ -483,9 +449,6 @@ class BridgetTest(OONITest):
self.d.addCallbacks(self.finished, log.err)
return self.d
-## So that getPlugins() can register the Test:
-#bridget = BridgetTest(None, None, None)
-
## ISIS' NOTES
## -----------
## TODO:
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 2f7c19f..f025d30 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,11 +1,3 @@
-__all__ = [
- 'config', 'inputunit', 'kit',
- 'lib', 'nettest', 'oonicli', 'reporter',
- 'runner', 'templates', 'utils',
- ## XXX below are legacy related modules
- 'ooniprobe', 'plugoo', 'plugins'
- ]
-
from . import config
from . import inputunit
from . import kit
@@ -17,7 +9,13 @@ from . import runner
from . import templates
from . import utils
-## XXX below are legacy related modules
+# XXX below are legacy related modules
#from . import ooniprobe
-from . import plugoo
+#from . import plugoo
#from . import plugins
+
+__all__ = ['config', 'inputunit', 'kit',
+ 'lib', 'nettest', 'oonicli', 'reporter',
+ 'runner', 'templates', 'utils']
+ # XXX below are legacy related modules
+ #'ooniprobe', 'plugoo', 'plugins']
diff --git a/ooni/inputunit.py b/ooni/inputunit.py
index 054b3a9..e5b6187 100644
--- a/ooni/inputunit.py
+++ b/ooni/inputunit.py
@@ -1,31 +1,5 @@
-# -*- coding: utf-8 -*-
-#
-# inputunit.py
-# ------------
-# Classes and function for working with OONI TestCase inputs.
-#
-# @authors: Arturo Filasto, Isis Lovecruft
-# @version: 0.1.0-alpha
-# @license: see included LICENSE file
-# @copyright: 2012 Arturo Filasto, Isis Lovecruft, The Tor Project Inc.
-#
-
from twisted.trial import unittest
-from zope.interface.exceptions import BrokenImplementation
-
-
-def simpleInputUnitProcessor(input_unit):
- """A simple InputUnit generator without parsing."""
- try:
- assert hasattr(input_unit, '__iter__'), "inputs must be iterable!"
- except AssertionError, ae:
- raise BrokenImplementation, ae
- else:
- for i in input_unit:
- yield i
-
-
class PatchedPyUnitResultAdapter(unittest.PyUnitResultAdapter):
def __init__(self, original):
"""
@@ -39,8 +13,8 @@ unittest.PyUnitResultAdapter = PatchedPyUnitResultAdapter
class InputUnitFactory(object):
"""
- This is a factory that takes the size of input units to be generated and a
- set of units that is a python iterable item, and outputs InputUnit objects
+ This is a factory that takes the size of input units to be generated a set
+ of units that is a python iterable item and outputs InputUnit objects
containing inputUnitSize elements.
This object is a python iterable, this means that it does not need to keep
@@ -74,6 +48,7 @@ class InputUnitFactory(object):
return InputUnit(input_unit_elements)
+
class InputUnit(object):
"""
This is a python iterable object that contains the input elements to be
@@ -101,107 +76,5 @@ class InputUnit(object):
def append(self, input):
self._inputs.append(input)
-class IUProcessorExit(GeneratorExit):
- """InputUnitProcessor has exited."""
-
-class IUProcessorAborted(Exception):
- """The InputUnitProcessor was aborted with Inputs remaining."""
-
-class InputUnitProcessor(InputUnit):
- """
- Create a generator for returning inputs one-by-one from a
- :class:`InputUnit` (or any other iterable defined within an instance of
- :class:`ooni.nettest.NetTestCase`), and a generator function.
-
- The :ivar:generator can be a custom generator, or chain of generators, for
- customized parsing of an InputUnit, or it can be an imported
- function. There are useful imported functions in the builtin
- :mod:`itertools`. If no :ivar:`generator` is given, the default one strips
- whitespace characters, then returns the Input if the input does not begin
- with a crunch (#) or bang (!). :)
-
- If :ivar:catchStopIter is True, then catch any StopIterations and return a
- 2-tuple containing (boolean, list(unprocessed)):
-
- (True, None) when :ivar:iterable is empty, or
- (False, [unprocessed]) if :ivar:iterable was not empty.
-
- If :ivar:catchStopIter is False (default), then we catch the StopIteration
- exception, mark :attr:`empty` as 'True', and reraise the StopIteration.
- XXX fill me in with parameter details
- XXX I'm not sure if we need this class anymore
- """
- empty = False
-
- def __init__(self, iterable, input_filter=None, catch_err=False):
- """
- Create an InputUnitProcessor.
-
- XXX fill me in
- """
- from itertools import takewhile
- from types import GeneratorType
-
- assert hasattr(iterable, "__iter__"), "That's not an iterable!"
-
- self._iterable = iterable
- self._infilter = input_filter
- self._noerr = catch_err
- self._empty = self.empty
- def __len__(self):
- return len(self.iterable)
-
- def __unprocessed__(self):
- if not self.isdone():
- unprocessed = \
- [u for u in self._unit if self._unit is not StopIteration]
- return (False, unprocessed)
- else:
- return (True, None)
-
- def isdone(self):
- return self._empty
-
- def throw(self, exception, message="", traceback=None):
- try:
- raise exception, message, traceback
- except:
- yield self._iterable.next()
-
- @staticmethod
- def __strip__(x):
- return x.strip()
-
- @staticmethod
- def _default_input_filter(x):
- if not x.startswith('#') and not x.startswith('!'):
- return True
- else:
- return False
-
- def __make_filter__(self):
- if self.input_filter and hasattr(self.input_filter, "__call__"):
- return self.input_filter
- else:
- return self._default_input_filter
-
- def finish(self):
- if self._noerr:
- return self.__unprocessed__()
- else:
- if not self.isdone():
- raise IUProcessorAborted
- else:
- raise IUProcessorExit
-
- def process(self):
- carbon = self.__make_filter__()
- try:
- yield takewhile(carbon,
- [i for i in self.__strip__(self._iterable)]
- ).next()
- except StopIteration, si:
- self._empty = True
- yield self.finish()
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 94c8c56..c2a8575 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -1,17 +1,8 @@
# -*- encoding: utf-8 -*-
#
-# nettest.py
-# ------------------->
-#
-# :authors: Arturo "hellais" Filastò <art at fuffa.org>,
-# Isis Lovecruft <isis at torproject.org>
+# :authors: Arturo "hellais" Filastò <art at fuffa.org>
# :licence: see LICENSE
-# :copyright: 2012 Arturo Filasto, Isis Lovecruft
-# :version: 0.1.0-alpha
-#
-# <-------------------
-from functools import partial
import sys
import os
import itertools
@@ -25,7 +16,6 @@ from ooni.utils import log
pyunit = __import__('unittest')
-
class InputTestSuite(pyunit.TestSuite):
"""
This in an extension of a unittest test suite. It adds support for inputs
@@ -66,306 +56,26 @@ class InputTestSuite(pyunit.TestSuite):
self._idx += 1
return result
-class NetTestAdaptor(unittest.TestCase):
- """
- XXX fill me in
- """
-
- # @classmethod
- # def __new__(cls, *args, **kwargs):
- # try:
- # setUpClass()
- # except Exception, e:
- # log.debug("NetTestAdaptor: constructor could not find setUpClass")
- # log.err(e)
- # return super( NetTestAdaptor, cls ).__new__(cls, *args, **kwargs)
-
- def __init__(self, *args, **kwargs):
- """
- If you override me, you must call
-
- ``super(NetTestCase, self).__init__(*args, **kwargs)``
-
- at the beginning of your __init__ method. Keyword arguments passed to
- the above statement become attributes of the adaptor, and can be used
- to alter the logic of input handling and parent class instantiation.
- Therefore, You probably do not need to pass me any keyword arguments
- when calling me, i.e. using ``(*args, **kwargs)`` will work just fine.
- """
- log.debug("NetTestAdaptor: created")
- if kwargs:
- if 'methodName' in kwargs:
- log.debug("NetTestAdaptor: found 'methodName' in kwargs")
- log.debug("NetTestAdaptor: calling unittest.TestCase.__init()")
- super( NetTestAdaptor, self ).__init__(
- methodName=kwargs['methodName'] )
- else:
- log.debug("NetTestAdaptor: calling unittest.TestCase.__init()")
- super( NetTestAdaptor, self ).__init__( )
-
- for key, value in kwargs.items(): ## Let subclasses define their
- if key != 'methodName': ## instantiation without
- if not hasattr(self, key): ## overriding parent classes
- log.debug("NetTestAdaptor: calling setattr(self,%s,%s)"
- % (key, value) )
- setattr(self, key, value)
-
- #setattr(self, "copyattr", __copy_attr__)
-
- ## Internal attribute copies:
- #self._input_parser = copyattr("inputParser", alt=__input_parser__)
- #self._nettest_name = copyattr("name", alt="NetTestAdaptor"))
-
- #self.setUpClass(self.__class__)
- #if hasattr(self, parsed_inputs):
- # self.inputs = self.parsed_inputs
- #else:
- # log.debug("Unable to find parsed inputs")
-
- @staticmethod
- def __copyattr__(obj, old, new=None, alt=None):
- """
- Assign me to a new attribute name to have a copy of the old
- attribute, if it exists.
-
- Example:
- >>> self.sun = "black"
- >>> self._watermelon = __copyattr__(self, "sun")
- >>> self._clocknoise = __copyattr__(self, "sound")
- >>> print self._watermelon
- black
- >>> print self._clocknoise
-
- :param old:
- A string representing the name of the old attribute
- :param new:
- (Optional) A string to set as the new attribute name.
- :param alt:
- (Optional) An alternate value to return if the old
- attribute is not found.
- :return:
- If :param:`old` is found, I return it's value.
-
- If :param:`old` is not found:
- If :param:`alt` is given, I return :param:`alt`.
- Else, I return None.
-
- If :param:`new` is set, I do not return anything, and
- instead I set the new attribute's name to :param:`name`
- and it's value to the value which I would have otherwise
- returned.
- """
- if not new:
- if not alt:
- if hasattr(obj, old):
- return getattr(obj, old)
- return
- else:
- if hasattr(obj, old):
- return getattr(obj, old)
- return alt
- else:
- if not alt:
- if hasattr(obj, old):
- _copy = getattr(obj, old)
- else:
- copy = None
- setattr(obj, new, _copy)
- else:
- if hasattr(obj, old):
- _copy = getattr(obj, old)
- else:
- _copy = alt
- setattr(obj, new, _copy)
-
- def copyattr(self, *args, **kwargs):
- if len(args) >= 1:
- _copy = partial(__copyattr__, args[0])
- if len(args) == 2:
- return _copy(new=args[1])
- elif len(args) == 3:
- return _copy(new=args[1], alt=args[2])
- elif kwargs:
- return _copy(kwargs)
- else:
- return
-
- @staticmethod
- def __input_parser__(one_input): return one_input
-
- @classmethod
- def __get_inputs__(cls):
- """
- I am called during class setup and you probably should not override
- me. I gather the internal inputs from :class:`NetTestCase` attributes
- and pass them through :meth:`NetTestCase.inputParser`. If you are
- looking for a way to parse inputs from inputFile, see
- :meth:`inputParser`. If :class:`NetTestCase` has an attribute
- :attr:`inputFile`, I also handle opening that file, striping each line
- of whitespace, and then sending the line to
- :meth:`NetTestCase.inputParser`.
-
- All inputs which I find, both statically set inputs and those returned
- from processing an inputFile, I add to a list :ivar:`parsed`, after
- parsing them. I return :ivar:`parsed`:
-
- :ivar parsed:
- A list of parsed inputs.
- :return:
- :ivar:`parsed`.
- """
- parsed = []
-
- if cls._raw_inputs:
- if isinstance(cls._raw_inputs, (list, tuple,)):
- if len(cls._raw_inputs) > 0:
- if len(cls._raw_inputs) == 1 and cls._raw_inputs[0] is None:
- pass ## don't burn cycles on testing null inputs
- else:
- log.msg("Received direct inputs:\n%s" % cls._raw_inputs)
- parsed.extend(
- [cls._input_parser(x) for x in cls._raw_inputs])
- elif isinstance(cls._raw_inputs, str):
- separated = cls._raw_inputs.translate(None, ',') ## space delineates
- inputlist = separated.split(' ')
- parsed.extend([cls._input_parser(x) for x in inputlist])
- else:
- log.debug("inputs not string or list; type: %s"
- % type(cls._raw_inputs))
-
- if cls.subarg_inputs:
- log.debug("NetTestAdaptor: __get_inputs__ found subarg_inputs=%s"
- % cls.subarg_inputs)
- parsed.extend([cls._input_parser(x) for x in cls.subarg_inputs])
-
- if cls._input_file:
- try:
- log.debug("NetTestAdaptor: __get_inputs__ Opening input file")
- fp = open(cls._input_file)
- except:
- log.debug("NetTestAdaptor: __get_inputs__ Couldn't open input file")
- else:
- log.debug("NetTestAdaptor: __get_inputs__ Running input file processor")
- lines = [line.strip() for line in fp.readlines()]
- fp.close()
-
- ## add to what we've already parsed, if any:
- log.debug("NetTestAdaptor: __get_inputs__ Parsing lines from input file")
- parsed.extend([cls._input_parser(ln) for ln in lines])
- else:
- log.debug("NetTestAdaptor: %s specified that it doesn't need inputFile."
- % cls._nettest_name)
-
- return parsed
-
- @classmethod
- def __getopt__(cls, parseArgs=None):
- """
- Constuctor for a custom t.p.usage.Options class, per NetTestCase.
-
- old code from runner.py:
- opts = Options()
- opts.parseOptions(config['subArgs'])
- cls.localOptions = opts
- """
- if cls._testopt_params or cls._input_file:
- if not cls._testopt_params:
- cls._testopt_params = []
-
- if cls._input_file:
- cls._testopt_params.append(cls.input_file)
-
- class NetTestOptions(usage.Options):
- """Per NetTestCase Options class."""
- optParameters = cls._testopt_params
- optFlags = cls._testopt_flags
- subOptions = cls._sub_options
- subCommands = cls._sub_commands
- defaultSubCommand = cls._default_subcmd
- ## XXX i'm not sure if this part will work:
- parseArgs = lambda a: cls.subarg_inputs.append(a)
-
- def opt_version(self):
- """Display test version and exit."""
- print "Test version: ", cls._nettest_version
- sys.exit(0)
-
- options = NetTestOptions()
- return options
-
- #if cls._input_file:
- # cls._input_file = cls.options[cls._input_file[0]]
-
- @classmethod
- def addSubArgToInputs(cls, subarg):
- cls.subarg_inputs.append(subarg)
-
- @classmethod
- def buildOptions(cls, from_global):
- log.debug("NetTestAdaptor: getTestOptions called")
- options = cls.__getopt__()
- log.debug("NetTestAdaptor: getTestOptions: cls.options = %s"
- % options)
- options.parseOptions(from_global)
- setattr(cls, "local_options", options)
- log.debug("NetTestAdaptor: getTestOptions: cls.local_options = %s"
- % cls.local_options)
-
- @classmethod
- def setUpClass(cls):
- """
- Create a NetTestCase. To add futher setup steps before a set of tests
- in a TestCase instance run, create a function called 'setUp'.
-
- Class attributes, such as `report`, `optParameters`, `name`, and
- `author` should be overriden statically as class attributes in any
- subclass of :class:`ooni.nettest.NetTestCase`, so that the calling
- functions during NetTestCase class setup can handle them correctly.
- """
-
- log.debug("NetTestAdaptor: setUpClass called")
-
- ## These internal inputs are for handling inputs and inputFile
- cls._raw_inputs = cls.__copyattr__(cls, "inputs")
- cls._input_file = cls.__copyattr__(cls, "inputFile")
- cls._input_parser = cls.__copyattr__(cls, "inputParser",
- alt=cls.__input_parser__)
- cls._nettest_name = cls.__copyattr__(cls, "name", alt="NetTestAdaptor")
-
- ## This creates a class attribute with all of the parsed inputs,
- ## which the instance will later set to be `self.inputs`.
- cls.parsed_inputs = cls.__get_inputs__()
- cls.subarg_inputs = cls.__copyattr__(cls, "subarg_inputs",
- alt=[])
-
- ## XXX we should handle options generation here
- cls._testopt_params = cls.__copyattr__(cls, "optParameters")
- cls._testopt_flags = cls.__copyattr__(cls, "optFlags")
- cls._sub_options = cls.__copyattr__(cls, "subOptions")
- cls._sub_commands = cls.__copyattr__(cls, "subCommands")
- cls._default_subcmd = cls.__copyattr__(cls, "defaultSubCommand")
- cls._nettest_version = cls.__copyattr__(cls, "version")
-
-class NetTestCase(NetTestAdaptor):
+class NetTestCase(unittest.TestCase):
"""
This is the monad of the OONI nettest universe. When you write a nettest
you will subclass this object.
* inputs: can be set to a static set of inputs. All the tests (the methods
- starting with the "test_" prefix) will be run once per input. At every
- run the _input_ attribute of the TestCase instance will be set to the
- value of the current iteration over inputs. Any python iterable object
- can be set to inputs.
+ starting with the "test_" prefix) will be run once per input. At every run
+ the _input_ attribute of the TestCase instance will be set to the value of
+ the current iteration over inputs. Any python iterable object can be set
+ to inputs.
- * inputFile: attribute should be set to an array containing the command
- line argument that should be used as the input file. Such array looks
- like this:
+ * inputFile: attribute should be set to an array containing the command line
+ argument that should be used as the input file. Such array looks like
+ this:
- ``["commandlinearg", "c", "The description"]``
+ ``["commandlinearg", "c", "default value" "The description"]``
- The second value of such arrray is the shorthand for the command line
- arg. The user will then be able to specify inputs to the test via:
+ The second value of such arrray is the shorthand for the command line arg.
+ The user will then be able to specify inputs to the test via:
``ooniprobe mytest.py --commandlinearg path/to/file.txt``
@@ -374,9 +84,9 @@ class NetTestCase(NetTestAdaptor):
``ooniprobe mytest.py -c path/to/file.txt``
- * inputParser: should be set to a function that takes as argument an
- iterable containing test inputs and it simply parses those inputs and
- returns them back to test instance.
+ * inputProcessor: should be set to a function that takes as argument an
+ open file descriptor and it will return the input to be passed to the test
+ instance.
* name: should be set to the name of the test.
@@ -388,26 +98,44 @@ class NetTestCase(NetTestAdaptor):
* version: is the version string of the test.
* requiresRoot: set to True if the test must be run as root.
+
+ * optFlags: is assigned a list of lists. Each list represents a flag parameter, as so:
+
+ optFlags = [['verbose', 'v', 'Makes it tell you what it doing.'], | ['quiet', 'q', 'Be vewy vewy quiet.']]
+
+ As you can see, the first item is the long option name (prefixed with
+ '--' on the command line), followed by the short option name (prefixed with
+ '-'), and the description. The description is used for the built-in handling of
+ the --help switch, which prints a usage summary.
+
+
+ * optParameters: is much the same, except the list also contains a default value:
+
+ | optParameters = [['outfile', 'O', 'outfile.log', 'Description...']]
+
+ * advancedOptParameters: a subclass of twisted.python.usage.Options for more advanced command line arguments fun.
+
"""
name = "I Did Not Change The Name"
author = "Jane Doe <foo at example.com>"
- version = "0.0.0"
+ version = "0"
+ inputs = [None]
inputFile = None
- inputs = [None]
report = {}
report['errors'] = []
- optParameters = None
optFlags = None
- subCommands = None
+ optParameters = None
+ advancedOptParameters = None
+
requiresRoot = False
def deferSetUp(self, ignored, result):
"""
- If we have the reporterFactory set we need to write the header. If
- such method is not present we will only run the test skipping header
+ If we have the reporterFactory set we need to write the header. If such
+ method is not present we will only run the test skipping header
writing.
"""
if result.reporterFactory.firstrun:
@@ -420,35 +148,34 @@ class NetTestCase(NetTestAdaptor):
log.debug("Not first run. Running test setup directly")
return unittest.TestCase.deferSetUp(self, ignored, result)
- def inputParser(self, inputs):
- """Replace me with a custom function for parsing inputs."""
- log.debug("Running custom input processor")
- return inputs
+ def inputProcessor(self, fp):
+ log.debug("Running default input processor")
+ for x in fp.readlines():
+ yield x.strip()
+ fp.close()
def getOptions(self):
log.debug("Getting options for test")
-
- if self.local_options:
- log.debug("NetTestCase: getOptions: self.local_options=%s"
- % str(self.local_options))
- else:
- log.debug("could not find cls.localOptions!")
-
- return {'inputs': self.parsed_inputs,
+ if self.inputFile:
+ try:
+ assert isinstance(self.inputFile, str)
+ except AssertionError, ae:
+ log.err(ae)
+ else:
+ if os.path.isfile(self.inputFile):
+ print self.inputFile
+ fp = open(self.inputFile)
+ self.inputs = self.inputProcessor(fp)
+ elif not self.inputs[0]:
+ pass
+ elif self.inputFile:
+ raise usage.UsageError("No input file specified!")
+ # XXX perhaps we may want to name and version to be inside of a
+ # different object that is not called options.
+ return {'inputs': self.inputs,
'name': self.name,
'version': self.version}
- # if options:
- # return options
- # else:
- # ## is this safe to do? it might turn Hofstaeder...
- # return self.__dict__
- ####################
- ## original return
- ####################
- #return {'inputs': self.inputs,
- # 'name': self.name,
- # 'version': self.version}
- '''
+
def __repr__(self):
return "<%s inputs=%s>" % (self.__class__, self.inputs)
- '''
+
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index b9f1dee..aa3cdc0 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -88,6 +88,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
def run():
log.start()
+ log.debug("Started logging")
if len(sys.argv) == 1:
sys.argv.append("--help")
@@ -106,3 +107,4 @@ def run():
for idx, cases in enumerate(casesList):
orunner = runner.ORunner(cases, options[idx], config)
orunner.run()
+
diff --git a/ooni/reporter.py b/ooni/reporter.py
index b856948..14327ef 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -16,7 +16,7 @@ from twisted.python.util import untilConcludes
from twisted.trial import reporter
from twisted.internet import defer
-from ooni.templates.httpt import BodyReceiver
+from ooni.templates.httpt import BodyReceiver, StringProducer
from ooni.utils import date, log, geodata
try:
@@ -178,9 +178,9 @@ class ReporterFactory(OReporter):
@defer.inlineCallbacks
def writeHeader(self):
self.firstrun = False
- (klass, options) = self.options
+ options = self.options
self._writeln("###########################################")
- self._writeln("# OONI Probe Report for %s test" % klass.name)
+ self._writeln("# OONI Probe Report for %s test" % options['name'])
self._writeln("# %s" % date.pretty_date())
self._writeln("###########################################")
@@ -207,8 +207,8 @@ class ReporterFactory(OReporter):
'probeLocation': {'city': client_geodata['city'],
'countrycode':
client_geodata['countrycode']},
- 'testName': klass.name,
- 'testVersion': klass.version,
+ 'testName': options['name'],
+ 'testVersion': options['version'],
}
self.writeYamlLine(test_details)
self._writeln('')
diff --git a/ooni/runner.py b/ooni/runner.py
index d711153..acf5519 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -18,20 +18,13 @@ from twisted.trial.runner import isTestCase
from twisted.trial.runner import filenameToModule
from ooni.inputunit import InputUnitFactory
-from ooni.nettest import InputTestSuite, NetTestCase
+from ooni.nettest import InputTestSuite
from ooni.plugoo import tests as oonitests
from ooni.reporter import ReporterFactory
from ooni.utils import log, date
from ooni.utils.legacy import LegacyOONITest
from ooni.utils.legacy import start_legacy_test, adapt_legacy_test
-
-def isTemplate(obj):
- origin = obj.__module__
- if origin.find('templates') >= 0:
- return True
- return False
-
def isLegacyTest(obj):
"""
Returns True if the test in question is written using the OONITest legacy
@@ -43,6 +36,58 @@ def isLegacyTest(obj):
except TypeError:
return False
+def processTest(obj, config):
+ """
+ Process the parameters and :class:`twisted.python.usage.Options` of a
+ :class:`ooni.nettest.Nettest`.
+
+ :param obj:
+ An uninstantiated old test, which should be a subclass of
+ :class:`ooni.plugoo.tests.OONITest`.
+ :param config:
+ A configured and instantiated :class:`twisted.python.usage.Options`
+ class.
+ """
+
+ input_file = obj.inputFile
+ if obj.requiresRoot:
+ if os.getuid() != 0:
+ raise Exception("This test requires root to run")
+
+ if obj.optParameters or input_file \
+ or obj.advancedOptParameters or obj.optFlags:
+
+ if not obj.optParameters:
+ obj.optParameters = []
+
+ if input_file:
+ obj.optParameters.append(input_file)
+
+ if obj.advancedOptParameters:
+ log.debug("Got advanced options")
+ options = obj.advancedOptParameters()
+ else:
+ log.debug("Got optParameters")
+ class Options(usage.Options):
+ optParameters = obj.optParameters
+ if obj.optFlags:
+ log.debug("Got optFlags")
+ optFlags = obj.optFlags
+ options = Options()
+
+ options.parseOptions(config['subArgs'])
+ obj.localOptions = options
+
+ if input_file:
+ obj.inputFile = options[input_file[0]]
+ try:
+ tmp_obj = obj()
+ tmp_obj.getOptions()
+ except usage.UsageError:
+ options.opt_help()
+
+ return obj
+
def findTestClassesFromConfig(config):
"""
Takes as input the command line config parameters and returns the test
@@ -57,174 +102,76 @@ def findTestClassesFromConfig(config):
A list of class objects found in a file or module given on the
commandline.
"""
+
filename = config['test']
classes = []
module = filenameToModule(filename)
for name, val in inspect.getmembers(module):
if isTestCase(val):
- if val != NetTestCase and not isTemplate(val):
- log.debug("findTestClassesFromConfig: detected %s"
- % val.__name__)
- classes.append(val)
+ log.debug("Detected TestCase %s" % val)
+ classes.append(processTest(val, config))
elif isLegacyTest(val):
log.debug("Detected Legacy Test %s" % val)
classes.append(adapt_legacy_test(val, config))
return classes
-def makeTestCases(klass, tests, method_prefix=None):
+def makeTestCases(klass, tests, method_prefix):
"""
Takes a class some tests and returns the test cases. method_prefix is how
the test case functions should be prefixed with.
"""
- if not method_prefix:
- method_prefix = 'test'
cases = []
for test in tests:
- log.debug("makeTestCases: making test case for %s" % test)
- method_name = str(method_prefix)+str(test)
- log.debug("makeTestCases: using methodName=%s" % method_name)
- cases.append(klass(methodName=method_name))
+ cases.append(klass(method_prefix+test))
return cases
-def getTestOptions(cls, subargs):
- """
- Process the parameters and :class:`twisted.python.usage.Options` of a
- :class:`ooni.nettest.Nettest`.
-
- :param cls:
- An subclass of :class:`ooni.nettest.NetTestCase`.
- :param config:
- A configured and instantiated :class:`twisted.python.usage.Options`
- class.
- """
- if cls.requiresRoot:
- if os.getuid() != 0:
- raise Exception("This test requires root to run")
-
- try:
- cls.buildOptions(subargs)
- except Exception, e:
- log.err(e)
-
- return cls.local_options
-
def loadTestsAndOptions(classes, config):
"""
Takes a list of test classes and returns their testcases and options.
Legacy tests will be adapted.
"""
- from inspect import isclass
method_prefix = 'test'
options = []
test_cases = []
- DEPRECATED = LegacyOONITest
+ _old_klass_type = LegacyOONITest
for klass in classes:
- if isinstance(klass, DEPRECATED):
+ if isinstance(klass, _old_klass_type):
try:
- cases, opts = processLegacyTest(klass, config)
+ cases = start_legacy_test(klass)
if cases:
- log.debug("loadTestsAndOptions: processing cases %s"
- % str(cases))
+ log.debug("Processing cases")
+ log.debug(str(cases))
return [], []
test_cases.append(cases)
- except Exception, e: log.err(e)
+ except Exception, e:
+ log.err(e)
else:
try:
opts = klass.local_options
- option.append(opts)
+ options.append(opts)
except AttributeError, ae:
options.append([])
log.err(ae)
-
- elif issubclass(klass, NetTestCase):
- try:
- cases, opts = processNetTest(klass, config, method_prefix)
- except Exception, e:
- log.err(e)
- else:
+ else:
+ tests = reflect.prefixedMethodNames(klass, method_prefix)
+ if tests:
+ cases = makeTestCases(klass, tests, method_prefix)
test_cases.append(cases)
+ try:
+ k = klass()
+ opts = k.getOptions()
options.append(opts)
+ except AttributeError, ae:
+ options.append([])
+ log.err(ae)
return test_cases, options
-def processNetTest(klass, config, method_prefix):
- try:
- klass.setUpClass()
- except Exception, e:
- log.err(e)
-
- subargs_from_config = config['subArgs']
- log.debug("processNetTest: received subargs from config: %s"
- % str(subargs_from_config))
- try:
- opts = getTestOptions(klass, subargs_from_config)
- except Exception, e:
- opts = []
- log.err(e)
-
- try:
- log.debug("processNetTest: processing cases for %s"
- % (klass.name if hasattr(klass, 'name') else 'Network Test'))
- tests = reflect.prefixedMethodNames(klass, method_prefix)
- except Exception, e:
- cases = []
- opts = []
- log.err(e)
- else:
- if tests:
- cases = makeTestCases(klass, tests, method_prefix)
- log.debug("processNetTest: test %s found cases %s"
- % (tests, cases))
- else:
- cases = []
-
- return cases, opts
-
-def processLegacyTest(klass, config):
- log.msg("Processing cases and options for legacy test %s"
- % ( klass.shortName if hasattr(klass, shortName) else 'oonitest' ))
- if hasattr(klass, description):
- log.msg("%s" % klass.description)
-
- subcmds = []
- if hasattr(klass, options): ## an unitiated Legacy test
- log.debug("%s.options found: %s " % (klass, klass.options))
- try:
- assert isclass(klass.options), "legacytest.options is not class"
- except AssertionError, ae:
- log.debug(ae)
- else:
- ok = klass.options
- ok.parseArgs = lambda x: subcmds.append(x)
- try:
- opts = ok()
- opts.parseOptions(config['subArgs'])
- except Exception, e:
- log.err(e)
- opts = {}
-
- elif hasattr(klass, local_options): ## we've been initialized already
- log.debug("processLegacyTest: %s.local_options found" % str(klass))
- try:
- opts = klass.local_options
- except AttributeError, ae: opts = {}; log.err(ae)
- log.debug("processLegacyTest: opts set to %s" % str(opts))
-
- try:
- cases = start_legacy_test(klass)
- ## XXX we need to get these results into the reporter
- if cases:
- log.debug("processLegacyTest: found cases: %s" % str(cases))
- return [], []
- except Exception, e: cases = []; log.err(e)
-
- return cases, opts
-
class ORunner(object):
"""
This is a specialized runner used by the ooniprobe command line tool.
@@ -237,29 +184,31 @@ class ORunner(object):
self.cases = cases
self.options = options
- log.debug("ORunner: cases=%s" % type(cases))
- log.debug("ORunner: options=%s" % options)
-
-
try:
- first = options.pop(0)
- except:
- first = options
-
- if 'inputs' in first:
- self.inputs = self.options['inputs']
+ assert len(options) != 0, "Length of options is zero!"
+ except AssertionError, ae:
+ self.inputs = []
+ log.err(ae)
else:
- log.msg("Could not find inputs!")
- self.inputs = [None]
+ try:
+ first = options.pop(0)
+ except:
+ first = options
+
+ if 'inputs' in first:
+ self.inputs = options['inputs']
+ else:
+ log.msg("Could not find inputs!")
+ log.msg("options[0] = %s" % first)
+ self.inputs = [None]
try:
reportFile = open(config['reportfile'], 'a+')
except:
filename = 'report_'+date.timestamp()+'.yaml'
reportFile = open(filename, 'a+')
- self.reporterFactory = ReporterFactory(
- reportFile, testSuite=self.baseSuite(self.cases)
- )
+ self.reporterFactory = ReporterFactory(reportFile,
+ testSuite=self.baseSuite(self.cases))
def runWithInputUnit(self, input_unit):
idx = 0
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index acad538..2e2d9e1 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -6,12 +6,14 @@
import random
from zope.interface import implements
+
from twisted.python import usage
from twisted.plugin import IPlugin
from twisted.internet import protocol, defer
from twisted.internet.ssl import ClientContextFactory
from twisted.web.http_headers import Headers
+from twisted.web.iweb import IBodyProducer
from ooni.nettest import NetTestCase
from ooni.utils import log
@@ -28,6 +30,23 @@ useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Geck
("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"),
("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")]
+class StringProducer(object):
+ implements(IBodyProducer)
+
+ def __init__(self, body):
+ self.body = body
+ self.length = len(body)
+
+ def startProducing(self, consumer):
+ consumer.write(self.body)
+ return defer.succeed(None)
+
+ def pauseProducing(self):
+ pass
+
+ def stopProducing(self):
+ pass
+
class BodyReceiver(protocol.Protocol):
def __init__(self, finished):
self.finished = finished
diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
index 9c80f62..fe97ca3 100644
--- a/ooni/utils/onion.py
+++ b/ooni/utils/onion.py
@@ -21,8 +21,8 @@ import sys
from twisted.internet import defer
from zope.interface import implements
-from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher
-from ooni.lib.txtorcon import TorState, TorConfig
+from txtorcon import CircuitListenerMixin, IStreamAttacher
+from txtorcon import TorState, TorConfig
from ooni.utils import log
from ooni.utils.timer import deferred_timeout, TimeoutError
More information about the tor-commits
mailing list