November 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] * Added a couple basic Exception classes to ooni.utils.net
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 187356a5ff61985b8691615334fa124ecd61ceab Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 16:00:39 2012 +0000 * Added a couple basic Exception classes to ooni.utils.net --- ooni/utils/net.py | 25 +++++++++++++++++++++++++ 1 files changed, 25 insertions(+), 0 deletions(-) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 218a74a..46c29e5 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -1,4 +1,29 @@ +# -*- encoding: utf-8 -*- +# +# net.py +# -------- +# OONI utilities for network infrastructure and hardware. +# +# :authors: Isis Lovecruft, Arturo Filasto +# :version: 0.0.1-pre-alpha +# :license: (c) 2012 Isis Lovecruft, Arturo Filasto +# see attached LICENCE file + + def getClientAddress(): address = {'asn': 'REPLACE_ME', 'ip': 'REPLACE_ME'} return address + + +class PermissionsError(SystemExit): + def __init__(self, *args, **kwargs): + if not args and not kwargs: + pe = "This test requires admin or root privileges to run. Exiting..." + super(PermissionsError, self).__init__(pe, *args, **kwargs) + else: + super(PermissionsError, self).__init__(*args, **kwargs) + +class IfaceError(SystemExit): + def __init__(self, *args, **kwargs): + super(IfaceError, self).__init__(*args, **kwargs)

1 0

[ooni-probe/master] * Initial outline of metaclasses for dynamic class generation.
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit d24d1ca2d439a185416374eacca460e7b8213f21 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 16:12:26 2012 +0000 * Initial outline of metaclasses for dynamic class generation. * Data Descriptor class. * Utilities for tricks and hacks to play to pass data between instances, and call super on a parent class that you don't yet know the name of. * I know this stuff looks hacky, and thus it's not being used anywhere yet, but I think it could provide later optimizations, both for runtime for the code and also for decision trees to handle progressions of tests without knowing what the future tests will be. Maybe. --- ooni/utils/hacks.py | 32 +++++++ ooni/utils/meta.py | 241 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+), 0 deletions(-) diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py index cf5f080..e778540 100644 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@ -57,3 +57,35 @@ def patched_reduce_ex(self, proto): else: return copy_reg._reconstructor, args +class MetaSuper(type): + """ + Metaclass for creating subclasses which have builtin name munging, so that + they are able to call self.__super.method() from an instance function + without knowing the instance class' base class name. + + For example: + + from hacks import MetaSuper + class A: + __metaclass__ = MetaSuper + def method(self): + return "A" + class B(A): + def method(self): + return "B" + self.__super.method() + class C(A): + def method(self): + return "C" + self.__super.method() + class D(C, B): + def method(self): + return "D" + self.__super.method() + + assert D().method() == "DCBA" + + Subclasses should not override "__init__", nor should subclasses have + the same name as any of their bases, or else much pain and suffering + will occur. + """ + def __init__(cls, name, bases, dict): + super(autosuper, cls).__init__(name, bases, dict) + setattr(cls, "_%s__super" % name, super(cls)) diff --git a/ooni/utils/meta.py b/ooni/utils/meta.py new file mode 100644 index 0000000..054e580 --- /dev/null +++ b/ooni/utils/meta.py @@ -0,0 +1,241 @@ +#-*- coding: utf-8 -*- +# +# meta.py +# ------- +# Meta classes for coercing subclasses which don't exist yet. +# +# :authors: Isis Lovecruft +# :version: 0.1.0-alpha +# :copyright: 2012 Isis Lovecruft +# :license: see attached LICENSE file +# + +class MetaDescriptor(type): + """ + bug: "Attribute error: class <> has no attribute __bases__" + + There are only objects. However, there *are* two kinds of objects: + type-objects and non-type-objects. + + There are only two objects which do not have an attribute named + "__bases__": + + 1) Instances of the builtin object ``object`` itself (i.e. the + superclass of any top-level class in python), whose __class__ is + itself, and whose type is ``type(type)``: + + >>> o = object() + >>> type(o) + <type 'type'> + >>> o.__class__ + <type 'object'> + + The o.__class__ part seems to imply that the __bases__ of + ``object`` should be itself. + + 2) Old Style Classes. Which are despicable demons, deserving to be + sent straight back to hell. No offense to the good Satanas. The + only thing these have by default is __doc__ and __module__. Also, + and this is importants too: Old Style Classes do not have an + attribute named __class__, because they do not derive from + anything. + + Incidentally, the type of ``type(type)`` is itself, and the "__bases__" of + ``type(type)`` is... + + >>> t = type(type) + >>> t.__name__ + 'type' + >>> type(t) + <type 'type'> + >>> t.__class__ + <type 'type'> + >>> t.__bases__ + (<type 'object'>,) + + ``type(object)``. WTF. This is to say that the "__bases__" of ``object`` + is the ``type`` of itself. This strange loop is where all black magic + enters into Python. + + If we do "class Metaclass(type): pass", we can then call + ``super(Metaclass, mcls)``, and the resulting ``super`` object is actually + just ``type``: + + o Its type is ``type(type)``. + o Its __bases__ ``type(object)``. + + For example, ``super(Metaclass, mcls).__new__(mcls, *a, *kw)`` is the same + as saying ``type(mcls, "Metaclass", (type, ), {} )``, except that super + does some namespace munging with calling "self.__super" on its own type, + which is probably equivalent to the strange loops with type(type) and + type(object), but my brain is already flipping out (and I keep a string + cosmology textbook on my nightstand!). + + However, we should not ever be able to call + + >>> super(type, type(mcls)).__new__(type(mcls), 'type', (type(type),) {} ) + TypeError: object.__new__(type) is not safe, use type.__new__() + + Q: Why all this fuss? + + A: We need to force future class-level attributes of subclasses of + TestCase to be accessible (also at the class-level, without + instatiations) by TestCase. I.e.: + 1) class SubTestCase has class attribute optParameters, but no + class for doing anything with them, and they shouldn't have to. + They should just be able to define the options. + 2) Therefore, TestCase needs to have data descriptors, which get + inherited. + 3) We need to be able to do this without dangerous namespace + munging, because we cannot control the namespace of future + tests. Therefore, we cannot use hacks like "self.__super". + + We need a Metaclass, which creates a Metafactory for classes which are + dynamic test descriptors. If this is confusing, leave it alone, there's + witches is these woods. + + http://stackoverflow.com/a/10707719 + http://docs.python.org/2/howto/descriptor.html + http://www.no-ack.org/2011/03/strange-behavior-with-properties-on.html + http://www.cafepy.com/article/python_types_and_objects/ + http://stackoverflow.com/questions/100003/what-is-a-metaclass-in-python + + """ + ## DO NOT GIVE A METACLASS ATTRIBUTES HERE. + #descriptors = { } + + from byteplay import Code, opmap + from functools import partial + + def __new__(im_so_meta_even_this_acronym, name, base, dict): + + def hofstaeder_decorator(meta_decorator): + def meta_decorator_factory(*args, **kwargs): + def meta_decorator_wrapper(func): + return meta_decorator(func, *args, **kwargs) + return meta_decorator_wrapper + return meta_decorator_factory + + def _transmute(opcode, arg): + if ((opcode == opmap['LOAD_GLOBAL']) and + (arg == 'self')): + return opmap['LOAD_FAST'], arg + return opcode, arg + + def selfless(child): + code = Code.from_code(child.func_code) + code.args = tuple(['self'] + list(code.args)) + code.code = [_transmute(op, arg) for op, arg in code.code] + function.func_code = code.to_code() + return function + + acronym = ( ( k,v ) for k,v in dict.items( ) + if not k.startswith('__') ) + metanym, polymorph = ( (k for ( k,v ) in acronym), + (v for ( k,v ) in acronym) ) + morphonemes = (("get%s"%n, "set%s"%n, "del%s"%n) for n in metanym ) + + oracles = [] + for getter, setter, deleter in morphonemes: + childnym = getter[3:] + + @hofstaeder_decorator + def meta_decorator(func, *args, **kwargs): + def decorator_wrapper(first, last): + return func(first, last) + return decorator_wrapper + + @meta_decorator(getter, setter, deleter) + def decorated_property(first, last): + childnym = getter[3:] + class DataDescriptor(object): + @selfless + def __init__(childnym=None, polymorph): + setattr(self, childnym, polymorph) + + @property + @selfless + def getter(): + return self.childnym + return DataDescriptor(first, last) + + oracles.append(decorated_property(childnym, polymorph)) + + return super( + MetaDescriptor, im_so_meta_even_this_acronym).__new__( + im_so_meta_even_this_acronym, + metanym, + polymorph, + dict(oracles) ) +''' + @property + def x(self): ## or getx + return self._x + @x.setter + def x(self, value): ## or setx + self._x = value + @x.deleter + def x(self): ## or delx + del self._x + ## or 'x = property(getx, setx, delx, "documentation")' + ## ^ ^ ^ ^ + ## just need @property's name, initial value can be None + +Metaclass + Creates Metaclasses for each data descriptor in each SubTestCase + so, per SubTestCase, we get (usually two) descriptors: + optParameters and input + +''' + +def applyClassAttribute(obj, cls, get='optParameters'): + """ + I get attributes from an outside instances' dictionary and attempt to + apply them to a class. I require that the attributes I am trying to set be + data descriptors which is just Python name munging trick that is mild and + harmless enough to have it's own builtin decorator helper: + + class Foo(object): + def __init__(self, *a, **kw): + if 'thing' in kw: + self._thing = thing + @property + def thing(self): + return self._thing + @property.setter + def thing(self, value): + self._thing = value + @property.delter + def thing(self): + return del(self) + """ + from ooni.utils.assertions import isClass, isNotClass + + try: + assert isNotClass(obj), "must be an instance" + assert isClass(cls), "not a class" + ## obj is probably an instance + C = obj.__class__ ## of a subclass of nettest.TestCase + + assert issubclass(C, cls), "not a subclass of %s" % cls + assert C.__dict__.__contains__('optParameters'), \ + "%s in %s.__dict__ not found" % (get, C) + except AssertionError, ae: + log.debug(ae) + else: + attributes = classify_class_attrs(C) + ## uncomment this to have class attributes spewn everywhere: + #log.debug("Found class attributes:\n%s" % pprint(attributes)) + for attr in attributes: + if attr.name == str(get): + setattr(obj, str(get), attr.object) + if not hasattr(obj, str(get)): + log.debug("Unable to find class attribute %s" % get) + else: + log.debug("Applying %s.%s = %s to descriptor..." + % (C.name, attr.name, attr.object)) + ## This was an unfinished attempt at fixing a class' __bases__, I do + ## not know if it was heading in the right direction. It can be + ## removed it is still crufting up the space. --isis + if '__bases__' or '_parents' in C.__dict__: + pass

1 0

[ooni-probe/master] * Added utilities file for assertions and checks.
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 63c11a5acf05a5ccf7cf360c0cf13d8372cbb63d Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 16:29:07 2012 +0000 * Added utilities file for assertions and checks. * Moved the usage.Options ValueChecker class to the utils.assertions file. --- ooni/utils/assertions.py | 180 ++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 180 insertions(+), 0 deletions(-) diff --git a/ooni/utils/assertions.py b/ooni/utils/assertions.py new file mode 100644 index 0000000..875dcf5 --- /dev/null +++ b/ooni/utils/assertions.py @@ -0,0 +1,180 @@ +#-*- coding: utf-8 -*- +# +# assertions.py +# ------------- +# Collection of utilies for checks and assertions. +# +# :authors: Isis Lovecruft +# :version: 0.1.0-alpha +# :license: see included LICENSE file +# :copyright: 2012 Isis Lovecruft, The Tor Project Inc. +# + +class ValueChecker(object): + """ + A class for general purpose value checks on commandline parameters + passed to subclasses of :class:`twisted.python.usage.Options`. + """ + default_doc = "fix me" + + def __init__(self, coerce_doc=None): + if not coerce_doc: + self.coerce_doc = default_doc + else: + self.coerce_doc = coerce_doc + + @classmethod + def port_check(cls, port, + range_min=1024, range_max=65535, + coerce_doc=None): + """ + Check that given ports are in the allowed range for an unprivileged + user. + + :param port: + The port to check. + :param range_min: + The minimum allowable port number. + :param range_max: + The minimum allowable port number. + :param coerce_doc: + The documentation string to show in the optParameters menu, see + :class:`twisted.python.usage.Options`. + """ + if not coerce_doc: + coerceDoc = cls.default_doc + + assert isinstance(port, int) + if port not in range(range_min, range_max): + raise ValueError("Port out of range") + log.err() + + @staticmethod + def uid_check(error_message): + """ + Check that we're not root. If we are, setuid(1000) to normal user if + we're running on a posix-based system, and if we're on Windows just + tell the user that we can't be run as root with the specified options + and then exit. + + :param error_message: + The string to log as an error message when the uid check fails. + """ + uid, gid = os.getuid(), os.getgid() + if uid == 0 and gid == 0: + log.msg(error_message) + if os.name == 'posix': + log.msg("Dropping privileges to normal user...") + os.setgid(1000) + os.setuid(1000) + else: + sys.exit(0) + + @staticmethod + def dir_check(d): + """Check that the given directory exists.""" + if not os.path.isdir(d): + raise ValueError("%s doesn't exist, or has wrong permissions" % d) + + @staticmethod + def file_check(f): + """Check that the given file exists.""" + if not os.path.isfile(f): + raise ValueError("%s does not exist, or has wrong permissions" % f) + +def isNewStyleClass(obj): + """ + Check if :param:`obj` is a new-style class, which is any class + derived by either + + NewStyleClass = type('NewStyleClass') + or + class NewStyleClass(object): + pass + + whereas old-style classes are (only in Python 2.x) derived by: + + class OldStyleClass: + pass + + There are recently implemented classes in many Python libraries, + including a few in Twisted, which are derived from old-style classes. + Thus, calling super() or attempting to retrieve the MRO of any + subclass of old-style classes can cause issues such as: + + o Calling Subclass.mro() goes through the class hierarchy until it + reaches OldStyleClass, which has no mro(), and raises an + AttributeError. + + o Calling super(Subclass, subcls) produces an ambiguous, rather than + algorithmic, class hierarchy, which (especially in Twisted's case) + breaks multiple inheritance. + + o Expected Subclass instance attributes, in particular __bases__ and + __class__, can be missing, which in turn leads to problems with + a whole bunch of builtin methods and modules. + + For more information, see: + http://www.python.org/download/releases/2.3/mro/ + http://www.cafepy.com/article/python_attributes_and_methods/ + + :return: + True if :param:`obj` is a new-style class derived from object; + False if :param:`obj` is an old-style class (or derived from + one). + """ + from types import ClassType + return not isinstance(type(object), ClassType) + +def isOldStyleClass(obj): + """ + Check if :param:`obj` is an old-style class, which is any class + derived in Python 2.x with: + + class OldStyleClass: + pass + + There are recently implemented classes in many Python libraries, + including a few in Twisted, which are derived from old-style classes, + and thus their types, bases, and attributes are generally just messed + up. + + :return: + True if :param:`obj` is a new-style class derived from object; + False if :param:`obj` is an old-style class (or derived from + one). + """ + from types import ClassType + return not isinstance(type(object), ClassType) + +def isClass(obj): + """ + Check if an object is *a* class (not a specific class), without trying + to call the obj.__class__ attribute, as that is not available in some + cases. This function will return True for both old- and new-style + classes, however, it will return False for class instances of either + style. An alternate way to do this (although it presumes that + :param:`obj` is a class instance) would be: + + from types import TypeType + return isinstance(object.__class__, TypeType) + + It turns out that objects with <type 'type'>, i.e. classes, don't + actually have the __class__ attribute...go figure. Instead, class + objects are only required to have the __doc__ and __module__ + attributes. + + :param obj: + Any object. + :return: + True if :param:`obj` is a class, False if otherwise (even if + :param:`obj` is an instance of a class). + """ + from types import ClassType + return isinstance(object, (type, ClassType)) + +def isNotClass(object): + """ + See :func:`isClass`. + """ + return True if not isClass(object) else False

1 0

[ooni-probe/master] * Forgot to remove the old ValueChecker. Made it dead.
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit fe055521bfd05c6fa0d262d48f8154ca249f9c0b Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 16:33:32 2012 +0000 * Forgot to remove the old ValueChecker. Made it dead. --- ooni/utils/config.py | 66 -------------------------------------------------- 1 files changed, 0 insertions(+), 66 deletions(-) diff --git a/ooni/utils/config.py b/ooni/utils/config.py index e429ef6..a5b32f8 100644 --- a/ooni/utils/config.py +++ b/ooni/utils/config.py @@ -53,69 +53,3 @@ class Config(Storage): self._cfgparser.write(cfgfile) finally: cfgfile.close() - -class ValueChecker(object): - """ - A class for general purpose value checks on commandline parameters - passed to subclasses of :class:`twisted.python.usage.Options`. - """ - def __init__(self, coerce_doc=None): - self.coerce_doc = coerce_doc - - def port_check(self, port, range_min=1024, range_max=65535): - """ - Check that given ports are in the allowed range for an unprivileged - user. - - :param port: - The port to check. - :param range_min: - The minimum allowable port number. - :param range_max: - The minimum allowable port number. - :param coerce_doc: - The documentation string to show in the optParameters menu, see - :class:`twisted.python.usage.Options`. - """ - if self.coerce_doc is not None: - coerceDoc = self.coerce_doc - - assert type(port) is int - if port not in range(range_min, range_max): - raise ValueError("Port out of range") - log.err() - - @staticmethod - def uid_check(error_message): - """ - Check that we're not root. If we are, setuid(1000) to normal user if - we're running on a posix-based system, and if we're on Windows just - tell the user that we can't be run as root with the specified options - and then exit. - - :param error_message: - The string to log as an error message when the uid check fails. - """ - uid, gid = os.getuid(), os.getgid() - if uid == 0 and gid == 0: - log.msg(error_message) - if os.name == 'posix': - log.msg("Dropping privileges to normal user...") - os.setgid(1000) - os.setuid(1000) - else: - sys.exit(0) - - @staticmethod - def dir_check(d): - """Check that the given directory exists.""" - if not os.path.isdir(d): - raise ValueError("%s doesn't exist, or has wrong permissions" - % d) - - @staticmethod - def file_check(f): - """Check that the given file exists.""" - if not os.path.isfile(f): - raise ValueError("%s does not exist, or has wrong permissions" - % f)

1 0

[ooni-probe/master] * Added a Singleton class which can be subclassed, ensuring that there can
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 0c15c4a0cabaed7eaa37ba4103756b1ad0e85df8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 16:30:43 2012 +0000 * Added a Singleton class which can be subclassed, ensuring that there can only ever be one instance of the subclass, to ooni.utils.process * Style-fixing on a couple of assert statements. --- ooni/utils/process.py | 25 ++++++++++++++++++++++--- 1 files changed, 22 insertions(+), 3 deletions(-) diff --git a/ooni/utils/process.py b/ooni/utils/process.py index e9f56a0..25d6368 100644 --- a/ooni/utils/process.py +++ b/ooni/utils/process.py @@ -39,9 +39,9 @@ def singleton_semaphore(deferred_process_init, callback chain has completed. This should be a fully initialized process connected to a :class:`twisted.internet.reactor`. """ - assert type(callbacks) is list - assert type(errbacks) is list - assert type(max_inits) is int + assert isinstance(callbacks, list) + assert isinstance(errbacks, list) + assert isinstance(max_inits, int) for cb in callbacks: deferred_process_init.addCallback(cb) @@ -52,3 +52,22 @@ def singleton_semaphore(deferred_process_init, singleton = yield only_this_many.run(deferred_process_init) defer.returnValue(singleton) +class Singleton(object): + """ + Generic Class for creating Singleton subclasses. + + Subclass me to create a singleton class, which will only ever have one + instance, regardless of how many times the subclass constructor is called. + + Any subclass of me should override ``init`` rather than ``__init__``, + because the latter is called whenever the constructor is called. + """ + def __new__(cls, *args, **kwds): + it = cls.__dict__.get("__it__") + if it is not None: + return it + cls.__it__ = it = object.__new__(cls) + it.init(*args, **kwds) + return it + def init(self, *args, **kwds): + pass

1 0

[ooni-probe/master] * Removed extra whitespace.
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit dd23d6ad9f2ec4695a8cf0d6db35e480fac49f5d Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 16:38:12 2012 +0000 * Removed extra whitespace. --- nettests/core/dnstamper.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/nettests/core/dnstamper.py b/nettests/core/dnstamper.py index 1028291..d2eebf8 100644 --- a/nettests/core/dnstamper.py +++ b/nettests/core/dnstamper.py @@ -236,4 +236,3 @@ class DNSTamperTest(nettest.TestCase): else: print "Still missing %s - %s" % (len(self.test_a_lookups), len(self.test_resolvers)) -

1 0

[ooni-probe/master] * Added an InputUnitProcessor class for generating generators for
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit b27f5ad66d0f6cf96c2277d3f4165d7d13e0b84f Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 16:46:40 2012 +0000 * Added an InputUnitProcessor class for generating generators for processing inputs. In ooni.inputunit * Also added a couple of exception classes. --- ooni/inputunit.py | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 129 insertions(+), 3 deletions(-) diff --git a/ooni/inputunit.py b/ooni/inputunit.py index e5b6187..ab46515 100644 --- a/ooni/inputunit.py +++ b/ooni/inputunit.py @@ -1,5 +1,31 @@ +# -*- coding: utf-8 -*- +# +# inputunit.py +# ------------ +# Classes and function for working with OONI TestCase inputs. +# +# @authors: Arturo Filasto, Isis Lovecruft +# @version: 0.1.0-alpha +# @license: see included LICENSE file +# @copyright: 2012 Arturo Filasto, Isis Lovecruft, The Tor Project Inc. +# + from twisted.trial import unittest +from zope.interface.exceptions import BrokenImplementation + + +def simpleInputUnitProcessor(input_unit): + """A simple InputUnit generator without parsing.""" + try: + assert hasattr(input_unit, '__iter__'), "inputs must be iterable!" + except AssertionError, ae: + raise BrokenImplementation, ae + else: + for i in input_unit: + yield i + + class PatchedPyUnitResultAdapter(unittest.PyUnitResultAdapter): def __init__(self, original): """ @@ -13,8 +39,8 @@ unittest.PyUnitResultAdapter = PatchedPyUnitResultAdapter class InputUnitFactory(object): """ - This is a factory that takes the size of input units to be generated a set - of units that is a python iterable item and outputs InputUnit objects + This is a factory that takes the size of input units to be generated and a + set of units that is a python iterable item, and outputs InputUnit objects containing inputUnitSize elements. This object is a python iterable, this means that it does not need to keep @@ -48,7 +74,6 @@ class InputUnitFactory(object): return InputUnit(input_unit_elements) - class InputUnit(object): """ This is a python iterable object that contains the input elements to be @@ -76,5 +101,106 @@ class InputUnit(object): def append(self, input): self._inputs.append(input) +class IUProcessorExit(GeneratorExit): + """InputUnitProcessor has exited.""" + +class IUProcessorAborted(Exception): + """The InputUnitProcessor was aborted with Inputs remaining.""" + +class InputUnitProcessor(InputUnit): + """ + Create a generator for returning inputs one-by-one from a + :class:`InputUnit` (or any other iterable defined within an instance of + :class:`ooni.nettest.TestCase`), and a generator function. + + The :ivar:generator can be a custom generator, or chain of generators, for + customized parsing of an InputUnit, or it can be an imported + function. There are useful imported functions in the builtin + :mod:`itertools`. If no :ivar:`generator` is given, the default one strips + whitespace characters, then returns the Input if the input does not begin + with a crunch (#) or bang (!). :) + + If :ivar:catchStopIter is True, then catch any StopIterations and return a + 2-tuple containing (boolean, list(unprocessed)): + + (True, None) when :ivar:iterable is empty, or + (False, [unprocessed]) if :ivar:iterable was not empty. + + If :ivar:catchStopIter is False (default), then we catch the StopIteration + exception, mark :attr:`empty` as 'True', and reraise the StopIteration. + xxx fill me in with parameter details + """ + empty = False + + def __init__(self, iterable, input_filter=None, catch_err=False): + """ + Create an InputUnitProcessor. + + xxx fill me in + """ + from itertools import takewhile + from types import GeneratorType + + assert hasattr(iterable, "__iter__"), "That's not an iterable!" + + self._iterable = iterable + self._infilter = input_filter + self._noerr = catch_err + self._empty = self.empty + def __len__(self): + return len(self.iterable) + + def __unprocessed__(self): + if not self.isdone(): + unprocessed = \ + [u for u in self._unit if self._unit is not StopIteration] + return (False, unprocessed) + else: + return (True, None) + + def isdone(self): + return self._empty + + def throw(self, exception, message="", traceback=None): + try: + raise exception, message, traceback + except: + yield self._iterable.next() + + @staticmethod + def __strip__(x): + return x.strip() + + @staticmethod + def _default_input_filter(x): + if not x.startswith('#') and not x.startswith('!'): + return True + else: + return False + + def __make_filter__(self): + if self.input_filter and hasattr(self.input_filter, "__call__"): + return self.input_filter + else: + return self._default_input_filter + + def finish(self): + if self._noerr: + return self.__unprocessed__() + else: + if not self.isdone(): + raise IUProcessorAborted + else: + raise IUProcessorExit + + def process(self): + carbon = self.__make_filter__() + try: + yield takewhile(carbon, + [i for i in self.__strip__(self._iterable)] + ).next() + except StopIteration, si: + self._empty = True + yield self.finish()

1 0

[ooni-probe/master] * More whitespace cleanup.
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit faba805f4cd625813b394c4a120f9c8b4177010c Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 16:43:54 2012 +0000 * More whitespace cleanup. --- ooni/oonicli.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 5ea92d7..5b0964b 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -112,4 +112,3 @@ def run(): for idx, cases in enumerate(casesList): orunner = runner.ORunner(cases, options[idx], config) orunner.run() -

1 0

[ooni-probe/master] * Refactored runner to get it to recognise test suboptions.
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit a778902283ff9ddf1c54e6e13d2debed16f28a0f Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 17:12:34 2012 +0000 * Refactored runner to get it to recognise test suboptions. --- ooni/runner.py | 136 +++++++++++++++++++++++++++++++------------------------- 1 files changed, 76 insertions(+), 60 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index b5c33a0..b1a21ac 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -18,6 +18,8 @@ import time import inspect import yaml +from pprint import pprint + from twisted.internet import defer, reactor from twisted.python import reflect, failure, usage from twisted.python import log as tlog @@ -53,45 +55,6 @@ def isLegacyTest(obj): except TypeError: return False -def processTest(obj, config): - """ - Process the parameters and :class:`twisted.python.usage.Options` of a - :class:`ooni.nettest.Nettest`. - - :param obj: - An uninstantiated old test, which should be a subclass of - :class:`ooni.plugoo.tests.OONITest`. - :param config: - A configured and instantiated :class:`twisted.python.usage.Options` - class. - """ - - inputFile = obj.inputFile - - if obj.optParameters or inputFile: - if not obj.optParameters: - obj.optParameters = [] - - if inputFile: - obj.optParameters.append(inputFile) - - class Options(usage.Options): - optParameters = obj.optParameters - - options = Options() - options.parseOptions(config['subArgs']) - obj.localOptions = options - - if inputFile: - obj.inputFile = options[inputFile[0]] - try: - tmp_obj = obj() - tmp_obj.getOptions() - except usage.UsageError: - options.opt_help() - - return obj - def findTestClassesFromConfig(config): """ Takes as input the command line config parameters and returns the test @@ -106,7 +69,6 @@ def findTestClassesFromConfig(config): A list of class objects found in a file or module given on the commandline. """ - filename = config['test'] classes = [] @@ -114,7 +76,7 @@ def findTestClassesFromConfig(config): for name, val in inspect.getmembers(module): if isTestCase(val): log.debug("Detected TestCase %s" % val) - classes.append(processTest(val, config)) + classes.append(val) elif isLegacyTest(val): log.debug("Detected Legacy Test %s" % val) classes.append(adapt_legacy_test(val, config)) @@ -125,57 +87,111 @@ def makeTestCases(klass, tests, methodPrefix): Takes a class some tests and returns the test cases. methodPrefix is how the test case functions should be prefixed with. """ - cases = [] for test in tests: cases.append(klass(methodPrefix+test)) return cases +def processTestOptions(cls, config): + """ + Process the parameters and :class:`twisted.python.usage.Options` of a + :class:`ooni.nettest.Nettest`. + + :param cls: + An subclass of :class:`ooni.nettest.TestCase`. + :param config: + A configured and instantiated :class:`twisted.python.usage.Options` + class. + """ + if cls.optParameters or cls.inputFile: + if not cls.optParameters: + cls.optParameters = [] + + if cls.inputFile: + cls.optParameters.append(cls.inputFile) + + class Options(usage.Options): + optParameters = cls.optParameters + + opts = Options() + opts.parseOptions(config['subArgs']) + cls.localOptions = opts + + if cls.inputFile: + cls.inputFile = opts[cls.inputFile[0]] + + """ + try: + log.debug("%s: trying %s.localoptions.getOptions()..." + % (__name__, cls.name)) + try: + assert hasattr(cls, 'getOptions') + except AssertionError, ae: + options = opts.opt_help() + raise Exception, "Cannot find %s.getOptions()" % cls.name + else: + options = cls.getOptions() + except usage.UsageError: + options = opts.opt_help() + else: + return cls, options + """ + + return cls, cls.localOptions + def loadTestsAndOptions(classes, config): """ Takes a list of test classes and returns their testcases and options. Legacy tests will be adapted. """ - methodPrefix = 'test' suiteFactory = InputTestSuite options = [] testCases = [] names = [] - _old_klass_type = LegacyOONITest + _old_class_type = LegacyOONITest - for klass in classes: - if isinstance(klass, _old_klass_type): + for cls in classes: + if isinstance(cls, _old_class_type): try: - cases = start_legacy_test(klass) - #cases.callback() - if cases: - print cases - return [], [] + cases = start_legacy_test(cls) testCases.append(cases) except Exception, e: log.err(e) else: try: - opts = klass.local_options + opts = cls.local_options options.append(opts) except AttributeError, ae: options.append([]) log.err(ae) + if cases: + print cases + return [], [] else: - tests = reflect.prefixedMethodNames(klass, methodPrefix) + tests = reflect.prefixedMethodNames(cls, methodPrefix) if tests: - cases = makeTestCases(klass, tests, methodPrefix) + cases = makeTestCases(cls, tests, methodPrefix) testCases.append(cases) try: - k = klass() - opts = k.getOptions() - options.append(opts) + #c = cls() + #cls, opts = processTestOptions(cls, config) + opts = processTestOptions(cls, config) except AttributeError, ae: options.append([]) log.err(ae) + else: + try: + instance = cls() + inputs = instance.__get_inputs__() + except Exception, e: + log.err(e) + else: + opts.update(inputs) + options.append(opts) + log.debug("runner.loadTestsAndOptions: OPTIONS: %s" % options) return testCases, options class ORunner(object): @@ -205,7 +221,6 @@ class ORunner(object): self.inputs = options['inputs'] else: log.msg("Could not find inputs!") - log.msg("options[0] = %s" % first) self.inputs = [None] try: @@ -213,8 +228,9 @@ class ORunner(object): except: filename = 'report_'+date.timestamp()+'.yaml' reportFile = open(filename, 'a+') - self.reporterFactory = ReporterFactory(reportFile, - testSuite=self.baseSuite(self.cases)) + self.reporterFactory = ReporterFactory( + reportFile, testSuite=self.baseSuite(self.cases) + ) def runWithInputUnit(self, inputUnit): idx = 0

1 0

[ooni-probe/master] * Refactored nettest.TestCase to get it to recognize test suboptions.
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 1928f5b0b898c1cdfe817e9432b92543c4763298 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 17:10:12 2012 +0000 * Refactored nettest.TestCase to get it to recognize test suboptions. --- ooni/nettest.py | 194 +++++++++++++++++++++++++++++++++++++++++-------------- 1 files changed, 146 insertions(+), 48 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index d35bd4d..3c87b7f 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -1,9 +1,21 @@ +# -*- coding: utf-8 -*- + import itertools import os -from twisted.trial import unittest, itrial -from twisted.internet import defer, utils -from ooni.utils import log +from inspect import classify_class_attrs +from pprint import pprint + +from twisted.internet import defer, utils +from twisted.python import usage +from twisted.trial import unittest, itrial +from zope.interface.exceptions import BrokenImplementation + +from ooni.inputunit import InputUnitProcessor +from ooni.utils import log +from ooni.utils.assertions import isClass, isNotClass +from ooni.utils.assertions import isOldStyleClass, isNewStyleClass + pyunit = __import__('unittest') @@ -33,19 +45,19 @@ class TestCase(unittest.TestCase): you will subclass this object. * inputs: can be set to a static set of inputs. All the tests (the methods - starting with the "test_" prefix) will be run once per input. At every run - the _input_ attribute of the TestCase instance will be set to the value of - the current iteration over inputs. Any python iterable object can be set - to inputs. + starting with the "test_" prefix) will be run once per input. At every + run the _input_ attribute of the TestCase instance will be set to the + value of the current iteration over inputs. Any python iterable object + can be set to inputs. - * inputFile: attribute should be set to an array containing the command line - argument that should be used as the input file. Such array looks like - this: + * inputFile: attribute should be set to an array containing the command + line argument that should be used as the input file. Such array looks + like this: ``["commandlinearg", "c", "The description"]`` - The second value of such arrray is the shorthand for the command line arg. - The user will then be able to specify inputs to the test via: + The second value of such arrray is the shorthand for the command line + arg. The user will then be able to specify inputs to the test via: ``ooniprobe mytest.py --commandlinearg path/to/file.txt`` @@ -55,8 +67,8 @@ class TestCase(unittest.TestCase): * inputProcessor: should be set to a function that takes as argument an - open file descriptor and it will yield the input to be passed to the test - instance. + open file descriptor and it will yield the input to be passed to the + test instance. * name: should be set to the name of the test. @@ -67,32 +79,47 @@ class TestCase(unittest.TestCase): * version: is the version string of the test. """ - name = "I Did Not Change The Name" - author = "John Doe <foo(a)example.com>" - version = "0" + name = "I Did Not Change The Name" + author = "John Doe <foo(a)example.com>" + version = "0.0.0" - inputs = [None] inputFile = None + inputs = [None] report = {} report['errors'] = [] optParameters = None + optFlags = None + subCommands = None - def _raaun(self, methodName, result): - from twisted.internet import reactor - method = getattr(self, methodName) - log.debug("Running %s" % methodName) - d = defer.maybeDeferred( - utils.runWithWarningsSuppressed, self._getSuppress(), method) - d.addBoth(lambda x : call.active() and call.cancel() or x) - return d + def setUpClass(self, *args, **kwargs): + """ + Create a TestCase instance. This function is equivalent to '__init__'. + To add futher setup steps before a set of tests in a TestCase instance + run, create a function called 'setUp'. + + Class attributes, such as `report`, `optParameters`, `name`, and + `author` should be overriden statically as class attributes in any + subclass of :class:`ooni.nettest.TestCase`, so that the calling + functions in ooni.runner can handle them correctly. + """ + methodName = 'runTest' + if kwargs: + if 'methodName' in kwargs: + methodName = kwargs['methodName'] + + super(TestCase, self).__init__(methodName=methodName) + #for key, value in kwargs.items(): + # setattr(self.__class__, key, value) + # + #self.inputs = self.getInputs() def deferSetUp(self, ignored, result): """ - If we have the reporterFactory set we need to write the header. If such - method is not present we will only run the test skipping header + If we have the reporterFactory set we need to write the header. If + such method is not present we will only run the test skipping header writing. """ if result.reporterFactory.firstrun: @@ -103,31 +130,102 @@ class TestCase(unittest.TestCase): else: return unittest.TestCase.deferSetUp(self, ignored, result) - def inputProcessor(self, fp): - for x in fp.readlines(): - yield x.strip() + def _raaun(self, methodName, result): + from twisted.internet import reactor + method = getattr(self, methodName) + log.debug("Running %s" % methodName) + d = defer.maybeDeferred( + utils.runWithWarningsSuppressed, self._getSuppress(), method) + d.addBoth(lambda x : call.active() and call.cancel() or x) + return d + + @staticmethod + def inputParser(inputs): + """Replace me with a custom function for parsing inputs.""" + return inputs + + def __input_file_processor__(self, fp): + """ + I open :attr:inputFile if there is one, and return inputs one by one + after stripping them of whitespace and running them through the parser + :meth:`inputParser`. + """ + for line in fp.readlines(): + yield self.inputParser(line.strip()) fp.close() - def getOptions(self): + def __get_inputs__(self): + """ + I am called from the ooni.runner and you probably should not override + me. I gather the internal inputs from an instantiated test class and + pass them to the rest of the runner. + + If you are looking for a way to parse inputs from inputFile, see + :meth:`inputParser`. + """ + processor = InputUnitProcessor(self.inputs, + input_filter=None, + catch_err=False) + processed = processor.process() + + log.msg("Received direct inputs:\n%s" % inputs) + log.debug("Our InputUnitProcessor is %s" % processor) + + #while processed is not StopIteration: + # self.inputs = processed + # yield self.inputs + #else: + # if self.inputFile: + if self.inputFile: try: - assert isinstance(self.inputFile, str) - except AssertionError, ae: - log.err(ae) + fp = open(self.inputFile) ## xxx fixme: + except Exception, e: ## bad news to leave file + log.err(e) ## descriptors open + else: + from_file = self.__input_file_processor__(fp) + self.inputs = itertools.chain(processor, from_file) + elif self.inputFile is False: + log.debug("%s specified that it doesn't need inputFile." + % self.__class__.__name__) + self.inputs = processed + else: + raise BrokenImplementation + + return self.inputs + + def getOptions(self): + ''' + for attr in attributes: + if not attr.name is 'optParameters' or attr.name is 'optFlags': + continue + elif attr.name is 'optParameters': + cls._optParameters = attr.object + elif attr.name is 'optFlags': + log.debug("Applying %s.%s = %s to data descriptor..." + % (cls.__name__, "_"+attr.name, attr.object)) + cls._optParameters = attr.object else: - if os.path.isfile(self.inputFile): - print self.inputFile - fp = open(self.inputFile) - self.inputs = self.inputProcessor(fp) - elif not self.inputs[0]: - pass - elif self.inputFile: - raise usage.UsageError("No input file specified!") - # XXX perhaps we may want to name and version to be inside of a - # different object that is not called options. - return {'inputs': self.inputs, - 'name': self.name, - 'version': self.version} + log.debug("How did we get here? attr.name = %s" % attr.name) + ''' + if self.localOptions: + if self.inputs[0] is not None or self.inputFile is not None: + self.__get_inputs__() + return self.localOptions + else: + raise Exception, "could not find cls.localOptions! 234" + + # if options: + # return options + # else: + # ## is this safe to do? it might turn Hofstaeder... + # return self.__dict__ + #################### + ## original return + #################### + #return {'inputs': self.inputs, + # 'name': self.name, + # 'version': self.version} def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs)

1 0