commit 1bbbad8b1553a7e9f2ca7b5bcb76ec47acce22f6 Author: Isis Lovecruft <isis@torproject.org> Date: Sat Oct 20 13:30:30 2012 +0000 * Refactored ooni/runner.py again, so that it actually runs old OONITests, new tests still work too. I tested all of them (both nettests and legacy tests) and then diffed the results from before and after these changes. Some tests *are* broken, e.g. dnstamper doesn't seem to find its inputs correctly, but none of the tests are any more broken than they were before this commit -- the difference is that old tests will run. * There was a rather nasty problem with old tests not finding their subOptions(), which was due to Twisted's usage.Options class only being able to handle one level of subOptions. When the old tests expected ooni/ooniprobe.py to have the global_options and themselves to have local_options, what was actually happening was that ooni/oonicli.py took the first level, and then the test took the second, and thus was missing any global_options. * There was also a bug where, before, ooni/runner.py would do things like: self.legacy_test.local_options = self.subOptions when self.legacy_test is an already instantiated subclass of ooni.plugoo.tests.OONITest. Because 'local_options' in this case is obviously an attribute of the class, we have to do setattr(). Because of this series of bugs, the legacy tests were missing a whole bunch of class attributes which they required to run, read test inputs from files, and really do anything at all. * The function ooni.utils.legacy.start_legacy_test() now handles creating a defer.gatherResults(defer.DeferredList), where the defer.DeferredList is all of the calls to ooni.plugoo.tests.OONITest.startTest(args) with all of the iterations of args. * Added and updated a bunch of documentation. * TODO some more documentation. * TODO there might be more bugs. I sure as hell hope not. :( * TODO go to bed now. I am getting grumpy. --- ooni/runner.py | 41 +++++++++-- ooni/utils/legacy.py | 184 ++++++++++++++++++++++++++++++-------------------- 2 files changed, 145 insertions(+), 80 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 55ad401..59c0c45 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -1,3 +1,16 @@ +#-*- coding: utf-8 -*- +# +# runner.py +# --------- +# Handles running ooni.nettests as well as ooni.plugoo.tests.OONITests. +# +# :authors: Isis Lovecruft, Arturo Filasto +# :license: see included LICENSE file +# :copyright: (c) 2012 Isis Lovecruft, Arturo Filasto, The Tor Project, Inc. +# :version: 0.1.0-pre-alpha +# + + import os import sys import types @@ -134,16 +147,24 @@ def loadTestsAndOptions(classes, config): _old_klass_type = LegacyOONITest for klass in classes: - - try: - assert not isinstance(klass, _old_klass_type), "Legacy test detected" - except: - assert isinstance(klass, _old_klass_type) + if isinstance(klass, _old_klass_type): try: - start_legacy_test(klass) + cases = start_legacy_test(klass) + #cases.callback() + if cases: + print cases + return [], [] + testCases.append(cases) except Exception, e: log.err(e) - else: + else: + try: + opts = klass.local_options + options.append(opts) + except AttributeError, ae: + options.append([]) + log.err(ae) + elif not isinstance(klass, _old_klass_type): tests = reflect.prefixedMethodNames(klass, methodPrefix) if tests: cases = makeTestCases(klass, tests, methodPrefix) @@ -155,6 +176,11 @@ def loadTestsAndOptions(classes, config): except AttributeError, ae: options.append([]) log.err(ae) + else: + try: + raise RuntimeError, "Class is some strange type!" + except RuntimeError, re: + log.err(re) return testCases, options @@ -220,4 +246,3 @@ class ORunner(object): self.reporterFactory.options = self.options for inputUnit in InputUnitFactory(self.inputs): self.runWithInputUnit(inputUnit) - diff --git a/ooni/utils/legacy.py b/ooni/utils/legacy.py index 0575670..aa415a9 100755 --- a/ooni/utils/legacy.py +++ b/ooni/utils/legacy.py @@ -15,7 +15,9 @@ import inspect import os import yaml -from twisted.internet import defer +from twisted.internet import defer, reactor +from twisted.python import log as tplog +from twisted.python.usage import Options as tpOptions from ooni import nettest from ooni.plugoo.tests import OONITest @@ -70,14 +72,13 @@ class LegacyOONITest(nettest.TestCase): from ooni.plugoo.tests import OONITest __bases__ = (OONITest, ) - def __getattr__(self, name): """ - Override of builtin getattr for :class:`ooni.runner.LegacyTest` so that - method calls to a LegacyTest instance or its parent class OONITest do - not return unhandled errors, but rather report that the method is unknown. + Override of builtin getattr for :class:`ooni.runner.LegacyTest` so + that method calls to a LegacyTest instance or its parent class + OONITest do not return unhandled errors, but rather report that the + method is unknown. """ - def __unknown_method__(*a): log.msg("Call to unknown method %s.%s" % (self.originalTest, name)) if a: @@ -127,7 +128,6 @@ class LegacyOONITest(nettest.TestCase): An instance of :meth:`inspect.attrgetter` which searches for methods within a test class named 'options'. """ - original_test = self.originalTest original_class = original_test.__class__ class_string = str(original_class) @@ -145,7 +145,7 @@ class LegacyOONITest(nettest.TestCase): except KeyError, keyerr: log.debug(keyerr) - options = {} + options = tpOptions try: options = options_finder(getattr(original_class, test_class)) except AttributeError: @@ -153,10 +153,12 @@ class LegacyOONITest(nettest.TestCase): except Exception, e: log.err(e) finally: - return sub_options + return options() def __init__(self, obj, config): """ + xxx fill me in + :param obj: An uninstantiated old test, which should be a subclass of :class:`ooni.plugoo.tests.OONITest`. @@ -169,56 +171,72 @@ class LegacyOONITest(nettest.TestCase): :ivar was_named: :attr subOptions: """ - super(LegacyOONITest, self).__init__() self.originalTest = obj - - self.subArgs = (None, ) - if 'subArgs' in config: - self.subArgs = config['subArgs'] - - self.name = 'LegacyOONITest' - self.was_named = False + self.start_time = date.now() + self.name = 'LegacyOONITest' + self.was_named = False try: - self.name = self.originalTest.shortName + self.name = self.originalTest.shortName self.was_named = True except AttributeError: if self.originalTest.name and self.originalTest.name != 'oonitest': - self.name = self.originalTest.name + self.name = self.originalTest.name self.was_named = True + if 'subArgs' in config: + self.subArgs = config['subArgs'] + else: + self.subArgs = (None, ) + log.msg("No suboptions to test %s found; continuing..."% self.name) + try: self.subOptions = self.originalTest.options() except AttributeError: if self.was_named is False: self.subOptions = self.find_missing_options() else: - self.subOptions = {} + self.subOptions = None log.msg("That test appears to have a name, but no options!") - self.legacy_test = self.originalTest(None, None, None, None) - self.legacy_test.name = self.name - print "self.legacy_test.name: %s" % self.legacy_test.name - print "self.name: %s" % self.name - self.legacy_test.start_time = date.now() - if self.subOptions is not None: - self.subOptions.parseOptions(self.subArgs) - self.legacy_test.local_options = self.subOptions - - self.reporter = [] - self.legacy_test.report = LegacyReporter(report_target=self.reporter) + if len(self.subArgs) > 0: + self.subOptions.parseOptions(self.subArgs) + self.local_options = self.subOptions + else: + print self.subOptions if 'reportfile' in config: self.reporter_file = config['reportfile'] else: - now = date.now() - time = date.rfc3339(now, utc=True, use_system_timezone=False) - filename = str(self.name) + "-" + str(time) + ".yaml" + filename = str(self.name) + "-" + str(date.timestamp()) + ".yaml" self.reporter_file = os.path.join(os.getcwd(), filename) + self.reporter = [] + self.report = LegacyReporter(report_target=self.reporter) + + self.legacy_test = self.originalTest(None, self.local_options, + None, self.report) + setattr(self.legacy_test, 'name', self.name) + setattr(self.legacy_test, 'start_time', self.start_time) + + self.inputs = {} + for keys, values in self.legacy_test.assets.items(): + self.inputs[keys] = values + setattr(self.legacy_test, 'inputs', self.inputs) - self.legacy_test.initialize() - self.legacy_test.assets = self.legacy_test.load_assets() + @defer.inlineCallbacks + def run_with_args(self, args): + """ + Handler for calling :meth:`ooni.plugoo.tests.OONITest.startTest` with + each :param:`args` that, in the old framework, would have been + generated one line at a time by + :class:`ooni.plugoo.assets.Asset`. This function is wrapped with + :meth:`twisted.internet.defer.inlineCallbacks` so that the result of + each call to :meth:`ooni.plugoo.tests.OONITest.experiment` is returned + immediately as :ivar:`returned`. + """ + result = yield self.legacy_test.startTest(args) + defer.returnValue(result) def adapt_legacy_test(obj, config): """ @@ -241,6 +259,10 @@ def adapt_legacy_test(obj, config): def report_legacy_test_to_file(legacy_test, file=None): """ + xxx this function current does not get used, and could easily be handled + by ooni.runner.loadTestsAndOptions, or some other function in + ooni.runner. + xxx fill me in """ reporter_file = legacy_test.reporter_file @@ -260,6 +282,21 @@ def report_legacy_test_to_file(legacy_test, file=None): log.msg("Finished reporting.") def log_legacy_test_results(result, legacy_test, args): + """ + Callback function for deferreds in :func:`start_legacy_test` which + handles updating the legacy_test's :class:`legacy_test.report`. + + :param result: + The possible result of a deferred which has been returned from + :meth:`ooni.plugoo.test.OONITest.experiment` and + :meth:`ooni.plugoo.test.OONITest.control`. + :param legacy_test: + The :class:`LegacyOONITest` which we're processing. + :param args: + The current inputs which we're giving to legacy_test.startTest(). + :return: + The :param:`legacy_test`. + """ if result: legacy_test.report({args: result}) log.debug("Legacy test %s with args:\n%s\nreturned result:\n%s" @@ -268,50 +305,53 @@ def log_legacy_test_results(result, legacy_test, args): legacy_test.report({args: None}) log.debug("No results return for %s with args:\n%s" % (legacy_test.name, args)) - -@defer.inlineCallbacks -def run_legacy_test_with_args(legacy_test, args): - """ - Handler for calling :meth:`ooni.plugoo.tests.OONITest.startTest` with each - :param:`args` that, in the old framework, would have been generated one - line at a time by :class:`ooni.plugoo.assets.Asset`. This function is - wrapped with :meth:`twisted.internet.defer.inlineCallbacks` so that the - result of each call to :meth:`ooni.plugoo.tests.OONITest.experiment` is - returned immediately as :ivar:`returned`. - """ - - result = yield legacy_test.startTest(args) - defer.returnValue(result) + return legacy_test def start_legacy_test(legacy_test): """ - xxx fill me in - - need a list of startTest(args) which return deferreds + This is the main function which should be used to call a legacy test, it + handles parsing the deprecated :class:`ooni.plugoo.assets.Asset` items as + inputs, and calls back to a custom, backwards-compatible Reporter. + + For each input to the legacy_test, this function creates a + :class:`twisted.internet.defer.Deferred` which has already received its + :meth:`callback`. The end result is a + :class:`twisted.internet.defer.gatherResults` of all the outcomes of + :param:`legacy_test` for each of the inputs. + + :param legacy_test: + A :class:`LegacyOONITest` to process. + :ivar results: + A list of :class:`twisted.internet.defer.Deferred`s which gets + processed as a :class:`twisted.internet.defer.DeferredList`. + :ivar current_input: + The input we are current working on, i.e. what would have been 'args' + (as in, 'experiment(args)') in the old design. + :return: + A :class:`twisted.internet.defer.gatherResults`. """ - results = [] + current_input = {} - if len(legacy_test.assets.items()) != 0: - for keys, values in legacy_test.assets.items(): + if len(legacy_test.inputs) > 0: + for keys, values in legacy_test.inputs: for value in values: - args[keys] = value - log.debug("Running %s with args: %s" % (legacy_test.name, - args)) - d = run_legacy_test_with_args(args) - d.addCallback(log_legacy_test_results, legacy_test, args) - d.addErrback(log.err) - d.addCallback(report_legacy_test_to_file, legacy_test) - d.addErrback(log.err) + current_input[keys] = value + log.debug("Running %s with args: %s" + % (legacy_test.name, current_input)) + d = legacy_test.run_with_args(current_input) + d.addCallback(log_legacy_test_results, legacy_test, + current_input) + d.addErrback(tplog.err) results.append(d) else: - args['zero_input_test'] = True - log.debug("Running %s with args: %s" % (legacy_test.name, args)) - d = run_legacy_test_with_args(args) - d.addCallback(log_legacy_test_results, legacy_test, args) - d.addErrback(log.err) - d.addCallback(report_legacy_test_to_file, legacy_test) - d.addErrback(log.err) + current_input['zero_input_test'] = True + log.debug("Running %s with current input: %s" + % (legacy_test.name, current_input)) + d = legacy_test.run_with_args(current_input) + d.addCallback(log_legacy_test_results, legacy_test, current_input) + d.addErrback(tplog.err) results.append(d) - defer.DeferredList(results) + dlist = defer.gatherResults(results) + return dlist