commit 2a93df352dd73989cb022551a85a755bc381eecd Author: Isis Lovecruft <isis@torproject.org> Date: Fri Oct 19 18:11:14 2012 +0000 * Moved legacy test classes and functions to a utility file at ooni/utils/legacy.py because they were getting a little cluttery. * Added documentation and cleaned up code in ooni/runner.py. --- ooni/runner.py | 169 +++++++-------------------- ooni/utils/legacy.py | 317 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 361 insertions(+), 125 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index cabc265..55ad401 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -3,23 +3,25 @@ import sys import types import time import inspect +import yaml from twisted.internet import defer, reactor -from twisted.python import reflect, failure, usage +from twisted.python import reflect, failure, usage +from twisted.python import log as tlog -from twisted.python import log as tlog - -from twisted.trial import unittest +from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader from twisted.trial.runner import isPackage, isTestCase, ErrorHolder from twisted.trial.runner import filenameToModule, _importFromFile -from ooni.reporter import ReporterFactory -from ooni.inputunit import InputUnitFactory -from ooni.nettest import InputTestSuite -from ooni import nettest -from ooni.utils import log, geodata, date -from ooni.plugoo import tests as oonitests +from ooni import nettest +from ooni.inputunit import InputUnitFactory +from ooni.nettest import InputTestSuite +from ooni.plugoo import tests as oonitests +from ooni.reporter import ReporterFactory +from ooni.utils import log, geodata, date +from ooni.utils.legacy import LegacyOONITest +from ooni.utils.legacy import start_legacy_test, adapt_legacy_test def isTestCase(thing): try: @@ -41,112 +43,19 @@ def isLegacyTest(obj): except TypeError: return False -class legacy_reporter(object): - def __init__(self, report_target): - self.report_target = report_target - - def __call__(self, what): - self.report_target.append(what) - -class LegacyOONITest(nettest.TestCase): - - ## we need bases so that inherited methods get parsed for prefixes too - from ooni.plugoo.tests import OONITest - __bases__ = (OONITest, ) - - def __init__(self, obj, config): - super(LegacyOONITest, self).__init__() - self.originalTest = obj - log.debug("obj: %s" % obj) - log.debug("originalTest: %s" % self.originalTest) - - self.subArgs = (None, ) - if 'subArgs' in config: - self.subArgs = config['subArgs'] - - try: - self.name = self.originalTest.shortName - except: - self.was_named = False - self.name = "LegacyOONITest" - - try: - self.subOptions = self.originalTest.options() - except AttributeError: - if self.was_named is False: - origClass = self.originalTest.__class__ - origClassStr = str(origClass) - fromModule = origClassStr.rsplit('.', 2)[:-1] - #origNamespace = globals()[origClass]() - #origAttr = getattr(origNamespace, fromModule) - log.debug("original class: %s" % origClassStr) - log.debug("from module: %s" % fromModule) - #log.debug("orginal namespace: %s" % origNamespace) - #log.debug("orginal attr: %s" % origAttr) - - def _options_from_name_tag(method_name, - orig_test=self.originalTest): - return orig_test.method_name.options() - - self.subOptions = _options_from_name_tag(fromModule, - self.originalTest) - else: - self.subOptions = None - log.err("That test appears to have a name, but no options!") - - if self.subOptions is not None: - self.subOptions.parseOptions(self.subArgs) - self.local_options = self.subOptions - - self.legacy_test = self.originalTest(None, None, None, None) - ## xxx fix me - #my_test.global_options = config['Options'] - self.legacy_test.local_options = self.subOptions - if self.was_named: - self.legacy_test.name = self.name - else: - self.legacy_test.name = fromModule - self.legacy_test.assets = self.legacy_test.load_assets() - self.legacy_test.report = legacy_reporter({}) - self.legacy_test.initialize() - - inputs = [] - - if len(self.legacy_test.assets.items()) == 0: - inputs.append('internal_asset_handler') - else: - for key, inputs in self.legacy_test.assets.items(): - pass - self.inputs = inputs - - def __getattr__(self, name): - def method(*args): - log.msg("Call to unknown method %s.%s" % (self.originalTest, name)) - if args: - log.msg("Unknown method %s parameters: %s" % str(args)) - return method - - @defer.inlineCallbacks - def test_start_legacy_test(self): - args = {} - for key, inputs in self.legacy_test.assets.items(): - args[key] = inputs - result = yield self.legacy_test.startTest(args) - self.report.update({'result': result}) - ## xxx we need to retVal on the defer.inlineCallbacks, right? - defer.returnValue(self.report) - -def adaptLegacyTest(obj, config): +def processTest(obj, config): """ - We take a legacy OONITest class and convert it into a nettest.TestCase. - This allows backward compatibility of old OONI tests. - - XXX perhaps we could implement another extra layer that makes the even - older test cases compatible with the new OONI. + Process the parameters and :class:`twisted.python.usage.Options` of a + :class:`ooni.nettest.Nettest`. + + :param obj: + An uninstantiated old test, which should be a subclass of + :class:`ooni.plugoo.tests.OONITest`. + :param config: + A configured and instantiated :class:`twisted.python.usage.Options` + class. """ - return LegacyOONITest(obj, config) -def processTest(obj, config): inputFile = obj.inputFile if obj.optParameters or inputFile: @@ -179,9 +88,16 @@ def findTestClassesFromConfig(config): case classes. If it detects that a certain test class is using the old OONIProbe format, then it will adapt it to the new testing system. + + :param config: + A configured and instantiated :class:`twisted.python.usage.Options` + class. + :return: + A list of class objects found in a file or module given on the + commandline. """ - filename = config['test'] + filename = config['test'] classes = [] module = filenameToModule(filename) @@ -189,7 +105,7 @@ def findTestClassesFromConfig(config): if isTestCase(val): classes.append(processTest(val, config)) elif isLegacyTest(val): - classes.append(adaptLegacyTest(val, config)) + classes.append(adapt_legacy_test(val, config)) return classes def makeTestCases(klass, tests, methodPrefix): @@ -197,6 +113,7 @@ def makeTestCases(klass, tests, methodPrefix): Takes a class some tests and returns the test cases. methodPrefix is how the test case functions should be prefixed with. """ + cases = [] for test in tests: cases.append(klass(methodPrefix+test)) @@ -204,27 +121,28 @@ def makeTestCases(klass, tests, methodPrefix): def loadTestsAndOptions(classes, config): """ - Takes a list of classes and returnes their testcases and options. + Takes a list of classes and returns their testcases and options. Legacy tests will be adapted. """ + methodPrefix = 'test' suiteFactory = InputTestSuite options = [] testCases = [] names = [] - from ooni.runner import LegacyOONITest _old_klass_type = LegacyOONITest for klass in classes: try: - assert not isinstance(klass, _old_klass_type) + assert not isinstance(klass, _old_klass_type), "Legacy test detected" except: assert isinstance(klass, _old_klass_type) - #log.debug(type(klass)) - #legacyTest = adaptLegacyTest(klass, config) - klass.test_start_legacy_test() + try: + start_legacy_test(klass) + except Exception, e: + log.err(e) else: tests = reflect.prefixedMethodNames(klass, methodPrefix) if tests: @@ -234,8 +152,9 @@ def loadTestsAndOptions(classes, config): k = klass() opts = k.getOptions() options.append(opts) - except AttributeError: + except AttributeError, ae: options.append([]) + log.err(ae) return testCases, options @@ -274,17 +193,17 @@ class ORunner(object): filename = 'report_'+date.timestamp()+'.yaml' reportFile = open(filename, 'a+') self.reporterFactory = ReporterFactory(reportFile, - testSuite=self.baseSuite(self.cases)) + testSuite=self.baseSuite(self.cases)) def runWithInputUnit(self, inputUnit): idx = 0 result = self.reporterFactory.create() - for input in inputUnit: + for inputs in inputUnit: result.reporterFactory = self.reporterFactory suite = self.baseSuite(self.cases) - suite.input = input + suite.input = inputs suite(result, idx) # XXX refactor all of this index bullshit to avoid having to pass diff --git a/ooni/utils/legacy.py b/ooni/utils/legacy.py new file mode 100755 index 0000000..0575670 --- /dev/null +++ b/ooni/utils/legacy.py @@ -0,0 +1,317 @@ +#-*- coding: utf-8 -*- +# +# legacy.py +# --------- +# Utilities for working with legacy OONI tests, i.e. tests which were created +# before the transition to the new twisted.trial based API. +# +# :authors: Isis Lovecruft, Arturo Filasto +# :license: see included LICENSE file +# :copyright: (c) 2012 Isis Lovecruft, Arturo Filasto, The Tor Project, Inc. +# :version: 0.1.0-pre-alpha + + +import inspect +import os +import yaml + +from twisted.internet import defer + +from ooni import nettest +from ooni.plugoo.tests import OONITest +from ooni.utils import log, date + +class LegacyReporter(object): + """ + Backwards compatibility class for creating a report object for results + from a :class:`ooni.runner.LegacyTest`. A + :class:`ooni.runner.LegacyReporter` object will eventually get wrapped in + a list when :mod:`ooni.oonicli` calls + :meth:`ooni.reporter.OONIReporter.stopTest`. + + :param report_target: + The type of object to write results to, by default a list. + """ + def __init__(self, report_target=[]): + self.report_target = report_target + if isinstance(self.report_target, dict): + self._type = dict + elif isinstance(self.report_target, list): + self._type = list + else: + self._type = type(self.report_target) + + def __call__(self, info): + if self._type is dict: + self.report_target.update(info) + elif self._type is list: + self.report_target.append(info) + else: + log.debug("ADD A NEW REPORT_TARGET TYPE!!") + +class LegacyOONITest(nettest.TestCase): + """ + Converts an old test, which should be a subclass of + :class:`ooni.plugoo.tests.OONITest`, to an :mod:`ooni.oonicli` + compatible class. + + :param obj: + An uninstantiated old test, which should be a subclass of + :class:`ooni.plugoo.tests.OONITest`. + :param config: + A configured and instantiated :class:`twisted.python.usage.Options` + class. + :meth start_legacy_test: + Handler for calling :meth:`ooni.plugoo.tests.OONITest.startTest`. + """ + + ## we need __bases__ because inspect.getmro() as well as + ## zope.interface.implements() both expect it: + from ooni.plugoo.tests import OONITest + __bases__ = (OONITest, ) + + + def __getattr__(self, name): + """ + Override of builtin getattr for :class:`ooni.runner.LegacyTest` so that + method calls to a LegacyTest instance or its parent class OONITest do + not return unhandled errors, but rather report that the method is unknown. + """ + + def __unknown_method__(*a): + log.msg("Call to unknown method %s.%s" % (self.originalTest, name)) + if a: + log.msg("Unknown method %s parameters: %s" % str(a)) + return __unknown_method__ + + def find_missing_options(self): + """ + In the case that our test is actually a class within a module named + after itself, i.e. 'ooni.plugins.bridget.bridget', we want dynamic + method discover so that we can search for the test's Options class. + + Example: + Let's say we want the Options class, which is at + ``ooni.plugins.bridget.bridget.options``. But in this case, our + original_test variable isn't going to have an attribute named + 'options', because original_test is actually the *first* occurence of + 'bridget'. + + In other words, our original_test is actually the module, so we need + to find the test, which is done with: + + getattr(original_test.__class__, test_class) + + After that, we've got our test stored as something like + ``ooni.plugins.bridget.bridget`` and we need to find 'options' as an + attribute under that, which is what + + options_finder = inspect.attrgetter('options') + + is used for. And the namespace stuff is just used for debugging edge + cases where we totally can't find the options. + + :ivar original_class: + The original subclass of OONITest, except that in this case, + because our test is a module, what we have here is + 'ooni.plugins.bridget.BridgeTest', while we actually need + something like 'ooni.plugins.bridget.bridget.BridgeTest' instead. + :ivar class_string: + The :ivar:`original_class` converted to a string. + :ivar from_module: + The parent module of :ivar:`original_class`, i.e. + `ooni.plugins.bridget`. + :ivar test_class: + The last part of :ivar:`from_module`, ie. 'bridget'. + :ivar options_finder: + An instance of :meth:`inspect.attrgetter` which searches for + methods within a test class named 'options'. + """ + + original_test = self.originalTest + original_class = original_test.__class__ + class_string = str(original_class) + from_module = inspect.getmodule(original_class) + test_class = class_string.rsplit('.', 1)[1] + options_finder = inspect.attrgetter('options') + + if self.was_named is False or self.name != test_class: + log.msg("Discovered legacy test named %s ..." % test_class) + setattr(self, 'name', test_class) + + try: + namespace = globals()[class_string] + log.debug("orginal namespace: %s" % namespace) + except KeyError, keyerr: + log.debug(keyerr) + + options = {} + try: + options = options_finder(getattr(original_class, test_class)) + except AttributeError: + self.__getattr__(test_class) + except Exception, e: + log.err(e) + finally: + return sub_options + + def __init__(self, obj, config): + """ + :param obj: + An uninstantiated old test, which should be a subclass of + :class:`ooni.plugoo.tests.OONITest`. + :param config: + A configured and instantiated + :class:`twisted.python.usage.Options` class. + :attr originalTest: + :attr subArgs: + :attr name: + :ivar was_named: + :attr subOptions: + """ + + super(LegacyOONITest, self).__init__() + self.originalTest = obj + + self.subArgs = (None, ) + if 'subArgs' in config: + self.subArgs = config['subArgs'] + + self.name = 'LegacyOONITest' + self.was_named = False + try: + self.name = self.originalTest.shortName + self.was_named = True + except AttributeError: + if self.originalTest.name and self.originalTest.name != 'oonitest': + self.name = self.originalTest.name + self.was_named = True + + try: + self.subOptions = self.originalTest.options() + except AttributeError: + if self.was_named is False: + self.subOptions = self.find_missing_options() + else: + self.subOptions = {} + log.msg("That test appears to have a name, but no options!") + + self.legacy_test = self.originalTest(None, None, None, None) + self.legacy_test.name = self.name + print "self.legacy_test.name: %s" % self.legacy_test.name + print "self.name: %s" % self.name + self.legacy_test.start_time = date.now() + + if self.subOptions is not None: + self.subOptions.parseOptions(self.subArgs) + self.legacy_test.local_options = self.subOptions + + self.reporter = [] + self.legacy_test.report = LegacyReporter(report_target=self.reporter) + + if 'reportfile' in config: + self.reporter_file = config['reportfile'] + else: + now = date.now() + time = date.rfc3339(now, utc=True, use_system_timezone=False) + filename = str(self.name) + "-" + str(time) + ".yaml" + self.reporter_file = os.path.join(os.getcwd(), filename) + + self.legacy_test.initialize() + self.legacy_test.assets = self.legacy_test.load_assets() + +def adapt_legacy_test(obj, config): + """ + Wrapper function for taking a legacy OONITest class and converting it into + a :class:`LegacyTest`, which is a variant of the new + :class:`ooni.nettest.TestCase` and is compatible with + :mod:`ooni.oonicli`. This allows for backward compatibility of old OONI + tests. + + :param obj: + An uninstantiated old test, which should be a subclass of + :class:`ooni.plugoo.tests.OONITest`. + :param config: + A configured and instantiated :class:`twisted.python.usage.Options` + class. + :return: + A :class:`LegacyOONITest`. + """ + return LegacyOONITest(obj, config) + +def report_legacy_test_to_file(legacy_test, file=None): + """ + xxx fill me in + """ + reporter_file = legacy_test.reporter_file + + if file is not None: + base = os.path.dirname(os.path.abspath(file)) + if base.endswith("ooni") or base == os.getcwd(): + reporter_file = file + else: + log.msg("Writing to %s not allowed, using default file %s." + % (base, reporter_file)) + + yams = yaml.safe_dump(legacy_test.reporter) + with open(reporter_file, 'a') as rosemary: + rosemary.write(yams) + rosemary.flush() + log.msg("Finished reporting.") + +def log_legacy_test_results(result, legacy_test, args): + if result: + legacy_test.report({args: result}) + log.debug("Legacy test %s with args:\n%s\nreturned result:\n%s" + % (legacy_test.name, args, result)) + else: + legacy_test.report({args: None}) + log.debug("No results return for %s with args:\n%s" + % (legacy_test.name, args)) + +@defer.inlineCallbacks +def run_legacy_test_with_args(legacy_test, args): + """ + Handler for calling :meth:`ooni.plugoo.tests.OONITest.startTest` with each + :param:`args` that, in the old framework, would have been generated one + line at a time by :class:`ooni.plugoo.assets.Asset`. This function is + wrapped with :meth:`twisted.internet.defer.inlineCallbacks` so that the + result of each call to :meth:`ooni.plugoo.tests.OONITest.experiment` is + returned immediately as :ivar:`returned`. + """ + + result = yield legacy_test.startTest(args) + defer.returnValue(result) + +def start_legacy_test(legacy_test): + """ + xxx fill me in + + need a list of startTest(args) which return deferreds + """ + + results = [] + + if len(legacy_test.assets.items()) != 0: + for keys, values in legacy_test.assets.items(): + for value in values: + args[keys] = value + log.debug("Running %s with args: %s" % (legacy_test.name, + args)) + d = run_legacy_test_with_args(args) + d.addCallback(log_legacy_test_results, legacy_test, args) + d.addErrback(log.err) + d.addCallback(report_legacy_test_to_file, legacy_test) + d.addErrback(log.err) + results.append(d) + else: + args['zero_input_test'] = True + log.debug("Running %s with args: %s" % (legacy_test.name, args)) + d = run_legacy_test_with_args(args) + d.addCallback(log_legacy_test_results, legacy_test, args) + d.addErrback(log.err) + d.addCallback(report_legacy_test_to_file, legacy_test) + d.addErrback(log.err) + results.append(d) + + defer.DeferredList(results)