commit 7643541a424655c689e252783a50037846368773 Author: Arturo Filastò <arturo@filasto.net> Date: Fri Sep 21 10:40:41 2012 +0000 Implement backward compatibility of new ooniprobe with legacy tests * Reporting system works with old tests * Properly document some of the functions --- ooni/input.py | 17 +++++------- ooni/nettest.py | 44 ++++----------------------------- ooni/oonicli.py | 8 ++++-- ooni/plugins/httpt.py | 9 ++++--- ooni/plugoo/tests.py | 7 ++--- ooni/reporter.py | 30 ++++++++++++++++++++--- ooni/runner.py | 64 +++++++++++++++++++++++++++++++++++++++---------- 7 files changed, 102 insertions(+), 77 deletions(-) diff --git a/ooni/input.py b/ooni/input.py index b931b82..69507b4 100644 --- a/ooni/input.py +++ b/ooni/input.py @@ -9,7 +9,7 @@ class InputUnitFactory(object): """ inputUnitSize = 3 def __init__(self, inputs=[]): - self._inputs = inputs + self._inputs = iter(inputs) self._idx = 0 self._ended = False @@ -21,16 +21,13 @@ class InputUnitFactory(object): raise StopIteration last_element_idx = self._idx + self.inputUnitSize - input_unit_elements = self._inputs[self._idx:last_element_idx] - try: - # XXX hack to fail when we reach the end of the list - antani = self._inputs[last_element_idx] - except: - if len(input_unit_elements) > 0: + input_unit_elements = [] + for i in xrange(last_element_idx): + try: + input_unit_elements.append(self._inputs.next()) + except: self._ended = True - return InputUnit(input_unit_elements) - else: - raise StopIteration + break self._idx += self.inputUnitSize diff --git a/ooni/nettest.py b/ooni/nettest.py index 0302d25..f3476cf 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -4,44 +4,11 @@ from twisted.trial import unittest, itrial pyunit = __import__('unittest') -def _iterateTests(testSuiteOrCase): +class InputTestSuite(pyunit.TestSuite): """ - Iterate through all of the test cases in C{testSuiteOrCase}. + This in an extension of a unittest test suite. It adds support for inputs + and the tracking of current index via idx. """ - try: - suite = iter(testSuiteOrCase) - except TypeError: - yield testSuiteOrCase - else: - for test in suite: - for subtest in _iterateTests(test): - yield subtest - - -class TestSuiteFactory(object): - def __init__(self, inputUnit, tests, basesuite): - self._baseSuite = basesuite - self._inputUnit = inputUnit - self._idx = 0 - self.tests = tests - - def __iter__(self): - return self - - def next(self): - try: - next_input = self._inputUnit.next() - print "Now dealing with %s %s" % (next_input, self._idx) - except: - raise StopIteration - new_test_suite = self._baseSuite(self.tests) - new_test_suite.input = next_input - new_test_suite._idx = self._idx - - self._idx += 1 - return new_test_suite - -class InputTestSuite(pyunit.TestSuite): def run(self, result, idx=0): self._idx = idx while self._tests: @@ -51,7 +18,6 @@ class InputTestSuite(pyunit.TestSuite): try: test.input = self.input test._idx = self._idx - print "IDX: %s" % self._idx test(result) except: test(result) @@ -59,8 +25,8 @@ class InputTestSuite(pyunit.TestSuite): return result class TestCase(unittest.TestCase): - name = "DefaultTestName" - inputs = [] + name = "DefaultOONITestCase" + inputs = [None] def getOptions(self): return {'inputs': self.inputs} diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 1e7613b..f14bf6e 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -73,10 +73,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): try: self['test'] = args[0] + self['subArgs'] = args[1:] except: raise usage.UsageError("No test filename specified!") - def postOptions(self): self['reporter'] = reporter.OONIReporter @@ -90,9 +90,11 @@ def run(): except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) - classes = runner.findTestClassesFromFile(config['test']) + classes = runner.findTestClassesFromConfig(config) + casesList, options = runner.loadTestsAndOptions(classes) for idx, cases in enumerate(casesList): - orunner = runner.ORunner(cases, options[idx]) + orunner = runner.ORunner(cases, options[idx], config) orunner.run() + diff --git a/ooni/plugins/httpt.py b/ooni/plugins/httpt.py index 46f3b17..358f1ea 100644 --- a/ooni/plugins/httpt.py +++ b/ooni/plugins/httpt.py @@ -65,13 +65,14 @@ class httptTest(http.HTTPTest): def processRedirect(self, location): self.result['redirect'] = None - if self.local_options['rules']: + try: + rules_file = self.local_options['rules'] import yaml - rules = yaml.load(open(self.local_options['rules'])) + rules = yaml.load(open(rules_file)) log.msg("Testing rules %s" % rules) redirect = self.testRules(rules, location) self.result['redirect'] = redirect - else: + except TypeError: log.msg("No rules file. Got a redirect, but nothing to do.") @@ -90,4 +91,4 @@ class httptTest(http.HTTPTest): # We need to instantiate it otherwise getPlugins does not detect it # XXX Find a way to load plugins without instantiating them. -httpt = httptTest(None, None, None) +#httpt = httptTest(None, None, None) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index e4ee187..37e0664 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -25,19 +25,19 @@ class OONITest(object): name = "oonitest" # By default we set this to False, meaning that we don't block blocking = False - reactor = None + reactor = reactor tool = False ended = False def __init__(self, local_options, global_options, report, ooninet=None, - reactor=None): + my_reactor=reactor): # These are the options that are read through the tests suboptions self.local_options = local_options # These are the options global to all of OONI self.global_options = global_options self.report = report #self.ooninet = ooninet - self.reactor = reactor + self.reactor = my_reactor self.result = {} self.initialize() @@ -133,7 +133,6 @@ class OONITest(object): @param args: the asset(s) lines that we are working on. """ self.start_time = date.now() - print "FOWID" log.msg("Starting test %s" % self.__class__) return self._do_experiment(args) diff --git a/ooni/reporter.py b/ooni/reporter.py index 14297cd..4b3ed6f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -11,6 +11,10 @@ from twisted.trial import unittest, reporter, runner pyunit = __import__('unittest') class OReporter(pyunit.TestResult): + """ + This is an extension of the unittest TestResult. It adds support for + reporting to yaml format. + """ def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, publisher=None, testSuite=None): super(OReporter, self).__init__() @@ -45,6 +49,10 @@ class OReporter(pyunit.TestResult): class ReporterFactory(OReporter): + """ + This is a reporter factory. It emits new instances of Reports. It is also + responsible for writing the OONI Report headers. + """ def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, publisher=None, testSuite=None): super(ReporterFactory, self).__init__(stream=stream, @@ -77,6 +85,14 @@ class ReporterFactory(OReporter): class OONIReporter(OReporter): + """ + This is a special reporter that has knowledge about the fact that there can + exist more test runs of the same kind per run. + These multiple test runs are kept track of through idx. + + An instance of such reporter should be created per InputUnit. Every input + unit will invoke size_of_input_unit * test_cases times startTest(). + """ def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, publisher=None): super(OONIReporter, self).__init__(stream=stream, @@ -119,10 +135,16 @@ class OONIReporter(OReporter): idx = self.getTestIndex(test) self._tests[idx]['lastTime'] = self._getTime() - self._tests[idx]['testStarted'] - # XXX I put a dict() here so that the object is re-instantiated and I - # actually end up with the report I want. This could either be a - # python bug or a yaml bug. - self._tests[idx]['report'] = dict(test.report) + # This is here for allowing reporting of legacy tests. + # XXX In the future this should be removed. + try: + self._tests[idx]['report'] = list(test.legacy_report) + except: + # XXX I put a dict() here so that the object is re-instantiated and I + # actually end up with the report I want. This could either be a + # python bug or a yaml bug. + self._tests[idx]['report'] = dict(test.report) + def done(self): """ diff --git a/ooni/runner.py b/ooni/runner.py index ddde83b..0bcafb0 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -15,6 +15,7 @@ from ooni.reporter import ReporterFactory from ooni.input import InputUnitFactory from ooni.nettest import InputTestSuite from ooni import nettest +from ooni.utils import log from ooni.plugoo import tests as oonitests def isTestCase(thing): @@ -37,7 +38,7 @@ def isLegacyTest(obj): except TypeError: return False -def adaptLegacyTest(obj, inputs=[None]): +def adaptLegacyTest(obj, config): """ We take a legacy OONITest class and convert it into a nettest.TestCase. This allows backward compatibility of old OONI tests. @@ -45,30 +46,62 @@ def adaptLegacyTest(obj, inputs=[None]): XXX perhaps we could implement another extra layer that makes the even older test cases compatible with the new OONI. """ + class legacy_reporter(object): + def __init__(self, report_target): + self.report_target = report_target + + def __call__(self, what): + self.report_target.append(what) + class LegacyOONITest(nettest.TestCase): - inputs = [None] - original_test = obj + try: + name = obj.shortName + except: + name = "LegacyOONITest" + + originalTest = obj + + subOptions = obj.options() + subOptions.parseOptions(config['subArgs']) + + test_class = obj(None, None, None, None) + test_class.local_options = subOptions + assets = test_class.load_assets() + + # XXX here we are only taking assets that are set to one item only. + for key, inputs in assets.items(): + pass + + inputs = inputs + local_options = subOptions @defer.inlineCallbacks def test_start_legacy_test(self): - print "bla bla bla" - print self.original_test - my_test = self.original_test(None, None, None) - yield my_test.startTest(self.input) + + self.legacy_report = [] + + my_test = self.originalTest(None, None, None) + my_test.report = legacy_reporter(self.legacy_report) + args = {} + args[self.key] = self.input + result = yield my_test.startTest(args) + print "Finished!" + print result return LegacyOONITest -def findTestClassesFromFile(filename): +def findTestClassesFromConfig(config): + filename = config['test'] + classes = [] - print "FILENAME %s" % filename module = filenameToModule(filename) for name, val in inspect.getmembers(module): if isTestCase(val): classes.append(val) elif isLegacyTest(val): - classes.append(adaptLegacyTest(val)) + classes.append(adaptLegacyTest(val, config)) return classes def makeTestCases(klass, tests, methodPrefix): @@ -99,13 +132,17 @@ def loadTestsAndOptions(classes): return testCases, options class ORunner(object): - def __init__(self, cases, options=None): + def __init__(self, cases, options=None, config=None): self.baseSuite = InputTestSuite self.cases = cases self.options = options self.inputs = options['inputs'] - self.reporterFactory = ReporterFactory(open('foo.log', 'a+'), - testSuite=self.baseSuite(self.cases)) + try: + reportFile = open(config['reportfile'], 'a+') + except: + reportFile = open('report.yaml', 'a+') + self.reporterFactory = ReporterFactory(reportFile, + testSuite=self.baseSuite(self.cases)) def runWithInputUnit(self, inputUnit): idx = 0 @@ -126,6 +163,7 @@ class ORunner(object): result.done() def run(self): + log.start() self.reporterFactory.writeHeader() for inputUnit in InputUnitFactory(self.inputs):