commit 80e6e316e6c2d5344d34453ed2006fd239824dea Author: Arturo Filastò art@fuffa.org Date: Fri Nov 23 19:41:41 2012 +0100

Implement postProcessor NetTestCase hook for processing the results of all the test methods * Such methods gets passed a report dict that contains the reports of the previous test runs keyed on the test method. This allows to implement censorship detection logic that is cumulative to the running of all tests * Refactor how we run tests to allow this hook to work. In particular all tests methods are run before moving onto the next input. * Add index for tracking the current input unit --- ooni/nettest.py | 13 ++++++ ooni/reporter.py | 5 +- ooni/runner.py | 116 ++++++++++++++++++++++++++++++++++-------------------- 3 files changed, 89 insertions(+), 45 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py index d460147..e96fba1 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -19,6 +19,9 @@ from twisted.python import usage

from ooni.utils import log

+class NoPostProcessor(Exception): + pass + class NetTestCase(object): """ This is the base of the OONI nettest universe. When you write a nettest @@ -101,6 +104,16 @@ class NetTestCase(object): """ pass

+ def postProcessor(self, report): + """ + Subclass this to do post processing tasks that are to occur once all + the test methods have been called. Once per input. + postProcessing works exactly like test methods, in the sense that + anything that gets written to the object self.report[] will be added to + the final test report. + """ + raise NoPostProcessor + def inputProcessor(self, filename=None): """ You may replace this with your own custom input processor. It takes as diff --git a/ooni/reporter.py b/ooni/reporter.py index 8df2264..8c76dc4 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -83,7 +83,7 @@ def getTestDetails(options): defer.returnValue(test_details)

class OReporter(object): - def createReport(options): + def createReport(self, options): """ Override this with your own logic to implement tests. """ @@ -99,7 +99,7 @@ class OReporter(object): pass

def testDone(self, test, test_name): - log.debug("Finished running %s" % test_name) + log.msg("Finished running %s" % test_name) log.debug("Writing report") test_report = dict(test.report)

@@ -114,6 +114,7 @@ class OReporter(object): report = {'input': test_input, 'test_name': test_name, 'test_started': test_started, + 'test_runtime': test_runtime, 'report': test_report} return self.writeReportEntry(report)

diff --git a/ooni/runner.py b/ooni/runner.py index 8843cd9..f5ab651 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -21,7 +21,7 @@ from twisted.trial.runner import filenameToModule from twisted.internet import reactor, threads

from ooni.inputunit import InputUnitFactory -from ooni.nettest import NetTestCase +from ooni.nettest import NetTestCase, NoPostProcessor

from ooni import reporter

@@ -69,8 +69,8 @@ def processTest(obj, cmd_line_options):

except usage.UsageError, e: test_name = tmp_test_case_object.name - print "There was an error in running %s!" % test_name - print "%s" % e + log.err("There was an error in running %s!" % test_name) + log.err("%s" % e) options.opt_help() raise usage.UsageError("Error in parsing command line args for %s" % test_name)

@@ -141,53 +141,87 @@ def loadTestsAndOptions(classes, cmd_line_options):

return test_cases, options

-def runTestWithInput(test_class, test_method, test_input, oreporter): - log.debug("Running %s with %s" % (test_method, test_input)) +def runTestCasesWithInput(test_cases, test_input, oreporter): + """ + Runs in parallel all the test methods that are inside of the specified test case. + Reporting happens every time a Test Method has concluded running. + Once all the test methods have been called we check to see if the + postProcessing class method returns something. If it does return something + we will write this as another entry inside of the report called post_processing. + """ + + # This is used to store a copy of all the test reports + tests_report = {}

def test_done(result, test_instance, test_name): log.debug("runTestWithInput: concluded %s" % test_name) + tests_report[test_name] = dict(test_instance.report) return oreporter.testDone(test_instance, test_name)

def test_error(failure, test_instance, test_name): log.exception(failure)

- test_instance = test_class() - test_instance.input = test_input - test_instance.report = {} - log.debug("Processing %s" % test_instance.name) - # use this to keep track of the test runtime - test_instance._start_time = time.time() - # call setups on the test - test_instance._setUp() - test_instance.setUp() - test = getattr(test_instance, test_method) - - d = defer.maybeDeferred(test) - d.addCallback(test_done, test_instance, test_method) - d.addErrback(test_error, test_instance, test_method) - log.debug("returning %s input" % test_method) - return d - -def runTestWithInputUnit(test_class, - test_method, input_unit, oreporter): + def tests_done(result, test_class): + test_instance = test_class() + test_instance.report = {} + test_instance.input = None + test_instance._start_time = time.time() + post = getattr(test_instance, 'postProcessor') + try: + post_processing = post(tests_report) + return oreporter.testDone(test_instance, 'summary') + except NoPostProcessor: + log.debug("No post processor configured") + + dl = [] + for test_case in test_cases: + log.debug("Processing %s" % test_case[1]) + test_class = test_case[0] + test_method = test_case[1] + + log.msg("Running %s with %s" % (test_method, test_input)) + + test_instance = test_class() + test_instance.input = test_input + test_instance.report = {} + log.msg("Processing %s" % test_instance.name) + # use this to keep track of the test runtime + test_instance._start_time = time.time() + # call setups on the test + test_instance._setUp() + test_instance.setUp() + test = getattr(test_instance, test_method) + + d = defer.maybeDeferred(test) + d.addCallback(test_done, test_instance, test_method) + d.addErrback(test_error, test_instance, test_method) + log.debug("returning %s input" % test_method) + dl.append(d) + + test_methods_d = defer.DeferredList(dl) + test_methods_d.addCallback(tests_done, test_cases[0][0]) + return test_methods_d + +def runTestCasesWithInputUnit(test_cases, input_unit, oreporter): """ - test_class: the uninstantiated class of the test to be run + Runs the Test Cases that are given as input parallely. + A Test Case is a subclass of ooni.nettest.NetTestCase and a list of + methods.

- test_method: a string representing the method name to be called + The deferred list will fire once all the test methods have been + run once per item in the input unit.

- input_unit: a generator that contains the inputs to be run on the test + test_cases: A list of tuples containing the test class and the test method as a string.

- oreporter: ooni.reporter.OReporter instance + input_unit: A generator that yields an input per iteration

- returns a deferred list containing all the tests to be run at this time + oreporter: An instance of a subclass of ooni.reporter.OReporter """ - + log.debug("Running test cases with input unit") dl = [] - log.debug("input unit %s" % input_unit) for test_input in input_unit: - log.debug("running with input: %s" % test_input) - d = runTestWithInput(test_class, - test_method, test_input, oreporter) + log.debug("Running test with this input %s" % test_input) + d = runTestCasesWithInput(test_cases, test_input, oreporter) dl.append(d) return defer.DeferredList(dl)

@@ -241,18 +275,14 @@ def runTestCases(test_cases, options, # This deferred list is a deferred list of deferred lists # it is used to store all the deferreds of the tests that # are run + input_unit_idx = 0 try: for input_unit in input_unit_factory: log.debug("Running this input unit %s" % input_unit) - # We do this because generators can't we rewound. - input_list = list(input_unit) - for test_case in test_cases: - log.debug("Processing %s" % test_case[1]) - test_class = test_case[0] - test_method = test_case[1] - yield runTestWithInputUnit(test_class, - test_method, input_list, - oreporter) + yield runTestCasesWithInputUnit(test_cases, input_unit, + oreporter) + input_unit_idx += 1 + except Exception: log.exception("Problem in running test") reactor.stop()