commit 98ebfe5e98824b4343a7a03f950f70e8f1c30faf Author: Arturo Filastò arturo@filasto.net Date: Tue Sep 18 10:07:35 2012 +0000
Import nettest work in progress related to reporting and testsuite --- ooni/nettest.py | 64 +++++++------------- ooni/oonicli.py | 18 ++++++ ooni/reporter.py | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 210 insertions(+), 47 deletions(-)
diff --git a/bin/ooniprobe b/bin/ooniprobe old mode 100644 new mode 100755 diff --git a/ooni/nettest.py b/ooni/nettest.py index ab009b1..097e947 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -18,6 +18,29 @@ def _iterateTests(testSuiteOrCase): yield subtest
+class TestSuite(pyunit.TestSuite): + inputUnit = [None] + def __repr__(self): + return "<%s input=%s, tests=%s>" % (self.__class__, self.inputUnit, list(self)) + + def run(self, result, inputUnit=[None]): + """ + Call C{run} on every member of the suite. + """ + # we implement this because Python 2.3 unittest defines this code + # in __call__, whereas 2.4 defines the code in run. + idx = 0 + for input, test in itertools.product(inputUnit, self._tests): + if result.shouldStop: + break + self.inputUnit = inputUnit + test.input = input + test.idx = idx + test(result) + idx += 1 + + return result + class TestCase(unittest.TestCase): """ A test case represents the minimum @@ -65,44 +88,3 @@ class TestCase(unittest.TestCase): result.stopTest(self)
-class TestSuite(pyunit.TestSuite): - """ - Extend the standard library's C{TestSuite} with support for the visitor - pattern and a consistently overrideable C{run} method. - """ - - def __init__(self, tests=(), inputs=()): - self._tests = [] - self._inputs = [] - self.addTests(tests, inputs) - print "Adding %s %s" % (tests, inputs) - - - def __call__(self, result): - return self.run(result) - - def __repr__(self): - return "<%s input=%s tests=%s>" % (self.__class__, - self._inputs, list(self)) - - def run(self, result, input=None): - """ - Call C{run} on every member of the suite. - """ - for test in self._tests: - if result.shouldStop: - break - return test(result, None) - - def addTests(self, tests, inputs=[]): - if isinstance(tests, basestring): - raise TypeError("tests must be and iterable of tests not a string") - for test in tests: - self.addTest(test, inputs) - - def addTest(self, test, inputs=[]): - #print "Adding: %s" % test - super(TestSuite, self).addTest(test) - self._inputs = inputs - - diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 199b4d4..8eabfb1 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -328,6 +328,23 @@ def _makeRunner(config): forceGarbageCollection=config['force-gc'])
+if 0: + loader = runner.TestLoader() + loader.suiteFactory = TestSuite + + for inputUnit in InputUnitFactory(FooTest.inputs): + print inputUnit + + suite = loader.loadClass(FooTest) + + reporterFactory = ReporterFactory(open('reporting.log', 'a+'), testSuite=suite) + reporterFactory.writeHeader() + #testUnitReport = OONIReporter(open('reporting.log', 'a+')) + #testUnitReport.writeHeader(FooTest) + for inputUnit in InputUnitFactory(FooTest.inputs): + testUnitReport = reporterFactory.create() + suite(testUnitReport, inputUnit) + testUnitReport.done()
def run(): if len(sys.argv) == 1: @@ -340,6 +357,7 @@ def run(): _initialDebugSetup(config) trialRunner = _makeRunner(config) suite = _getSuite(config) + print suite test_result = trialRunner.run(suite) if config.tracer: sys.settrace(None) diff --git a/ooni/reporter.py b/ooni/reporter.py index d20160f..07cffad 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -1,11 +1,174 @@ -from twisted.trial import reporter +import time +import sys +import yaml +import itertools
-class OONIReporter(reporter.Reporter): +sys.path.insert(0, '/home/x/Documents/pen_drive_bitcoin2012/ooni-probe/ENV/lib/python2.7/site-packages') +from datetime import datetime +from twisted.python.util import OrderedDict, untilConcludes +from twisted.trial import unittest, reporter, runner
- def startTest(self, test, input=None): - print "Running %s" % test - print "Input %s" % input - self._input = input +pyunit = __import__('unittest') + +class OReporter(pyunit.TestResult): + def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, + publisher=None, testSuite=None): + super(OReporter, self).__init__() + self.report = {'tests': []} + self._stream = reporter.SafeStream(stream) + self.tbformat = tbformat + self.realtime = realtime + self._startTime = None + self._warningCache = set() + + self._publisher = publisher + + def _getTime(self): + return time.time() + + def _write(self, format, *args): + s = str(format) + assert isinstance(s, type('')) + if args: + self._stream.write(s % args) + else: + self._stream.write(s) + untilConcludes(self._stream.flush) + + def _writeln(self, format, *args): + self._write(format, *args) + self._write('\n') + + def writeYamlLine(self, line): + self._write(yaml.dump([line])) + + + +class ReporterFactory(OReporter): + def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, + publisher=None, testSuite=None): + super(ReporterFactory, self).__init__(stream=stream, + tbformat=tbformat, realtime=realtime, publisher=publisher) + + self._testSuite = testSuite + self._reporters = [] + + def writeHeader(self): + pretty_date = "XXX Replace me with date.pretty_date()" + self._writeln("###########################################") + self._writeln("# OONI Probe Report for Test %s" % "XXX replace with with the test suite name") + self._writeln("# %s" % pretty_date) + self._writeln("###########################################") + + address = {'asn': 'XXX replace me with ASN', + 'ip': 'XXX replace me with IP'} + test_details = {'start_time': datetime.now(), + 'asn': address['asn'], + 'test_name': 'XXX replace me with the test name', + 'addr': address['ip']} + self.writeYamlLine(test_details) + self._writeln('') + + def create(self): + r = OONIReporter(self._stream, self.tbformat, self.realtime, + self._publisher) + self._reporters.append(OONIReporter) + return r + + +class OONIReporter(OReporter): + def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, + publisher=None): + super(OONIReporter, self).__init__(stream=stream, + tbformat=tbformat, realtime=realtime, publisher=publisher) + + self._tests = {} + self._publisher = publisher + if publisher is not None: + publisher.addObserver(self._observeWarnings) + + def getTestIndex(self, test): + try: + idx = test.idx + except: + idx = 0 + return idx + + + def startTest(self, test): super(OONIReporter, self).startTest(test)
+ idx = self.getTestIndex(test) + if not self._startTime: + self._startTime = self._getTime() + + test.report = {} + + self._tests[idx] = {} + self._tests[idx]['testStarted'] = self._getTime() + self._tests[idx]['input'] = test.input + self._tests[idx]['idx'] = idx + self._tests[idx]['name'] = test.name + self._tests[idx]['test'] = test + + + def stopTest(self, test): + super(OONIReporter, self).stopTest(test) + + idx = self.getTestIndex(test) + + self._tests[idx]['lastTime'] = self._getTime() - self._tests[idx]['testStarted'] + # XXX I put a dict() here so that the object is re-instantiated and I + # actually end up with the report I want. This could either be a + # python bug or a yaml bug. + self._tests[idx]['report'] = dict(test.report) + + def done(self): + """ + Summarize the result of the test run. + + The summary includes a report of all of the errors, todos, skips and + so forth that occurred during the run. It also includes the number of + tests that were run and how long it took to run them (not including + load time). + + Expects that L{_printErrors}, L{_writeln}, L{_write}, L{_printSummary} + and L{_separator} are all implemented. + """ + if self._publisher is not None: + self._publisher.removeObserver(self._observeWarnings) + if self._startTime is not None: + self.report['startTime'] = self._startTime + self.report['runTime'] = time.time() - self._startTime + self.report['testsRun'] = self.testsRun + self.report['tests'] = self._tests + self.writeReport() + + def writeReport(self): + self.writeYamlLine(self.report) + + def addSuccess(self, test): + super(OONIReporter, self).addSuccess(test) + #self.report['result'] = {'value': 'success'} + + def addError(self, *args): + super(OONIReporter, self).addError(*args) + #self.report['result'] = {'value': 'error', 'args': args} + + def addFailure(self, *args): + super(OONIReporter, self).addFailure(*args) + #self.report['result'] = {'value': 'failure', 'args': args} + + def addSkip(self, *args): + super(OONIReporter, self).addSkip(*args) + #self.report['result'] = {'value': 'skip', 'args': args} + + def addExpectedFailure(self, *args): + super(OONIReporter, self).addExpectedFailure(*args) + #self.report['result'] = {'value': 'expectedFailure', 'args': args} + + def addUnexpectedSuccess(self, *args): + super(OONIReporter, self).addUnexpectedSuccess(*args) + #self.report['result'] = {'args': args, 'value': 'unexpectedSuccess'} +