commit 603d5cdc27a13aef4d9d25ffbd9108fb3bdbdcac Author: Arturo Filastò arturo@filasto.net Date: Tue Sep 11 15:35:21 2012 +0000
Implement a working ooniclient based off of trial --- ooni/nettest.py | 97 +++++++++++++++++++++++++++ ooni/oonicli.py | 73 ++++----------------- ooni/plugoo/tests.py | 1 + ooni/reporter.py | 16 ++--- ooni/runner.py | 177 +++++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 292 insertions(+), 72 deletions(-)
diff --git a/ooni/nettest.py b/ooni/nettest.py new file mode 100644 index 0000000..fe8c05c --- /dev/null +++ b/ooni/nettest.py @@ -0,0 +1,97 @@ + +from twisted.python import log +from twisted.trial import unittest, itrial + +pyunit = __import__('unittest') + +def _iterateTests(testSuiteOrCase): + """ + Iterate through all of the test cases in C{testSuiteOrCase}. + """ + try: + suite = iter(testSuiteOrCase) + except TypeError: + if not testSuiteOrCase.inputs: + yield testSuiteOrCase + else: + inputs = iter(testSuiteOrCase.inputs) + print "Detected Sub shit! %s" % inputs + for input in inputs: + yield testSuiteOrCase, input + else: + for test in suite: + for subtest in _iterateTests(test): + yield subtest + + +class TestCase(unittest.TestCase): + """ + A test case represents the minimum + """ + def run(self, result, input): + """ + Run the test case, storing the results in C{result}. + + First runs C{setUp} on self, then runs the test method (defined in the + constructor), then runs C{tearDown}. As with the standard library + L{unittest.TestCase}, the return value of these methods is disregarded. + In particular, returning a L{Deferred} has no special additional + consequences. + + @param result: A L{TestResult} object. + """ + log.msg("--> %s <--" % (self.id())) + new_result = itrial.IReporter(result, None) + if new_result is None: + result = PyUnitResultAdapter(result) + else: + result = new_result + result.startTest(self) + if self.getSkip(): # don't run test methods that are marked as .skip + result.addSkip(self, self.getSkip()) + result.stopTest(self) + return + + self._passed = False + self._warnings = [] + + self._installObserver() + # All the code inside _runFixturesAndTest will be run such that warnings + # emitted by it will be collected and retrievable by flushWarnings. + unittest._collectWarnings(self._warnings.append, self._runFixturesAndTest, result) + + # Any collected warnings which the test method didn't flush get + # re-emitted so they'll be logged or show up on stdout or whatever. + for w in self.flushWarnings(): + try: + warnings.warn_explicit(**w) + except: + result.addError(self, failure.Failure()) + + result.stopTest(self) + + +class TestSuite(pyunit.TestSuite): + """ + Extend the standard library's C{TestSuite} with support for the visitor + pattern and a consistently overrideable C{run} method. + """ + + def __call__(self, result, input): + return self.run(result, input) + + + def run(self, result, input): + """ + Call C{run} on every member of the suite. + """ + # we implement this because Python 2.3 unittest defines this code + # in __call__, whereas 2.4 defines the code in run. + for test in self._tests: + if result.shouldStop: + break + print test + print "----------------" + test(result, input) + return result + diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 8ace160..199b4d4 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -22,19 +22,8 @@ from twisted.python.filepath import FilePath from twisted import plugin from twisted.python.util import spewer from twisted.python.compat import set -from twisted.trial import runner, itrial, reporter - - -# Yea, this is stupid. Leave it for for command-line compatibility for a -# while, though. -TBFORMAT_MAP = { - 'plain': 'default', - 'default': 'default', - 'emacs': 'brief', - 'brief': 'brief', - 'cgitb': 'verbose', - 'verbose': 'verbose' - } +from twisted.trial import itrial +from ooni import runner, reporter
def _parseLocalVariables(line): @@ -98,26 +87,6 @@ def isTestFile(filename): and os.path.splitext(basename)[1] == ('.py'))
-def _reporterAction(): - return usage.CompleteList([p.longOpt for p in - plugin.getPlugins(itrial.IReporter)]) - -class Options(usage.Options): - - optParameters = [ - ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], - ['output', 'o', 'report.log', "Specify output report file"], - ['log', 'l', 'oonicli.log', "Specify output log file"] - ] - - def opt_version(self): - """ - Display OONI version and exit. - """ - print "OONI version:", __version__ - sys.exit(0) - - class Options(usage.Options, app.ReactorSelectionMixin): synopsis = """%s [options] [[file|package|module|TestCase|testmethod]...] """ % (os.path.basename(sys.argv[0]),) @@ -136,12 +105,11 @@ class Options(usage.Options, app.ReactorSelectionMixin): ["nopm", None, "don't automatically jump into debugger for " "postmorteming of exceptions"], ["dry-run", 'n', "do everything but run the tests"], - ["force-gc", None, "Have Trial run gc.collect() before and " + ["force-gc", None, "Have OONI run gc.collect() before and " "after each test case."], ["profile", None, "Run tests under the Python profiler"], ["unclean-warnings", None, "Turn dirty reactor errors into warnings"], - ["until-failure", "u", "Repeat test until it fails"], ["no-recurse", "N", "Don't recurse into packages"], ['help-reporters', None, "Help on available output plugins (reporters)"] @@ -153,13 +121,12 @@ class Options(usage.Options, app.ReactorSelectionMixin): "Run tests in random order using the specified seed"], ['temp-directory', None, '_trial_temp', 'Path to use as working directory for tests.'], - ['reporter', None, 'verbose', + ['reporter', None, 'default', 'The reporter to use for this test run. See --help-reporters for ' 'more info.']]
compData = usage.Completions( optActions={"tbformat": usage.CompleteList(["plain", "emacs", "cgitb"]), - "reporter": _reporterAction, "logfile": usage.CompleteFiles(descr="log file name"), "random": usage.Completer(descr="random seed")}, extraActions=[usage.CompleteFiles( @@ -167,7 +134,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): repeat=True)], )
- fallbackReporter = reporter.TreeReporter + fallbackReporter = reporter.OONIReporter tracer = None
def __init__(self): @@ -208,7 +175,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): # value to the test suite as a module. # # This parameter allows automated processes (like Buildbot) to pass - # a list of files to Trial with the general expectation of "these files, + # a list of files to OONI with the general expectation of "these files, # whatever they are, will get tested" if not os.path.isfile(filename): sys.stderr.write("File %r doesn't exist\n" % (filename,)) @@ -300,21 +267,11 @@ class Options(usage.Options, app.ReactorSelectionMixin): self['tests'].update(args)
- def _loadReporterByName(self, name): - for p in plugin.getPlugins(itrial.IReporter): - qual = "%s.%s" % (p.module, p.klass) - if p.longOpt == name: - return reflect.namedAny(qual) - raise usage.UsageError("Only pass names of Reporter plugins to " - "--reporter. See --help-reporters for " - "more info.") - - def postOptions(self): # Only load reporters now, as opposed to any earlier, to avoid letting # application-defined plugins muck up reactor selecting by importing # t.i.reactor and causing the default to be installed. - self['reporter'] = self._loadReporterByName(self['reporter']) + self['reporter'] = reporter.OONIReporter
if 'tbformat' not in self: self['tbformat'] = 'default' @@ -338,10 +295,10 @@ def _initialDebugSetup(config): def _getSuite(config): loader = _getLoader(config) recurse = not config['no-recurse'] + print "loadByNames %s" % config['tests'] return loader.loadByNames(config['tests'], recurse)
- def _getLoader(config): loader = runner.TestLoader() if config['random']: @@ -349,8 +306,6 @@ def _getLoader(config): randomer.seed(config['random']) loader.sorter = lambda x : randomer.random() print 'Running tests shuffled with seed %d\n' % config['random'] - if not config['until-failure']: - loader.suiteFactory = runner.DestructiveTestSuite return loader
@@ -358,10 +313,11 @@ def _getLoader(config): def _makeRunner(config): mode = None if config['debug']: - mode = runner.TrialRunner.DEBUG + mode = runner.OONIRunner.DEBUG if config['dry-run']: - mode = runner.TrialRunner.DRY_RUN - return runner.TrialRunner(config['reporter'], + mode = runner.OONIRunner.DRY_RUN + print "using %s" % config['reporter'] + return runner.OONIRunner(config['reporter'], mode=mode, profile=config['profile'], logfile=config['logfile'], @@ -384,10 +340,7 @@ def run(): _initialDebugSetup(config) trialRunner = _makeRunner(config) suite = _getSuite(config) - if config['until-failure']: - test_result = trialRunner.runUntilFailure(suite) - else: - test_result = trialRunner.run(suite) + test_result = trialRunner.run(suite) if config.tracer: sys.settrace(None) results = config.tracer.results() diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 5fad85e..2b1e87c 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -22,6 +22,7 @@ class OONITest(object): developer to benefit from OONIs reporting system and command line argument parsing system. """ + name = "oonitest" # By default we set this to False, meaning that we don't block blocking = False reactor = None diff --git a/ooni/reporter.py b/ooni/reporter.py index 0ecf2ea..d20160f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -1,13 +1,11 @@ from twisted.trial import reporter
-class TestResult(reporter.TestResult): - """ - Accumulates the results of several ooni.nettest.TestCases. - - The output format of a TestResult is YAML and it will contain all the basic - information that a test result should contain. - """ - def __init__(self): - super(TestResult, self).__init__() +class OONIReporter(reporter.Reporter): + + def startTest(self, test, input=None): + print "Running %s" % test + print "Input %s" % input + self._input = input + super(OONIReporter, self).startTest(test)
diff --git a/ooni/runner.py b/ooni/runner.py index d7caa9d..c6ad90b 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -1,7 +1,178 @@ -from twisted.trial import runner +import types +import time +import inspect
+from twisted.internet import defer +from twisted.trial import unittest +from twisted.trial.runner import TrialRunner, TestLoader +from twisted.trial.runner import isPackage, isTestCase
-class TestLoader(runner.TestLoader): - pass +from ooni import nettest +from ooni.plugoo import tests as oonitests + +def isLegacyTest(obj): + """ + Returns True if the test in question is written using the OONITest legacy + class. + We do this for backward compatibility of the OONIProbe API. + """ + try: + return issubclass(obj, oonitests.OONITest) + except TypeError: + return False + +def adaptLegacyTest(obj): + """ + We take a legacy OONITest class and convert it into a nettest.TestCase. + This allows backward compatibility of old OONI tests. + + XXX perhaps we could implement another extra layer that makes the even + older test cases compatible with the new OONI. + """ + class LegacyOONITest(nettest.TestCase): + pass + + +class LoggedSuite(nettest.TestSuite): + """ + Any errors logged in this suite will be reported to the L{TestResult} + object. + """ + + def run(self, result, input): + """ + Run the suite, storing all errors in C{result}. If an error is logged + while no tests are running, then it will be added as an error to + C{result}. + + @param result: A L{TestResult} object. + """ + observer = unittest._logObserver + observer._add() + super(LoggedSuite, self).run(result, input) + observer._remove() + for error in observer.getErrors(): + result.addError(TestHolder(NOT_IN_TEST), error) + observer.flushErrors() + + +class OONISuite(nettest.TestSuite): + """ + Suite to wrap around every single test in a C{trial} run. Used internally + by OONI to set up things necessary for OONI tests to work, regardless of + what context they are run in. + """ + + def __init__(self, tests=()): + suite = LoggedSuite(tests) + super(OONISuite, self).__init__([suite]) + + def _bail(self): + from twisted.internet import reactor + d = defer.Deferred() + reactor.addSystemEventTrigger('after', 'shutdown', + lambda: d.callback(None)) + reactor.fireSystemEvent('shutdown') # radix's suggestion + # As long as TestCase does crap stuff with the reactor we need to + # manually shutdown the reactor here, and that requires util.wait + # :( + # so that the shutdown event completes + nettest.TestCase('mktemp')._wait(d) + + def run(self, result, input): + try: + nettest.TestSuite.run(self, result, input) + finally: + self._bail() + + +class OONIRunner(TrialRunner): + def run(self, test): + return TrialRunner.run(self, test) + + def _runWithoutDecoration(self, test): + """ + Private helper that runs the given test but doesn't decorate it. + """ + result = self._makeResult() + # decorate the suite with reactor cleanup and log starting + # This should move out of the runner and be presumed to be + # present + suite = OONISuite([test]) + print "HERE IS THE TEST:" + print test + print "-------------" + try: + inputs = test.inputs + except: + inputs = [None] + + startTime = time.time() + if self.mode == self.DRY_RUN: + for single in nettest._iterateTests(suite): + input = None + if type(single) == type(tuple()): + single, input = single + result.startTest(single, input) + result.addSuccess(single) + result.stopTest(single) + else: + if self.mode == self.DEBUG: + # open question - should this be self.debug() instead. + debugger = self._getDebugger() + run = lambda x: debugger.runcall(suite.run, result, x) + else: + run = lambda x: suite.run(result, x) + + oldDir = self._setUpTestdir() + try: + self._setUpLogFile() + # XXX work on this better + for input in inputs: + run(input) + finally: + self._tearDownLogFile() + self._tearDownTestdir(oldDir) + + endTime = time.time() + done = getattr(result, 'done', None) + if done is None: + warnings.warn( + "%s should implement done() but doesn't. Falling back to " + "printErrors() and friends." % reflect.qual(result.__class__), + category=DeprecationWarning, stacklevel=3) + result.printErrors() + result.writeln(result.separator) + result.writeln('Ran %d tests in %.3fs', result.testsRun, + endTime - startTime) + result.write('\n') + result.printSummary() + else: + result.done() + return result + + +class TestLoader(TestLoader): + """ + Reponsible for finding the modules that can work as tests and running them. + If we detect that a certain test is written using the legacy OONI API we + will wrap it around a next gen class to make it work here too. + """ + def __init__(self): + super(TestLoader, self).__init__() + self.suiteFactory = nettest.TestSuite + + def findTestClasses(self, module): + classes = [] + for name, val in inspect.getmembers(module): + if isTestCase(val): + classes.append(val) + # This is here to allow backward compatibility with legacy OONI + # tests. + elif isLegacyTest(val): + #val = adaptLegacyTest(val) + classes.append(val) + return self.sort(classes) + #return runner.TestLoader.findTestClasses(self, module)