commit cbfcbdd0344d8e9a80d565ba01b64c7a40c97352 Author: Arturo Filastò arturo@filasto.net Date: Thu Sep 20 19:26:46 2012 +0000

Implement working trial inspired OONIprobe refactoring --- ooni/nettest.py | 41 +++---- ooni/oonicli.py | 66 ++++------- ooni/plugoo/tests.py | 1 + ooni/runner.py | 330 ++++++++++++------------------------------------- 4 files changed, 122 insertions(+), 316 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py index 4ab1e0f..0302d25 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -1,5 +1,4 @@ import itertools - from twisted.python import log from twisted.trial import unittest, itrial

@@ -42,35 +41,29 @@ class TestSuiteFactory(object): self._idx += 1 return new_test_suite

-class TestSuite(pyunit.TestSuite): - def __init__(self, tests=()): - self._tests = [] - self.input = None - self._idx = 0 - self.addTests(tests) - - def __repr__(self): - return "<%s input=%s tests=%s>" % (self.__class__, - self.input, self._tests) - - def run(self, result): - """ - Call C{run} on every member of the suite. - """ - # we implement this because Python 2.3 unittest defines this code - # in __call__, whereas 2.4 defines the code in run. - for i, test in enumerate(self._tests): +class InputTestSuite(pyunit.TestSuite): + def run(self, result, idx=0): + self._idx = idx + while self._tests: if result.shouldStop: break - test.input = self.input - test._idx = self._idx + i - test(result) - + test = self._tests.pop(0) + try: + test.input = self.input + test._idx = self._idx + print "IDX: %s" % self._idx + test(result) + except: + test(result) + self._idx += 1 return result

class TestCase(unittest.TestCase): name = "DefaultTestName" - inputs = [None] + inputs = [] + + def getOptions(self): + return {'inputs': self.inputs}

def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index be73d30..f0287b4 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -15,6 +15,15 @@

import sys, os, random, gc, time, warnings

+import unittest +import inspect + +from ooni.input import InputUnitFactory +from ooni.reporter import ReporterFactory +from ooni.nettest import InputTestSuite +from ooni.plugoo import tests +from ooni import nettest, runner, reporter + from twisted.internet import defer from twisted.application import app from twisted.python import usage, reflect, failure, log @@ -24,7 +33,6 @@ from twisted.python.util import spewer from twisted.python.compat import set from twisted.trial import itrial from twisted.trial import runner as irunner -from ooni import runner, reporter

def _parseLocalVariables(line): @@ -132,11 +140,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): repeat=True)], )

- fallbackReporter = reporter.OONIReporter tracer = None

def __init__(self): - self['tests'] = set() + self['test'] = None usage.Options.__init__(self)

@@ -178,7 +185,10 @@ class Options(usage.Options, app.ReactorSelectionMixin): if not os.path.isfile(filename): sys.stderr.write("File %r doesn't exist\n" % (filename,)) return + filename = os.path.abspath(filename) + self['test'] = filename + if isTestFile(filename): self['tests'].add(filename) else: @@ -248,7 +258,10 @@ class Options(usage.Options, app.ReactorSelectionMixin):

def parseArgs(self, *args): - self['tests'].update(args) + try: + self['test'] = args[0] + except: + raise usage.UsageError("No test filename specified!")

def postOptions(self): @@ -265,39 +278,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): "--nopm ") failure.DO_POST_MORTEM = False

- -def _initialDebugSetup(config): - # do this part of debug setup first for easy debugging of import failures - if config['debug']: - failure.startDebugMode() - if config['debug'] or config['debug-stacktraces']: - defer.setDebugging(True) - - -def _getSuitesAndInputs(config): - #loader = irunner.TestLoader() - loader = runner.NetTestLoader() - recurse = not config['no-recurse'] - print "loadByNames %s" % config['tests'] - inputs, suites = loader.loadByNamesWithInput(config['tests'], recurse) - return inputs, suites - -def _makeRunner(config): - mode = None - if config['debug']: - mode = runner.OONIRunner.DEBUG - print "using %s" % config['reporter'] - return runner.OONIRunner(config['reporter'], - reportfile=config["reportfile"], - mode=mode, - logfile=config['logfile'], - tracebackFormat=config['tbformat'], - realTimeErrors=config['rterrors'], - uncleanWarnings=config['unclean-warnings'], - workingDirectory=config['temp-directory'], - forceGarbageCollection=config['force-gc']) - - def run(): if len(sys.argv) == 1: sys.argv.append("--help") @@ -307,10 +287,10 @@ def run(): except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue)

- _initialDebugSetup(config) - trialRunner = _makeRunner(config) - inputs, testSuites = _getSuitesAndInputs(config) - log.startLogging(sys.stdout) - for i, suite in enumerate(testSuites): - test_result = trialRunner.run(suite, inputs[i]) + file_name = os.path.abspath('nettests/simpletest.py') + classes = runner.findTestClassesFromFile(config['test']) + casesList, options = runner.loadTestsAndOptions(classes) + for idx, cases in enumerate(casesList): + orunner = runner.ORunner(cases, options[idx]) + orunner.run()

diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 2b1e87c..e4ee187 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -133,6 +133,7 @@ class OONITest(object): @param args: the asset(s) lines that we are working on. """ self.start_time = date.now() + print "FOWID" log.msg("Starting test %s" % self.__class__) return self._do_experiment(args)

diff --git a/ooni/runner.py b/ooni/runner.py index 604942d..ddde83b 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -13,9 +13,16 @@ from twisted.trial.runner import filenameToModule, _importFromFile

from ooni.reporter import ReporterFactory from ooni.input import InputUnitFactory +from ooni.nettest import InputTestSuite from ooni import nettest from ooni.plugoo import tests as oonitests

+def isTestCase(thing): + try: + return issubclass(thing, unittest.TestCase) + except TypeError: + return False + def isLegacyTest(obj): """ Returns True if the test in question is written using the OONITest legacy @@ -23,7 +30,10 @@ def isLegacyTest(obj): We do this for backward compatibility of the OONIProbe API. """ try: - return issubclass(obj, oonitests.OONITest) + if issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest: + return True + else: + return False except TypeError: return False

@@ -36,267 +46,89 @@ def adaptLegacyTest(obj, inputs=[None]): older test cases compatible with the new OONI. """ class LegacyOONITest(nettest.TestCase): - inputs = [1] + inputs = [None] original_test = obj

+ @defer.inlineCallbacks def test_start_legacy_test(self): print "bla bla bla" - my_test = self.original_test() - print my_test - print "foobat" - my_test.startTest(self.input) - print "HHAHAHA" + print self.original_test + my_test = self.original_test(None, None, None) + yield my_test.startTest(self.input)

return LegacyOONITest

-class LoggedSuite(nettest.TestSuite): - """ - Any errors logged in this suite will be reported to the L{TestResult} - object. - """ - - def run(self, result): - """ - Run the suite, storing all errors in C{result}. If an error is logged - while no tests are running, then it will be added as an error to - C{result}. - - @param result: A L{TestResult} object. - """ - observer = unittest._logObserver - observer._add() - super(LoggedSuite, self).run(result) - observer._remove() - for error in observer.getErrors(): - result.addError(TestHolder(NOT_IN_TEST), error) - observer.flushErrors() +def findTestClassesFromFile(filename): + classes = []

+ print "FILENAME %s" % filename + module = filenameToModule(filename) + for name, val in inspect.getmembers(module): + if isTestCase(val): + classes.append(val) + elif isLegacyTest(val): + classes.append(adaptLegacyTest(val)) + return classes

-class OONISuite(nettest.TestSuite): - """ - Suite to wrap around every single test in a C{trial} run. Used internally - by OONI to set up things necessary for OONI tests to work, regardless of - what context they are run in. - """ +def makeTestCases(klass, tests, methodPrefix): + cases = [] + for test in tests: + cases.append(klass(methodPrefix+test)) + return cases

- def __init__(self, tests=()): - suite = LoggedSuite(tests) - super(OONISuite, self).__init__([suite]) - - def _bail(self): - from twisted.internet import reactor - d = defer.Deferred() - reactor.addSystemEventTrigger('after', 'shutdown', - lambda: d.callback(None)) - reactor.fireSystemEvent('shutdown') # radix's suggestion - # As long as TestCase does crap stuff with the reactor we need to - # manually shutdown the reactor here, and that requires util.wait - # :( - # so that the shutdown event completes - nettest.TestCase('mktemp')._wait(d) - - def run(self, result): - try: - nettest.TestSuite.run(self, result) - finally: - self._bail() - - -class NetTestLoader(TestLoader): - """ - Reponsible for finding the modules that can work as tests and running them. - If we detect that a certain test is written using the legacy OONI API we - will wrap it around a next gen class to make it work here too. - - XXX This class needs to be cleaned up a *lot* of all the things we actually - don't need. - """ +def loadTestsAndOptions(classes): methodPrefix = 'test' - modulePrefix = 'test_' - - def __init__(self): - self.suiteFactory = nettest.TestSuite - self._importErrors = [] - - def findTestClasses(self, module): - classes = [] - for name, val in inspect.getmembers(module): - if isTestCase(val): - classes.append(val) - # This is here to allow backward compatibility with legacy OONI - # tests. - elif isLegacyTest(val): - print "adapting! %s" % val - val = adaptLegacyTest(val) - classes.append(val) - return classes - - def loadClass(self, klass): - """ - Given a class which contains test cases, return a sorted list of - C{TestCase} instances. - """ - if not (isinstance(klass, type) or isinstance(klass, types.ClassType)): - raise TypeError("%r is not a class" % (klass,)) - if not isTestCase(klass): - raise ValueError("%r is not a test case" % (klass,)) - names = self.getTestCaseNames(klass) - tests = [] - for name in names: - tests.append(self._makeCase(klass, self.methodPrefix+name)) - - suite = self.suiteFactory(tests) - print "**+*" - print tests - print "**+*" - - return suite - loadTestsFromTestCase = loadClass - - def findAllInputs(self, thing): - testClasses = self.findTestClasses(thing) - # XXX will there ever be more than 1 test class with inputs? - for klass in testClasses: - try: - inputs = klass.inputs - except: - pass - return inputs - - def loadByNamesWithInput(self, names, recurse=False): - """ - Construct a OONITestSuite containing all the tests found in 'names', where - names is a list of fully qualified python names and/or filenames. The - suite returned will have no duplicate tests, even if the same object - is named twice. - - This test suite will have set the attribute inputs to the inputs found - inside of the tests. - """ - inputs = [] - things = [] - errors = [] - for name in names: - try: - thing = self.findByName(name) - things.append(thing) - except: - errors.append(ErrorHolder(name, failure.Failure())) - suites = [] - for thing in self._uniqueTests(things): - inputs.append(self.findAllInputs(thing)) - suite = self.loadAnything(thing, recurse) - suites.append(suite) - - suites.extend(errors) - return inputs, suites - -class OONIRunner(object): - """ - A specialised runner that is used by the ooniprobe frontend to run tests. - Heavily inspired by the trial TrialRunner class. - """ - - DEBUG = 'debug' - DRY_RUN = 'dry-run' - - def _getDebugger(self): - dbg = pdb.Pdb() + suiteFactory = InputTestSuite + options = [] + testCases = [] + for klass in classes: try: - import readline - except ImportError: - print "readline module not available" - sys.exc_clear() - for path in ('.pdbrc', 'pdbrc'): - if os.path.exists(path): - try: - rcFile = file(path, 'r') - except IOError: - sys.exc_clear() - else: - dbg.rcLines.extend(rcFile.readlines()) - return dbg - - - def _setUpTestdir(self): - self._tearDownLogFile() - currentDir = os.getcwd() - base = filepath.FilePath(self.workingDirectory) - testdir, self._testDirLock = util._unusedTestDirectory(base) - os.chdir(testdir.path) - return currentDir - - - def _tearDownTestdir(self, oldDir): - os.chdir(oldDir) - self._testDirLock.unlock() - - - _log = log - def _makeResult(self): - reporter = self.reporterFactory(self.stream, self.tbformat, - self.rterrors, self._log) - if self.uncleanWarnings: - reporter = UncleanWarningsReporterWrapper(reporter) - return reporter - - def __init__(self, reporterFactory, - reportfile="report.yaml", - mode=None, - logfile='test.log', - stream=sys.stdout, - profile=False, - tracebackFormat='default', - realTimeErrors=False, - uncleanWarnings=False, - workingDirectory=None, - forceGarbageCollection=False): - self.reporterFactory = reporterFactory - self._reportfile = reportfile - self.logfile = logfile - self.mode = mode - self.stream = stream - self.tbformat = tracebackFormat - self.rterrors = realTimeErrors - self.uncleanWarnings = uncleanWarnings - self._result = None - self.workingDirectory = workingDirectory or '_trial_temp' - self._logFileObserver = None - self._logFileObject = None - self._forceGarbageCollection = forceGarbageCollection - if profile: - self.run = util.profiled(self.run, 'profile.data') - - def _tearDownLogFile(self): - if self._logFileObserver is not None: - log.removeObserver(self._logFileObserver.emit) - self._logFileObserver = None - if self._logFileObject is not None: - self._logFileObject.close() - self._logFileObject = None - - def _setUpLogFile(self): - self._tearDownLogFile() - if self.logfile == '-': - logFile = sys.stdout + k = klass() + options.append(k.getOptions()) + except AttributeError: + options.append([]) + + tests = reflect.prefixedMethodNames(klass, methodPrefix) + if tests: + cases = makeTestCases(klass, tests, methodPrefix) + testCases.append(cases) else: - logFile = file(self.logfile, 'a') - self._logFileObject = logFile - self._logFileObserver = log.FileLogObserver(logFile) - log.startLoggingWithObserver(self._logFileObserver.emit, 0) + options.pop() + + return testCases, options + +class ORunner(object): + def __init__(self, cases, options=None): + self.baseSuite = InputTestSuite + self.cases = cases + self.options = options + self.inputs = options['inputs'] + self.reporterFactory = ReporterFactory(open('foo.log', 'a+'), + testSuite=self.baseSuite(self.cases)) + + def runWithInputUnit(self, inputUnit): + idx = 0 + result = self.reporterFactory.create() + for input in inputUnit: + suite = self.baseSuite(self.cases) + suite.input = input + suite(result, idx) + + # XXX refactor all of this index bullshit to avoid having to pass + # this index around. Probably what I want to do is go and make + # changes to report to support the concept of having multiple runs + # of the same test. + # We currently need to do this addition in order to get the number + # of times the test cases that have run inside of the test suite. + idx += (suite._idx - idx) + + result.done() + + def run(self): + self.reporterFactory.writeHeader() + + for inputUnit in InputUnitFactory(self.inputs): + self.runWithInputUnit(inputUnit)

- def run(self, tests, inputs=[None]): - """ - Run the test or suite and return a result object. - """ - reporterFactory = ReporterFactory(open(self._reportfile, 'a+'), - testSuite=tests) - reporterFactory.writeHeader() - for inputUnit in InputUnitFactory(inputs): - testSuiteFactory = nettest.TestSuiteFactory(inputUnit, tests, nettest.TestSuite) - testUnitReport = reporterFactory.create() - for suite in testSuiteFactory: - suite(testUnitReport) - testUnitReport.done()