commit 7dce07bfc58566d2fdec4eec8a831d4ecd545d5f Author: Arturo Filastò arturo@filasto.net Date: Tue Sep 18 17:51:07 2012 +0000

Made some progress, not there yet. * After this commit I am going to throw away a lot of code and start over --- ooni/input.py | 10 ++- ooni/nettest.py | 50 +++++++-- ooni/oonicli.py | 56 +++-------- ooni/reporter.py | 5 +- ooni/runner.py | 305 +++++++++-------------------------------------------- 5 files changed, 117 insertions(+), 309 deletions(-)

diff --git a/ooni/input.py b/ooni/input.py index f534393..b931b82 100644 --- a/ooni/input.py +++ b/ooni/input.py @@ -43,7 +43,7 @@ class InputUnit(object): passed onto a TestCase. """ def __init__(self, inputs=[]): - self._inputs = inputs + self._inputs = iter(inputs)

def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self._inputs) @@ -53,7 +53,13 @@ class InputUnit(object): self._inputs.append(input)

def __iter__(self): - return iter(self._inputs) + return self + + def next(self): + try: + return self._inputs.next() + except: + raise StopIteration

def append(self, input): self._inputs.append(input) diff --git a/ooni/nettest.py b/ooni/nettest.py index 0c8858b..4ab1e0f 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -19,28 +19,60 @@ def _iterateTests(testSuiteOrCase): yield subtest

+class TestSuiteFactory(object): + def __init__(self, inputUnit, tests, basesuite): + self._baseSuite = basesuite + self._inputUnit = inputUnit + self._idx = 0 + self.tests = tests + + def __iter__(self): + return self + + def next(self): + try: + next_input = self._inputUnit.next() + print "Now dealing with %s %s" % (next_input, self._idx) + except: + raise StopIteration + new_test_suite = self._baseSuite(self.tests) + new_test_suite.input = next_input + new_test_suite._idx = self._idx + + self._idx += 1 + return new_test_suite + class TestSuite(pyunit.TestSuite): - inputUnit = [None] + def __init__(self, tests=()): + self._tests = [] + self.input = None + self._idx = 0 + self.addTests(tests) + def __repr__(self): - return "<%s input=%s, tests=%s>" % (self.__class__, self.inputUnit, list(self)) + return "<%s input=%s tests=%s>" % (self.__class__, + self.input, self._tests)

- def run(self, result, inputUnit=[None]): + def run(self, result): """ Call C{run} on every member of the suite. """ # we implement this because Python 2.3 unittest defines this code # in __call__, whereas 2.4 defines the code in run. - idx = 0 - for input, test in itertools.product(inputUnit, self._tests): + for i, test in enumerate(self._tests): if result.shouldStop: break - self.inputUnit = inputUnit - test.input = input - test.idx = idx + test.input = self.input + test._idx = self._idx + i test(result) - idx += 1

return result

class TestCase(unittest.TestCase): name = "DefaultTestName" + inputs = [None] + + def __repr__(self): + return "<%s inputs=%s>" % (self.__class__, self.inputs) + + diff --git a/ooni/oonicli.py b/ooni/oonicli.py index fa7742f..be73d30 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -17,12 +17,13 @@ import sys, os, random, gc, time, warnings

from twisted.internet import defer from twisted.application import app -from twisted.python import usage, reflect, failure +from twisted.python import usage, reflect, failure, log from twisted.python.filepath import FilePath from twisted import plugin from twisted.python.util import spewer from twisted.python.compat import set from twisted.trial import itrial +from twisted.trial import runner as irunner from ooni import runner, reporter

@@ -104,10 +105,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): "callback stack traces"], ["nopm", None, "don't automatically jump into debugger for " "postmorteming of exceptions"], - ["dry-run", 'n', "do everything but run the tests"], ["force-gc", None, "Have OONI run gc.collect() before and " "after each test case."], - ["profile", None, "Run tests under the Python profiler"], ["unclean-warnings", None, "Turn dirty reactor errors into warnings"], ["no-recurse", "N", "Don't recurse into packages"], @@ -118,9 +117,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): optParameters = [ ["reportfile", "o", "report.yaml", "report file name"], ["logfile", "l", "test.log", "log file name"], - ["random", "z", None, - "Run tests in random order using the specified seed"], - ['temp-directory', None, '_trial_temp', + ['temp-directory', None, '_ooni_temp', 'Path to use as working directory for tests.'], ['reporter', None, 'default', 'The reporter to use for this test run. See --help-reporters for ' @@ -129,7 +126,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): compData = usage.Completions( optActions={"tbformat": usage.CompleteList(["plain", "emacs", "cgitb"]), "logfile": usage.CompleteFiles(descr="log file name"), - "random": usage.Completer(descr="random seed")}, + }, extraActions=[usage.CompleteFiles( "*.py", descr="file | module | package | TestCase | testMethod", repeat=True)], @@ -238,20 +235,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): "argument to recursionlimit must be an integer")

- def opt_random(self, option): - try: - self['random'] = long(option) - except ValueError: - raise usage.UsageError( - "Argument to --random must be a positive integer") - else: - if self['random'] < 0: - raise usage.UsageError( - "Argument to --random must be a positive integer") - elif self['random'] == 0: - self['random'] = long(time.time() * 100) - - def opt_without_module(self, option): """ Fake the lack of the specified modules, separated with commas. @@ -283,7 +266,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): failure.DO_POST_MORTEM = False

- def _initialDebugSetup(config): # do this part of debug setup first for easy debugging of import failures if config['debug']: @@ -292,35 +274,22 @@ def _initialDebugSetup(config): defer.setDebugging(True)

- -def _getSuites(config): - loader = _getLoader(config) +def _getSuitesAndInputs(config): + #loader = irunner.TestLoader() + loader = runner.NetTestLoader() recurse = not config['no-recurse'] print "loadByNames %s" % config['tests'] - return loader.loadByNames(config['tests'], recurse) - - -def _getLoader(config): - loader = runner.NetTestLoader() - if config['random']: - randomer = random.Random() - randomer.seed(config['random']) - loader.sorter = lambda x : randomer.random() - print 'Running tests shuffled with seed %d\n' % config['random'] - return loader - + inputs, suites = loader.loadByNamesWithInput(config['tests'], recurse) + return inputs, suites

def _makeRunner(config): mode = None if config['debug']: mode = runner.OONIRunner.DEBUG - if config['dry-run']: - mode = runner.OONIRunner.DRY_RUN print "using %s" % config['reporter'] return runner.OONIRunner(config['reporter'], reportfile=config["reportfile"], mode=mode, - profile=config['profile'], logfile=config['logfile'], tracebackFormat=config['tbformat'], realTimeErrors=config['rterrors'], @@ -340,7 +309,8 @@ def run():

_initialDebugSetup(config) trialRunner = _makeRunner(config) - suites = _getSuites(config) - for suite in suites: - test_result = trialRunner.run(suite) + inputs, testSuites = _getSuitesAndInputs(config) + log.startLogging(sys.stdout) + for i, suite in enumerate(testSuites): + test_result = trialRunner.run(suite, inputs[i])

diff --git a/ooni/reporter.py b/ooni/reporter.py index 07cffad..14297cd 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -89,7 +89,7 @@ class OONIReporter(OReporter):

def getTestIndex(self, test): try: - idx = test.idx + idx = test._idx except: idx = 0 return idx @@ -109,7 +109,8 @@ class OONIReporter(OReporter): self._tests[idx]['input'] = test.input self._tests[idx]['idx'] = idx self._tests[idx]['name'] = test.name - self._tests[idx]['test'] = test + #self._tests[idx]['test'] = test + print "Now starting %s" % self._tests[idx]

def stopTest(self, test): diff --git a/ooni/runner.py b/ooni/runner.py index 12ab9ad..604942d 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -4,7 +4,7 @@ import types import time import inspect

-from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.python import reflect, log, failure from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader @@ -27,7 +27,7 @@ def isLegacyTest(obj): except TypeError: return False

-def adaptLegacyTest(obj): +def adaptLegacyTest(obj, inputs=[None]): """ We take a legacy OONITest class and convert it into a nettest.TestCase. This allows backward compatibility of old OONI tests. @@ -36,8 +36,18 @@ def adaptLegacyTest(obj): older test cases compatible with the new OONI. """ class LegacyOONITest(nettest.TestCase): - pass + inputs = [1] + original_test = obj

+ def test_start_legacy_test(self): + print "bla bla bla" + my_test = self.original_test() + print my_test + print "foobat" + my_test.startTest(self.input) + print "HHAHAHA" + + return LegacyOONITest

class LoggedSuite(nettest.TestSuite): @@ -93,11 +103,14 @@ class OONISuite(nettest.TestSuite): self._bail()

-class NetTestLoader(object): +class NetTestLoader(TestLoader): """ Reponsible for finding the modules that can work as tests and running them. If we detect that a certain test is written using the legacy OONI API we will wrap it around a next gen class to make it work here too. + + XXX This class needs to be cleaned up a *lot* of all the things we actually + don't need. """ methodPrefix = 'test' modulePrefix = 'test_' @@ -106,67 +119,19 @@ class NetTestLoader(object): self.suiteFactory = nettest.TestSuite self._importErrors = []

- def findTestClasses(self, module): classes = [] for name, val in inspect.getmembers(module): - try: - inputs = val.inputs - except: - inputs = None if isTestCase(val): - classes.append((val, inputs)) + classes.append(val) # This is here to allow backward compatibility with legacy OONI # tests. elif isLegacyTest(val): - #val = adaptLegacyTest(val) - classes.append((val, inputs)) + print "adapting! %s" % val + val = adaptLegacyTest(val) + classes.append(val) return classes

- def findByName(self, name): - """ - Return a Python object given a string describing it. - - @param name: a string which may be either a filename or a - fully-qualified Python name. - - @return: If C{name} is a filename, return the module. If C{name} is a - fully-qualified Python name, return the object it refers to. - """ - if os.path.exists(name): - return filenameToModule(name) - return reflect.namedAny(name) - - - def loadModule(self, module): - """ - Return a test suite with all the tests from a module. - - Included are TestCase subclasses and doctests listed in the module's - __doctests__ module. If that's not good for you, put a function named - either C{testSuite} or C{test_suite} in your module that returns a - TestSuite, and I'll use the results of that instead. - - If C{testSuite} and C{test_suite} are both present, then I'll use - C{testSuite}. - """ - ## XXX - should I add an optional parameter to disable the check for - ## a custom suite. - ## OR, should I add another method - if not isinstance(module, types.ModuleType): - raise TypeError("%r is not a module" % (module,)) - if hasattr(module, 'testSuite'): - return module.testSuite() - elif hasattr(module, 'test_suite'): - return module.test_suite() - - suite = self.suiteFactory() - for testClass, inputs in self.findTestClasses(module): - testCases = self.loadClass(testClass) - - return testCases - loadTestsFromModule = loadModule - def loadClass(self, klass): """ Given a class which contains test cases, return a sorted list of @@ -182,186 +147,50 @@ class NetTestLoader(object): tests.append(self._makeCase(klass, self.methodPrefix+name))

suite = self.suiteFactory(tests) - suite.inputs = klass.inputs - return suite - loadTestsFromTestCase = loadClass - - def getTestCaseNames(self, klass): - """ - Given a class that contains C{TestCase}s, return a list of names of - methods that probably contain tests. - """ - return reflect.prefixedMethodNames(klass, self.methodPrefix) - - def loadMethod(self, method): - """ - Given a method of a C{TestCase} that represents a test, return a - C{TestCase} instance for that test. - """ - if not isinstance(method, types.MethodType): - raise TypeError("%r not a method" % (method,)) - return self._makeCase(method.im_class, _getMethodNameInClass(method)) - - def _makeCase(self, klass, methodName): - return klass(methodName) + print "**+*" + print tests + print "**+*"

- def loadPackage(self, package, recurse=False): - """ - Load tests from a module object representing a package, and return a - TestSuite containing those tests. - - Tests are only loaded from modules whose name begins with 'test_' - (or whatever C{modulePrefix} is set to). - - @param package: a types.ModuleType object (or reasonable facsimilie - obtained by importing) which may contain tests. - - @param recurse: A boolean. If True, inspect modules within packages - within the given package (and so on), otherwise, only inspect modules - in the package itself. - - @raise: TypeError if 'package' is not a package. - - @return: a TestSuite created with my suiteFactory, containing all the - tests. - """ - if not isPackage(package): - raise TypeError("%r is not a package" % (package,)) - pkgobj = modules.getModule(package.__name__) - if recurse: - discovery = pkgobj.walkModules() - else: - discovery = pkgobj.iterModules() - discovered = [] - for disco in discovery: - if disco.name.split(".")[-1].startswith(self.modulePrefix): - discovered.append(disco) - suite = self.suiteFactory() - for modinfo in self.sort(discovered): - try: - module = modinfo.load() - except: - thingToAdd = ErrorHolder(modinfo.name, failure.Failure()) - else: - thingToAdd = self.loadModule(module) - suite.addTest(thingToAdd) return suite + loadTestsFromTestCase = loadClass

- def loadDoctests(self, module): - """ - Return a suite of tests for all the doctests defined in C{module}. - - @param module: A module object or a module name. - """ - if isinstance(module, str): + def findAllInputs(self, thing): + testClasses = self.findTestClasses(thing) + # XXX will there ever be more than 1 test class with inputs? + for klass in testClasses: try: - module = reflect.namedAny(module) + inputs = klass.inputs except: - return ErrorHolder(module, failure.Failure()) - if not inspect.ismodule(module): - warnings.warn("trial only supports doctesting modules") - return - extraArgs = {} - if sys.version_info > (2, 4): - # Work around Python issue2604: DocTestCase.tearDown clobbers globs - def saveGlobals(test): - """ - Save C{test.globs} and replace it with a copy so that if - necessary, the original will be available for the next test - run. - """ - test._savedGlobals = getattr(test, '_savedGlobals', test.globs) - test.globs = test._savedGlobals.copy() - extraArgs['setUp'] = saveGlobals - return doctest.DocTestSuite(module, **extraArgs) - - def loadAnything(self, thing, recurse=False): - """ - Given a Python object, return whatever tests that are in it. Whatever - 'in' might mean. - - @param thing: A Python object. A module, method, class or package. - @param recurse: Whether or not to look in subpackages of packages. - Defaults to False. - - @return: A C{TestCase} or C{TestSuite}. - """ - print "Loading anything! %s" % thing - ret = None - if isinstance(thing, types.ModuleType): - if isPackage(thing): - ret = self.loadPackage(thing, recurse) - ret = self.loadModule(thing) - elif isinstance(thing, types.ClassType): - ret = self.loadClass(thing) - elif isinstance(thing, type): - ret = self.loadClass(thing) - elif isinstance(thing, types.MethodType): - ret = self.loadMethod(thing) - if not ret: - raise TypeError("No loader for %r. Unrecognized type" % (thing,)) - try: - ret.inputs = ret.inputs - except: - ret.inputs = [None] - return ret - - def loadByName(self, name, recurse=False): - """ - Given a string representing a Python object, return whatever tests - are in that object. - - If C{name} is somehow inaccessible (e.g. the module can't be imported, - there is no Python object with that name etc) then return an - L{ErrorHolder}. - - @param name: The fully-qualified name of a Python object. - """ - print "Load by Name!" - try: - thing = self.findByName(name) - except: - return ErrorHolder(name, failure.Failure()) - return self.loadAnything(thing, recurse) - loadTestsFromName = loadByName + pass + return inputs

- def loadByNames(self, names, recurse=False): + def loadByNamesWithInput(self, names, recurse=False): """ - Construct a TestSuite containing all the tests found in 'names', where + Construct a OONITestSuite containing all the tests found in 'names', where names is a list of fully qualified python names and/or filenames. The suite returned will have no duplicate tests, even if the same object is named twice. + + This test suite will have set the attribute inputs to the inputs found + inside of the tests. """ - print "Load by Names!" + inputs = [] things = [] errors = [] for name in names: try: - things.append(self.findByName(name)) + thing = self.findByName(name) + things.append(thing) except: errors.append(ErrorHolder(name, failure.Failure())) - suites = [self.loadAnything(thing, recurse) - for thing in self._uniqueTests(things)] - suites.extend(errors) - return suites - #return self.suiteFactory(suites) - - - def _uniqueTests(self, things): - """ - Gather unique suite objects from loaded things. This will guarantee - uniqueness of inherited methods on TestCases which would otherwise hash - to same value and collapse to one test unexpectedly if using simpler - means: e.g. set(). - """ - entries = [] - for thing in things: - if isinstance(thing, types.MethodType): - entries.append((thing, thing.im_class)) - else: - entries.append((thing,)) - return [entry[0] for entry in set(entries)] + suites = [] + for thing in self._uniqueTests(things): + inputs.append(self.findAllInputs(thing)) + suite = self.loadAnything(thing, recurse) + suites.append(suite)

+ suites.extend(errors) + return inputs, suites

class OONIRunner(object): """ @@ -457,47 +286,17 @@ class OONIRunner(object): self._logFileObserver = log.FileLogObserver(logFile) log.startLoggingWithObserver(self._logFileObserver.emit, 0)

- def run(self, test): + def run(self, tests, inputs=[None]): """ Run the test or suite and return a result object. """ - print test - inputs = test.inputs reporterFactory = ReporterFactory(open(self._reportfile, 'a+'), - testSuite=test) + testSuite=tests) reporterFactory.writeHeader() - #testUnitReport = OONIReporter(open('reporting.log', 'a+')) - #testUnitReport.writeHeader(FooTest) for inputUnit in InputUnitFactory(inputs): + testSuiteFactory = nettest.TestSuiteFactory(inputUnit, tests, nettest.TestSuite) testUnitReport = reporterFactory.create() - test(testUnitReport, inputUnit) + for suite in testSuiteFactory: + suite(testUnitReport) testUnitReport.done()

- def _runWithInput(self, test, input): - """ - Private helper that runs the given test with the given input. - """ - result = self._makeResult() - # decorate the suite with reactor cleanup and log starting - # This should move out of the runner and be presumed to be - # present - suite = TrialSuite([test]) - startTime = time.time() - - ## XXX replace this with the actual way of running the test. - run = lambda: suite.run(result) - - oldDir = self._setUpTestdir() - try: - self._setUpLogFile() - run() - finally: - self._tearDownLogFile() - self._tearDownTestdir(oldDir) - - endTime = time.time() - done = getattr(result, 'done', None) - result.done() - return result - -