[tor-commits] [ooni-probe/master] Made some progress, not there yet.
isis at torproject.org
isis at torproject.org
Thu Oct 4 14:41:15 UTC 2012
commit 7dce07bfc58566d2fdec4eec8a831d4ecd545d5f
Author: Arturo Filastò <arturo at filasto.net>
Date: Tue Sep 18 17:51:07 2012 +0000
Made some progress, not there yet.
* After this commit I am going to throw away a lot of code and start over
---
ooni/input.py | 10 ++-
ooni/nettest.py | 50 +++++++--
ooni/oonicli.py | 56 +++--------
ooni/reporter.py | 5 +-
ooni/runner.py | 305 +++++++++--------------------------------------------
5 files changed, 117 insertions(+), 309 deletions(-)
diff --git a/ooni/input.py b/ooni/input.py
index f534393..b931b82 100644
--- a/ooni/input.py
+++ b/ooni/input.py
@@ -43,7 +43,7 @@ class InputUnit(object):
passed onto a TestCase.
"""
def __init__(self, inputs=[]):
- self._inputs = inputs
+ self._inputs = iter(inputs)
def __repr__(self):
return "<%s inputs=%s>" % (self.__class__, self._inputs)
@@ -53,7 +53,13 @@ class InputUnit(object):
self._inputs.append(input)
def __iter__(self):
- return iter(self._inputs)
+ return self
+
+ def next(self):
+ try:
+ return self._inputs.next()
+ except:
+ raise StopIteration
def append(self, input):
self._inputs.append(input)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 0c8858b..4ab1e0f 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -19,28 +19,60 @@ def _iterateTests(testSuiteOrCase):
yield subtest
+class TestSuiteFactory(object):
+ def __init__(self, inputUnit, tests, basesuite):
+ self._baseSuite = basesuite
+ self._inputUnit = inputUnit
+ self._idx = 0
+ self.tests = tests
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ try:
+ next_input = self._inputUnit.next()
+ print "Now dealing with %s %s" % (next_input, self._idx)
+ except:
+ raise StopIteration
+ new_test_suite = self._baseSuite(self.tests)
+ new_test_suite.input = next_input
+ new_test_suite._idx = self._idx
+
+ self._idx += 1
+ return new_test_suite
+
class TestSuite(pyunit.TestSuite):
- inputUnit = [None]
+ def __init__(self, tests=()):
+ self._tests = []
+ self.input = None
+ self._idx = 0
+ self.addTests(tests)
+
def __repr__(self):
- return "<%s input=%s, tests=%s>" % (self.__class__, self.inputUnit, list(self))
+ return "<%s input=%s tests=%s>" % (self.__class__,
+ self.input, self._tests)
- def run(self, result, inputUnit=[None]):
+ def run(self, result):
"""
Call C{run} on every member of the suite.
"""
# we implement this because Python 2.3 unittest defines this code
# in __call__, whereas 2.4 defines the code in run.
- idx = 0
- for input, test in itertools.product(inputUnit, self._tests):
+ for i, test in enumerate(self._tests):
if result.shouldStop:
break
- self.inputUnit = inputUnit
- test.input = input
- test.idx = idx
+ test.input = self.input
+ test._idx = self._idx + i
test(result)
- idx += 1
return result
class TestCase(unittest.TestCase):
name = "DefaultTestName"
+ inputs = [None]
+
+ def __repr__(self):
+ return "<%s inputs=%s>" % (self.__class__, self.inputs)
+
+
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index fa7742f..be73d30 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -17,12 +17,13 @@ import sys, os, random, gc, time, warnings
from twisted.internet import defer
from twisted.application import app
-from twisted.python import usage, reflect, failure
+from twisted.python import usage, reflect, failure, log
from twisted.python.filepath import FilePath
from twisted import plugin
from twisted.python.util import spewer
from twisted.python.compat import set
from twisted.trial import itrial
+from twisted.trial import runner as irunner
from ooni import runner, reporter
@@ -104,10 +105,8 @@ class Options(usage.Options, app.ReactorSelectionMixin):
"callback stack traces"],
["nopm", None, "don't automatically jump into debugger for "
"postmorteming of exceptions"],
- ["dry-run", 'n', "do everything but run the tests"],
["force-gc", None, "Have OONI run gc.collect() before and "
"after each test case."],
- ["profile", None, "Run tests under the Python profiler"],
["unclean-warnings", None,
"Turn dirty reactor errors into warnings"],
["no-recurse", "N", "Don't recurse into packages"],
@@ -118,9 +117,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
optParameters = [
["reportfile", "o", "report.yaml", "report file name"],
["logfile", "l", "test.log", "log file name"],
- ["random", "z", None,
- "Run tests in random order using the specified seed"],
- ['temp-directory', None, '_trial_temp',
+ ['temp-directory', None, '_ooni_temp',
'Path to use as working directory for tests.'],
['reporter', None, 'default',
'The reporter to use for this test run. See --help-reporters for '
@@ -129,7 +126,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
compData = usage.Completions(
optActions={"tbformat": usage.CompleteList(["plain", "emacs", "cgitb"]),
"logfile": usage.CompleteFiles(descr="log file name"),
- "random": usage.Completer(descr="random seed")},
+ },
extraActions=[usage.CompleteFiles(
"*.py", descr="file | module | package | TestCase | testMethod",
repeat=True)],
@@ -238,20 +235,6 @@ class Options(usage.Options, app.ReactorSelectionMixin):
"argument to recursionlimit must be an integer")
- def opt_random(self, option):
- try:
- self['random'] = long(option)
- except ValueError:
- raise usage.UsageError(
- "Argument to --random must be a positive integer")
- else:
- if self['random'] < 0:
- raise usage.UsageError(
- "Argument to --random must be a positive integer")
- elif self['random'] == 0:
- self['random'] = long(time.time() * 100)
-
-
def opt_without_module(self, option):
"""
Fake the lack of the specified modules, separated with commas.
@@ -283,7 +266,6 @@ class Options(usage.Options, app.ReactorSelectionMixin):
failure.DO_POST_MORTEM = False
-
def _initialDebugSetup(config):
# do this part of debug setup first for easy debugging of import failures
if config['debug']:
@@ -292,35 +274,22 @@ def _initialDebugSetup(config):
defer.setDebugging(True)
-
-def _getSuites(config):
- loader = _getLoader(config)
+def _getSuitesAndInputs(config):
+ #loader = irunner.TestLoader()
+ loader = runner.NetTestLoader()
recurse = not config['no-recurse']
print "loadByNames %s" % config['tests']
- return loader.loadByNames(config['tests'], recurse)
-
-
-def _getLoader(config):
- loader = runner.NetTestLoader()
- if config['random']:
- randomer = random.Random()
- randomer.seed(config['random'])
- loader.sorter = lambda x : randomer.random()
- print 'Running tests shuffled with seed %d\n' % config['random']
- return loader
-
+ inputs, suites = loader.loadByNamesWithInput(config['tests'], recurse)
+ return inputs, suites
def _makeRunner(config):
mode = None
if config['debug']:
mode = runner.OONIRunner.DEBUG
- if config['dry-run']:
- mode = runner.OONIRunner.DRY_RUN
print "using %s" % config['reporter']
return runner.OONIRunner(config['reporter'],
reportfile=config["reportfile"],
mode=mode,
- profile=config['profile'],
logfile=config['logfile'],
tracebackFormat=config['tbformat'],
realTimeErrors=config['rterrors'],
@@ -340,7 +309,8 @@ def run():
_initialDebugSetup(config)
trialRunner = _makeRunner(config)
- suites = _getSuites(config)
- for suite in suites:
- test_result = trialRunner.run(suite)
+ inputs, testSuites = _getSuitesAndInputs(config)
+ log.startLogging(sys.stdout)
+ for i, suite in enumerate(testSuites):
+ test_result = trialRunner.run(suite, inputs[i])
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 07cffad..14297cd 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -89,7 +89,7 @@ class OONIReporter(OReporter):
def getTestIndex(self, test):
try:
- idx = test.idx
+ idx = test._idx
except:
idx = 0
return idx
@@ -109,7 +109,8 @@ class OONIReporter(OReporter):
self._tests[idx]['input'] = test.input
self._tests[idx]['idx'] = idx
self._tests[idx]['name'] = test.name
- self._tests[idx]['test'] = test
+ #self._tests[idx]['test'] = test
+ print "Now starting %s" % self._tests[idx]
def stopTest(self, test):
diff --git a/ooni/runner.py b/ooni/runner.py
index 12ab9ad..604942d 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -4,7 +4,7 @@ import types
import time
import inspect
-from twisted.internet import defer
+from twisted.internet import defer, reactor
from twisted.python import reflect, log, failure
from twisted.trial import unittest
from twisted.trial.runner import TrialRunner, TestLoader
@@ -27,7 +27,7 @@ def isLegacyTest(obj):
except TypeError:
return False
-def adaptLegacyTest(obj):
+def adaptLegacyTest(obj, inputs=[None]):
"""
We take a legacy OONITest class and convert it into a nettest.TestCase.
This allows backward compatibility of old OONI tests.
@@ -36,8 +36,18 @@ def adaptLegacyTest(obj):
older test cases compatible with the new OONI.
"""
class LegacyOONITest(nettest.TestCase):
- pass
+ inputs = [1]
+ original_test = obj
+ def test_start_legacy_test(self):
+ print "bla bla bla"
+ my_test = self.original_test()
+ print my_test
+ print "foobat"
+ my_test.startTest(self.input)
+ print "HHAHAHA"
+
+ return LegacyOONITest
class LoggedSuite(nettest.TestSuite):
@@ -93,11 +103,14 @@ class OONISuite(nettest.TestSuite):
self._bail()
-class NetTestLoader(object):
+class NetTestLoader(TestLoader):
"""
Reponsible for finding the modules that can work as tests and running them.
If we detect that a certain test is written using the legacy OONI API we
will wrap it around a next gen class to make it work here too.
+
+ XXX This class needs to be cleaned up a *lot* of all the things we actually
+ don't need.
"""
methodPrefix = 'test'
modulePrefix = 'test_'
@@ -106,67 +119,19 @@ class NetTestLoader(object):
self.suiteFactory = nettest.TestSuite
self._importErrors = []
-
def findTestClasses(self, module):
classes = []
for name, val in inspect.getmembers(module):
- try:
- inputs = val.inputs
- except:
- inputs = None
if isTestCase(val):
- classes.append((val, inputs))
+ classes.append(val)
# This is here to allow backward compatibility with legacy OONI
# tests.
elif isLegacyTest(val):
- #val = adaptLegacyTest(val)
- classes.append((val, inputs))
+ print "adapting! %s" % val
+ val = adaptLegacyTest(val)
+ classes.append(val)
return classes
- def findByName(self, name):
- """
- Return a Python object given a string describing it.
-
- @param name: a string which may be either a filename or a
- fully-qualified Python name.
-
- @return: If C{name} is a filename, return the module. If C{name} is a
- fully-qualified Python name, return the object it refers to.
- """
- if os.path.exists(name):
- return filenameToModule(name)
- return reflect.namedAny(name)
-
-
- def loadModule(self, module):
- """
- Return a test suite with all the tests from a module.
-
- Included are TestCase subclasses and doctests listed in the module's
- __doctests__ module. If that's not good for you, put a function named
- either C{testSuite} or C{test_suite} in your module that returns a
- TestSuite, and I'll use the results of that instead.
-
- If C{testSuite} and C{test_suite} are both present, then I'll use
- C{testSuite}.
- """
- ## XXX - should I add an optional parameter to disable the check for
- ## a custom suite.
- ## OR, should I add another method
- if not isinstance(module, types.ModuleType):
- raise TypeError("%r is not a module" % (module,))
- if hasattr(module, 'testSuite'):
- return module.testSuite()
- elif hasattr(module, 'test_suite'):
- return module.test_suite()
-
- suite = self.suiteFactory()
- for testClass, inputs in self.findTestClasses(module):
- testCases = self.loadClass(testClass)
-
- return testCases
- loadTestsFromModule = loadModule
-
def loadClass(self, klass):
"""
Given a class which contains test cases, return a sorted list of
@@ -182,186 +147,50 @@ class NetTestLoader(object):
tests.append(self._makeCase(klass, self.methodPrefix+name))
suite = self.suiteFactory(tests)
- suite.inputs = klass.inputs
- return suite
- loadTestsFromTestCase = loadClass
-
- def getTestCaseNames(self, klass):
- """
- Given a class that contains C{TestCase}s, return a list of names of
- methods that probably contain tests.
- """
- return reflect.prefixedMethodNames(klass, self.methodPrefix)
-
- def loadMethod(self, method):
- """
- Given a method of a C{TestCase} that represents a test, return a
- C{TestCase} instance for that test.
- """
- if not isinstance(method, types.MethodType):
- raise TypeError("%r not a method" % (method,))
- return self._makeCase(method.im_class, _getMethodNameInClass(method))
-
- def _makeCase(self, klass, methodName):
- return klass(methodName)
+ print "**+*"
+ print tests
+ print "**+*"
- def loadPackage(self, package, recurse=False):
- """
- Load tests from a module object representing a package, and return a
- TestSuite containing those tests.
-
- Tests are only loaded from modules whose name begins with 'test_'
- (or whatever C{modulePrefix} is set to).
-
- @param package: a types.ModuleType object (or reasonable facsimilie
- obtained by importing) which may contain tests.
-
- @param recurse: A boolean. If True, inspect modules within packages
- within the given package (and so on), otherwise, only inspect modules
- in the package itself.
-
- @raise: TypeError if 'package' is not a package.
-
- @return: a TestSuite created with my suiteFactory, containing all the
- tests.
- """
- if not isPackage(package):
- raise TypeError("%r is not a package" % (package,))
- pkgobj = modules.getModule(package.__name__)
- if recurse:
- discovery = pkgobj.walkModules()
- else:
- discovery = pkgobj.iterModules()
- discovered = []
- for disco in discovery:
- if disco.name.split(".")[-1].startswith(self.modulePrefix):
- discovered.append(disco)
- suite = self.suiteFactory()
- for modinfo in self.sort(discovered):
- try:
- module = modinfo.load()
- except:
- thingToAdd = ErrorHolder(modinfo.name, failure.Failure())
- else:
- thingToAdd = self.loadModule(module)
- suite.addTest(thingToAdd)
return suite
+ loadTestsFromTestCase = loadClass
- def loadDoctests(self, module):
- """
- Return a suite of tests for all the doctests defined in C{module}.
-
- @param module: A module object or a module name.
- """
- if isinstance(module, str):
+ def findAllInputs(self, thing):
+ testClasses = self.findTestClasses(thing)
+ # XXX will there ever be more than 1 test class with inputs?
+ for klass in testClasses:
try:
- module = reflect.namedAny(module)
+ inputs = klass.inputs
except:
- return ErrorHolder(module, failure.Failure())
- if not inspect.ismodule(module):
- warnings.warn("trial only supports doctesting modules")
- return
- extraArgs = {}
- if sys.version_info > (2, 4):
- # Work around Python issue2604: DocTestCase.tearDown clobbers globs
- def saveGlobals(test):
- """
- Save C{test.globs} and replace it with a copy so that if
- necessary, the original will be available for the next test
- run.
- """
- test._savedGlobals = getattr(test, '_savedGlobals', test.globs)
- test.globs = test._savedGlobals.copy()
- extraArgs['setUp'] = saveGlobals
- return doctest.DocTestSuite(module, **extraArgs)
-
- def loadAnything(self, thing, recurse=False):
- """
- Given a Python object, return whatever tests that are in it. Whatever
- 'in' might mean.
-
- @param thing: A Python object. A module, method, class or package.
- @param recurse: Whether or not to look in subpackages of packages.
- Defaults to False.
-
- @return: A C{TestCase} or C{TestSuite}.
- """
- print "Loading anything! %s" % thing
- ret = None
- if isinstance(thing, types.ModuleType):
- if isPackage(thing):
- ret = self.loadPackage(thing, recurse)
- ret = self.loadModule(thing)
- elif isinstance(thing, types.ClassType):
- ret = self.loadClass(thing)
- elif isinstance(thing, type):
- ret = self.loadClass(thing)
- elif isinstance(thing, types.MethodType):
- ret = self.loadMethod(thing)
- if not ret:
- raise TypeError("No loader for %r. Unrecognized type" % (thing,))
- try:
- ret.inputs = ret.inputs
- except:
- ret.inputs = [None]
- return ret
-
- def loadByName(self, name, recurse=False):
- """
- Given a string representing a Python object, return whatever tests
- are in that object.
-
- If C{name} is somehow inaccessible (e.g. the module can't be imported,
- there is no Python object with that name etc) then return an
- L{ErrorHolder}.
-
- @param name: The fully-qualified name of a Python object.
- """
- print "Load by Name!"
- try:
- thing = self.findByName(name)
- except:
- return ErrorHolder(name, failure.Failure())
- return self.loadAnything(thing, recurse)
- loadTestsFromName = loadByName
+ pass
+ return inputs
- def loadByNames(self, names, recurse=False):
+ def loadByNamesWithInput(self, names, recurse=False):
"""
- Construct a TestSuite containing all the tests found in 'names', where
+ Construct a OONITestSuite containing all the tests found in 'names', where
names is a list of fully qualified python names and/or filenames. The
suite returned will have no duplicate tests, even if the same object
is named twice.
+
+ This test suite will have set the attribute inputs to the inputs found
+ inside of the tests.
"""
- print "Load by Names!"
+ inputs = []
things = []
errors = []
for name in names:
try:
- things.append(self.findByName(name))
+ thing = self.findByName(name)
+ things.append(thing)
except:
errors.append(ErrorHolder(name, failure.Failure()))
- suites = [self.loadAnything(thing, recurse)
- for thing in self._uniqueTests(things)]
- suites.extend(errors)
- return suites
- #return self.suiteFactory(suites)
-
-
- def _uniqueTests(self, things):
- """
- Gather unique suite objects from loaded things. This will guarantee
- uniqueness of inherited methods on TestCases which would otherwise hash
- to same value and collapse to one test unexpectedly if using simpler
- means: e.g. set().
- """
- entries = []
- for thing in things:
- if isinstance(thing, types.MethodType):
- entries.append((thing, thing.im_class))
- else:
- entries.append((thing,))
- return [entry[0] for entry in set(entries)]
+ suites = []
+ for thing in self._uniqueTests(things):
+ inputs.append(self.findAllInputs(thing))
+ suite = self.loadAnything(thing, recurse)
+ suites.append(suite)
+ suites.extend(errors)
+ return inputs, suites
class OONIRunner(object):
"""
@@ -457,47 +286,17 @@ class OONIRunner(object):
self._logFileObserver = log.FileLogObserver(logFile)
log.startLoggingWithObserver(self._logFileObserver.emit, 0)
- def run(self, test):
+ def run(self, tests, inputs=[None]):
"""
Run the test or suite and return a result object.
"""
- print test
- inputs = test.inputs
reporterFactory = ReporterFactory(open(self._reportfile, 'a+'),
- testSuite=test)
+ testSuite=tests)
reporterFactory.writeHeader()
- #testUnitReport = OONIReporter(open('reporting.log', 'a+'))
- #testUnitReport.writeHeader(FooTest)
for inputUnit in InputUnitFactory(inputs):
+ testSuiteFactory = nettest.TestSuiteFactory(inputUnit, tests, nettest.TestSuite)
testUnitReport = reporterFactory.create()
- test(testUnitReport, inputUnit)
+ for suite in testSuiteFactory:
+ suite(testUnitReport)
testUnitReport.done()
- def _runWithInput(self, test, input):
- """
- Private helper that runs the given test with the given input.
- """
- result = self._makeResult()
- # decorate the suite with reactor cleanup and log starting
- # This should move out of the runner and be presumed to be
- # present
- suite = TrialSuite([test])
- startTime = time.time()
-
- ## XXX replace this with the actual way of running the test.
- run = lambda: suite.run(result)
-
- oldDir = self._setUpTestdir()
- try:
- self._setUpLogFile()
- run()
- finally:
- self._tearDownLogFile()
- self._tearDownTestdir(oldDir)
-
- endTime = time.time()
- done = getattr(result, 'done', None)
- result.done()
- return result
-
-
More information about the tor-commits
mailing list