commit 712f665423f24701c93889d32d040eb533065dfe Author: Arturo Filastò arturo@filasto.net Date: Sat Sep 15 14:54:51 2012 +0200
Continue work on trial based nettest Network Unit Testing framework * Implement InputUnit and InputUnitFactory --- docs/design.dia | Bin 1706 -> 1706 bytes ooni/input.py | 62 +++ ooni/nettest.py | 45 ++- ooni/runner.py | 458 ++++++++++++++++---- 4 files changed, 472 insertions(+), 93 deletions(-)
diff --git a/bin/oonib b/bin/oonib old mode 100755 new mode 100644 diff --git a/bin/ooniprobe b/bin/ooniprobe old mode 100755 new mode 100644 diff --git a/docs/design.dia b/docs/design.dia old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/bin/ooni-probe b/old-to-be-ported-code/bin/ooni-probe old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/captive_portal.py b/old-to-be-ported-code/ooni/captive_portal.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/dns_cc_check.py b/old-to-be-ported-code/ooni/dns_cc_check.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/dns_poisoning.py b/old-to-be-ported-code/ooni/dns_poisoning.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/namecheck.py b/old-to-be-ported-code/ooni/namecheck.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py b/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/netalyzr_plgoo.py b/old-to-be-ported-code/ooni/plugins/netalyzr_plgoo.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/connectback.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/connectback.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/dirconntest.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/dirconntest.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/generic-host-test.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/generic-host-test.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/host-prep.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/host-prep.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/install-probe.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/install-probe.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/run-tests.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/run-tests.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/twitter-test.sh b/old-to-be-ported-code/ooni/plugins/old_stuff_to_convert/twitter-test.sh old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/simple_dns_plgoo.py b/old-to-be-ported-code/ooni/plugins/simple_dns_plgoo.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugins/skel_plgoo.py b/old-to-be-ported-code/ooni/plugins/skel_plgoo.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/plugooni.py b/old-to-be-ported-code/ooni/plugooni.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/transparenthttp.py b/old-to-be-ported-code/ooni/transparenthttp.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/ooni/yamlooni.py b/old-to-be-ported-code/ooni/yamlooni.py old mode 100755 new mode 100644 diff --git a/old-to-be-ported-code/proxy-lists/parse-trusted-xff.sh b/old-to-be-ported-code/proxy-lists/parse-trusted-xff.sh old mode 100755 new mode 100644 diff --git a/ooni/input.py b/ooni/input.py new file mode 100644 index 0000000..f534393 --- /dev/null +++ b/ooni/input.py @@ -0,0 +1,62 @@ +class InputUnitFactory(object): + """ + This is a factory that takes the size of input units to be generated a set + of units that is a python iterable item and outputs InputUnit objects + containing inputUnitSize elements. + + This object is a python iterable, this means that it does not need to keep + all the elements in memory to be able to produce InputUnits. + """ + inputUnitSize = 3 + def __init__(self, inputs=[]): + self._inputs = inputs + self._idx = 0 + self._ended = False + + def __iter__(self): + return self + + def next(self): + if self._ended: + raise StopIteration + + last_element_idx = self._idx + self.inputUnitSize + input_unit_elements = self._inputs[self._idx:last_element_idx] + try: + # XXX hack to fail when we reach the end of the list + antani = self._inputs[last_element_idx] + except: + if len(input_unit_elements) > 0: + self._ended = True + return InputUnit(input_unit_elements) + else: + raise StopIteration + + self._idx += self.inputUnitSize + + return InputUnit(input_unit_elements) + + +class InputUnit(object): + """ + This is a python iterable object that contains the input elements to be + passed onto a TestCase. + """ + def __init__(self, inputs=[]): + self._inputs = inputs + + def __repr__(self): + return "<%s inputs=%s>" % (self.__class__, self._inputs) + + def __add__(self, inputs): + for input in inputs: + self._inputs.append(input) + + def __iter__(self): + return iter(self._inputs) + + def append(self, input): + self._inputs.append(input) + + + diff --git a/ooni/nettest.py b/ooni/nettest.py index fe8c05c..ab009b1 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -11,13 +11,7 @@ def _iterateTests(testSuiteOrCase): try: suite = iter(testSuiteOrCase) except TypeError: - if not testSuiteOrCase.inputs: - yield testSuiteOrCase - else: - inputs = iter(testSuiteOrCase.inputs) - print "Detected Sub shit! %s" % inputs - for input in inputs: - yield testSuiteOrCase, input + yield testSuiteOrCase else: for test in suite: for subtest in _iterateTests(test): @@ -28,7 +22,7 @@ class TestCase(unittest.TestCase): """ A test case represents the minimum """ - def run(self, result, input): + def run(self, result): """ Run the test case, storing the results in C{result}.
@@ -77,21 +71,38 @@ class TestSuite(pyunit.TestSuite): pattern and a consistently overrideable C{run} method. """
- def __call__(self, result, input): - return self.run(result, input) + def __init__(self, tests=(), inputs=()): + self._tests = [] + self._inputs = [] + self.addTests(tests, inputs) + print "Adding %s %s" % (tests, inputs) + + + def __call__(self, result): + return self.run(result)
+ def __repr__(self): + return "<%s input=%s tests=%s>" % (self.__class__, + self._inputs, list(self))
- def run(self, result, input): + def run(self, result, input=None): """ Call C{run} on every member of the suite. """ - # we implement this because Python 2.3 unittest defines this code - # in __call__, whereas 2.4 defines the code in run. for test in self._tests: if result.shouldStop: break - print test - print "----------------" - test(result, input) - return result + return test(result, None) + + def addTests(self, tests, inputs=[]): + if isinstance(tests, basestring): + raise TypeError("tests must be and iterable of tests not a string") + for test in tests: + self.addTest(test, inputs) + + def addTest(self, test, inputs=[]): + #print "Adding: %s" % test + super(TestSuite, self).addTest(test) + self._inputs = inputs +
diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py old mode 100755 new mode 100644 diff --git a/ooni/runner.py b/ooni/runner.py index c6ad90b..a8485af 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -3,6 +3,7 @@ import time import inspect
from twisted.internet import defer +from twisted.python import reflect from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader from twisted.trial.runner import isPackage, isTestCase @@ -39,7 +40,7 @@ class LoggedSuite(nettest.TestSuite): object. """
- def run(self, result, input): + def run(self, result): """ Run the suite, storing all errors in C{result}. If an error is logged while no tests are running, then it will be added as an error to @@ -49,7 +50,7 @@ class LoggedSuite(nettest.TestSuite): """ observer = unittest._logObserver observer._add() - super(LoggedSuite, self).run(result, input) + super(LoggedSuite, self).run(result) observer._remove() for error in observer.getErrors(): result.addError(TestHolder(NOT_IN_TEST), error) @@ -79,100 +80,405 @@ class OONISuite(nettest.TestSuite): # so that the shutdown event completes nettest.TestCase('mktemp')._wait(d)
- def run(self, result, input): + def run(self, result): try: - nettest.TestSuite.run(self, result, input) + nettest.TestSuite.run(self, result) finally: self._bail()
-class OONIRunner(TrialRunner): - def run(self, test): - return TrialRunner.run(self, test) - - def _runWithoutDecoration(self, test): - """ - Private helper that runs the given test but doesn't decorate it. - """ - result = self._makeResult() - # decorate the suite with reactor cleanup and log starting - # This should move out of the runner and be presumed to be - # present - suite = OONISuite([test]) - print "HERE IS THE TEST:" - print test - print "-------------" - try: - inputs = test.inputs - except: - inputs = [None] - - startTime = time.time() - if self.mode == self.DRY_RUN: - for single in nettest._iterateTests(suite): - input = None - if type(single) == type(tuple()): - single, input = single - result.startTest(single, input) - result.addSuccess(single) - result.stopTest(single) - else: - if self.mode == self.DEBUG: - # open question - should this be self.debug() instead. - debugger = self._getDebugger() - run = lambda x: debugger.runcall(suite.run, result, x) - else: - run = lambda x: suite.run(result, x) - - oldDir = self._setUpTestdir() - try: - self._setUpLogFile() - # XXX work on this better - for input in inputs: - run(input) - finally: - self._tearDownLogFile() - self._tearDownTestdir(oldDir) - - endTime = time.time() - done = getattr(result, 'done', None) - if done is None: - warnings.warn( - "%s should implement done() but doesn't. Falling back to " - "printErrors() and friends." % reflect.qual(result.__class__), - category=DeprecationWarning, stacklevel=3) - result.printErrors() - result.writeln(result.separator) - result.writeln('Ran %d tests in %.3fs', result.testsRun, - endTime - startTime) - result.write('\n') - result.printSummary() - else: - result.done() - return result - - -class TestLoader(TestLoader): +class NetTestLoader(object): """ Reponsible for finding the modules that can work as tests and running them. If we detect that a certain test is written using the legacy OONI API we will wrap it around a next gen class to make it work here too. """ + methodPrefix = 'test' + modulePrefix = 'test_' + def __init__(self): - super(TestLoader, self).__init__() self.suiteFactory = nettest.TestSuite + self.sorter = name + self._importErrors = [] + + def sort(self, xs): + """ + Sort the given things using L{sorter}. + + @param xs: A list of test cases, class or modules. + """ + return sorted(xs, key=self.sorter) +
def findTestClasses(self, module): classes = [] for name, val in inspect.getmembers(module): + try: + inputs = val.inputs + except: + inputs = None if isTestCase(val): - classes.append(val) + classes.append((val, inputs)) # This is here to allow backward compatibility with legacy OONI # tests. elif isLegacyTest(val): #val = adaptLegacyTest(val) - classes.append(val) - return self.sort(classes) - #return runner.TestLoader.findTestClasses(self, module) + classes.append((val, inputs)) + return classes + + def findByName(self, name): + """ + Return a Python object given a string describing it.
+ @param name: a string which may be either a filename or a + fully-qualified Python name.
+ @return: If C{name} is a filename, return the module. If C{name} is a + fully-qualified Python name, return the object it refers to. + """ + if os.path.exists(name): + return filenameToModule(name) + return reflect.namedAny(name) + + + def loadModule(self, module): + """ + Return a test suite with all the tests from a module. + + Included are TestCase subclasses and doctests listed in the module's + __doctests__ module. If that's not good for you, put a function named + either C{testSuite} or C{test_suite} in your module that returns a + TestSuite, and I'll use the results of that instead. + + If C{testSuite} and C{test_suite} are both present, then I'll use + C{testSuite}. + """ + ## XXX - should I add an optional parameter to disable the check for + ## a custom suite. + ## OR, should I add another method + if not isinstance(module, types.ModuleType): + raise TypeError("%r is not a module" % (module,)) + if hasattr(module, 'testSuite'): + return module.testSuite() + elif hasattr(module, 'test_suite'): + return module.test_suite() + + suite = self.suiteFactory() + for testClass, inputs in self.findTestClasses(module): + testCases = self.loadClass(testClass) + + return testCases + loadTestsFromModule = loadModule + + def loadClass(self, klass): + """ + Given a class which contains test cases, return a sorted list of + C{TestCase} instances. + """ + if not (isinstance(klass, type) or isinstance(klass, types.ClassType)): + raise TypeError("%r is not a class" % (klass,)) + if not isTestCase(klass): + raise ValueError("%r is not a test case" % (klass,)) + names = self.getTestCaseNames(klass) + print "Names %s" % names + tests = self.sort([self._makeCase(klass, self.methodPrefix+name) + for name in names]) + print "Tests %s" % tests + suite = self.suiteFactory(tests) + print "Suite: %s" % suite + return suite + loadTestsFromTestCase = loadClass + + def getTestCaseNames(self, klass): + """ + Given a class that contains C{TestCase}s, return a list of names of + methods that probably contain tests. + """ + return reflect.prefixedMethodNames(klass, self.methodPrefix) + + def loadMethod(self, method): + """ + Given a method of a C{TestCase} that represents a test, return a + C{TestCase} instance for that test. + """ + if not isinstance(method, types.MethodType): + raise TypeError("%r not a method" % (method,)) + return self._makeCase(method.im_class, _getMethodNameInClass(method)) + + def _makeCase(self, klass, methodName): + return klass(methodName) + + def loadPackage(self, package, recurse=False): + """ + Load tests from a module object representing a package, and return a + TestSuite containing those tests. + + Tests are only loaded from modules whose name begins with 'test_' + (or whatever C{modulePrefix} is set to). + + @param package: a types.ModuleType object (or reasonable facsimilie + obtained by importing) which may contain tests. + + @param recurse: A boolean. If True, inspect modules within packages + within the given package (and so on), otherwise, only inspect modules + in the package itself. + + @raise: TypeError if 'package' is not a package. + + @return: a TestSuite created with my suiteFactory, containing all the + tests. + """ + if not isPackage(package): + raise TypeError("%r is not a package" % (package,)) + pkgobj = modules.getModule(package.__name__) + if recurse: + discovery = pkgobj.walkModules() + else: + discovery = pkgobj.iterModules() + discovered = [] + for disco in discovery: + if disco.name.split(".")[-1].startswith(self.modulePrefix): + discovered.append(disco) + suite = self.suiteFactory() + for modinfo in self.sort(discovered): + try: + module = modinfo.load() + except: + thingToAdd = ErrorHolder(modinfo.name, failure.Failure()) + else: + thingToAdd = self.loadModule(module) + suite.addTest(thingToAdd) + return suite + + def loadDoctests(self, module): + """ + Return a suite of tests for all the doctests defined in C{module}. + + @param module: A module object or a module name. + """ + if isinstance(module, str): + try: + module = reflect.namedAny(module) + except: + return ErrorHolder(module, failure.Failure()) + if not inspect.ismodule(module): + warnings.warn("trial only supports doctesting modules") + return + extraArgs = {} + if sys.version_info > (2, 4): + # Work around Python issue2604: DocTestCase.tearDown clobbers globs + def saveGlobals(test): + """ + Save C{test.globs} and replace it with a copy so that if + necessary, the original will be available for the next test + run. + """ + test._savedGlobals = getattr(test, '_savedGlobals', test.globs) + test.globs = test._savedGlobals.copy() + extraArgs['setUp'] = saveGlobals + return doctest.DocTestSuite(module, **extraArgs) + + def loadAnything(self, thing, recurse=False): + """ + Given a Python object, return whatever tests that are in it. Whatever + 'in' might mean. + + @param thing: A Python object. A module, method, class or package. + @param recurse: Whether or not to look in subpackages of packages. + Defaults to False. + + @return: A C{TestCase} or C{TestSuite}. + """ + if isinstance(thing, types.ModuleType): + if isPackage(thing): + return self.loadPackage(thing, recurse) + return self.loadModule(thing) + elif isinstance(thing, types.ClassType): + return self.loadClass(thing) + elif isinstance(thing, type): + return self.loadClass(thing) + elif isinstance(thing, types.MethodType): + return self.loadMethod(thing) + raise TypeError("No loader for %r. Unrecognized type" % (thing,)) + + def loadByName(self, name, recurse=False): + """ + Given a string representing a Python object, return whatever tests + are in that object. + + If C{name} is somehow inaccessible (e.g. the module can't be imported, + there is no Python object with that name etc) then return an + L{ErrorHolder}. + + @param name: The fully-qualified name of a Python object. + """ + try: + thing = self.findByName(name) + except: + return ErrorHolder(name, failure.Failure()) + return self.loadAnything(thing, recurse) + loadTestsFromName = loadByName + + def loadByNames(self, names, recurse=False): + """ + Construct a TestSuite containing all the tests found in 'names', where + names is a list of fully qualified python names and/or filenames. The + suite returned will have no duplicate tests, even if the same object + is named twice. + """ + things = [] + errors = [] + for name in names: + try: + things.append(self.findByName(name)) + except: + errors.append(ErrorHolder(name, failure.Failure())) + suites = [self.loadAnything(thing, recurse) + for thing in self._uniqueTests(things)] + suites.extend(errors) + return self.suiteFactory(suites) + + + def _uniqueTests(self, things): + """ + Gather unique suite objects from loaded things. This will guarantee + uniqueness of inherited methods on TestCases which would otherwise hash + to same value and collapse to one test unexpectedly if using simpler + means: e.g. set(). + """ + entries = [] + for thing in things: + if isinstance(thing, types.MethodType): + entries.append((thing, thing.im_class)) + else: + entries.append((thing,)) + return [entry[0] for entry in set(entries)] + + + + +class OONIRunner(object): + """ + A specialised runner that is used by the ooniprobe frontend to run tests. + Heavily inspired by the trial TrialRunner class. + """ + + DEBUG = 'debug' + DRY_RUN = 'dry-run' + + def _getDebugger(self): + dbg = pdb.Pdb() + try: + import readline + except ImportError: + print "readline module not available" + sys.exc_clear() + for path in ('.pdbrc', 'pdbrc'): + if os.path.exists(path): + try: + rcFile = file(path, 'r') + except IOError: + sys.exc_clear() + else: + dbg.rcLines.extend(rcFile.readlines()) + return dbg + + + def _setUpTestdir(self): + self._tearDownLogFile() + currentDir = os.getcwd() + base = filepath.FilePath(self.workingDirectory) + testdir, self._testDirLock = util._unusedTestDirectory(base) + os.chdir(testdir.path) + return currentDir + + + def _tearDownTestdir(self, oldDir): + os.chdir(oldDir) + self._testDirLock.unlock() + + + _log = log + def _makeResult(self): + reporter = self.reporterFactory(self.stream, self.tbformat, + self.rterrors, self._log) + if self.uncleanWarnings: + reporter = UncleanWarningsReporterWrapper(reporter) + return reporter + + def __init__(self, reporterFactory, + mode=None, + logfile='test.log', + stream=sys.stdout, + profile=False, + tracebackFormat='default', + realTimeErrors=False, + uncleanWarnings=False, + workingDirectory=None, + forceGarbageCollection=False): + self.reporterFactory = reporterFactory + self.logfile = logfile + self.mode = mode + self.stream = stream + self.tbformat = tracebackFormat + self.rterrors = realTimeErrors + self.uncleanWarnings = uncleanWarnings + self._result = None + self.workingDirectory = workingDirectory or '_trial_temp' + self._logFileObserver = None + self._logFileObject = None + self._forceGarbageCollection = forceGarbageCollection + if profile: + self.run = util.profiled(self.run, 'profile.data') + + def _tearDownLogFile(self): + if self._logFileObserver is not None: + log.removeObserver(self._logFileObserver.emit) + self._logFileObserver = None + if self._logFileObject is not None: + self._logFileObject.close() + self._logFileObject = None + + def _setUpLogFile(self): + self._tearDownLogFile() + if self.logfile == '-': + logFile = sys.stdout + else: + logFile = file(self.logfile, 'a') + self._logFileObject = logFile + self._logFileObserver = log.FileLogObserver(logFile) + log.startLoggingWithObserver(self._logFileObserver.emit, 0) + + def run(self, test, inputs): + """ + Run the test or suite and return a result object. + """ + for input in inputs: + self._runWithInput(test, input) + + def _runWithInput(self, test, input): + """ + Private helper that runs the given test with the given input. + """ + result = self._makeResult() + # decorate the suite with reactor cleanup and log starting + # This should move out of the runner and be presumed to be + # present + suite = TrialSuite([test]) + startTime = time.time() + + ## XXX replace this with the actual way of running the test. + run = lambda: suite.run(result) + + oldDir = self._setUpTestdir() + try: + self._setUpLogFile() + run() + finally: + self._tearDownLogFile() + self._tearDownTestdir(oldDir) + + endTime = time.time() + done = getattr(result, 'done', None) + result.done() + return result diff --git a/ooni/scaffolding.py b/ooni/scaffolding.py old mode 100755 new mode 100644 diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py old mode 100755 new mode 100644