[tor-commits] [ooni-probe/master] Implement working trial inspired OONIprobe refactoring

isis at torproject.org isis at torproject.org
Thu Oct 4 14:41:15 UTC 2012


commit cbfcbdd0344d8e9a80d565ba01b64c7a40c97352
Author: Arturo Filastò <arturo at filasto.net>
Date:   Thu Sep 20 19:26:46 2012 +0000

    Implement working trial inspired OONIprobe refactoring
---
 ooni/nettest.py      |   41 +++----
 ooni/oonicli.py      |   66 ++++-------
 ooni/plugoo/tests.py |    1 +
 ooni/runner.py       |  330 ++++++++++++-------------------------------------
 4 files changed, 122 insertions(+), 316 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py
index 4ab1e0f..0302d25 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -1,5 +1,4 @@
 import itertools
-
 from twisted.python import log
 from twisted.trial import unittest, itrial
 
@@ -42,35 +41,29 @@ class TestSuiteFactory(object):
         self._idx += 1
         return new_test_suite
 
-class TestSuite(pyunit.TestSuite):
-    def __init__(self, tests=()):
-        self._tests = []
-        self.input = None
-        self._idx = 0
-        self.addTests(tests)
-
-    def __repr__(self):
-        return "<%s input=%s tests=%s>" % (self.__class__,
-                self.input, self._tests)
-
-    def run(self, result):
-        """
-        Call C{run} on every member of the suite.
-        """
-        # we implement this because Python 2.3 unittest defines this code
-        # in __call__, whereas 2.4 defines the code in run.
-        for i, test in enumerate(self._tests):
+class InputTestSuite(pyunit.TestSuite):
+    def run(self, result, idx=0):
+        self._idx = idx
+        while self._tests:
             if result.shouldStop:
                 break
-            test.input = self.input
-            test._idx = self._idx + i
-            test(result)
-
+            test = self._tests.pop(0)
+            try:
+                test.input = self.input
+                test._idx = self._idx
+                print "IDX: %s" % self._idx
+                test(result)
+            except:
+                test(result)
+            self._idx += 1
         return result
 
 class TestCase(unittest.TestCase):
     name = "DefaultTestName"
-    inputs = [None]
+    inputs = []
+
+    def getOptions(self):
+        return {'inputs': self.inputs}
 
     def __repr__(self):
         return "<%s inputs=%s>" % (self.__class__, self.inputs)
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index be73d30..f0287b4 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -15,6 +15,15 @@
 
 import sys, os, random, gc, time, warnings
 
+import unittest
+import inspect
+
+from ooni.input import InputUnitFactory
+from ooni.reporter import ReporterFactory
+from ooni.nettest import InputTestSuite
+from ooni.plugoo import tests
+from ooni import nettest, runner, reporter
+
 from twisted.internet import defer
 from twisted.application import app
 from twisted.python import usage, reflect, failure, log
@@ -24,7 +33,6 @@ from twisted.python.util import spewer
 from twisted.python.compat import set
 from twisted.trial import itrial
 from twisted.trial import runner as irunner
-from ooni import runner, reporter
 
 
 def _parseLocalVariables(line):
@@ -132,11 +140,10 @@ class Options(usage.Options, app.ReactorSelectionMixin):
                 repeat=True)],
         )
 
-    fallbackReporter = reporter.OONIReporter
     tracer = None
 
     def __init__(self):
-        self['tests'] = set()
+        self['test'] = None
         usage.Options.__init__(self)
 
 
@@ -178,7 +185,10 @@ class Options(usage.Options, app.ReactorSelectionMixin):
         if not os.path.isfile(filename):
             sys.stderr.write("File %r doesn't exist\n" % (filename,))
             return
+
         filename = os.path.abspath(filename)
+        self['test'] = filename
+
         if isTestFile(filename):
             self['tests'].add(filename)
         else:
@@ -248,7 +258,10 @@ class Options(usage.Options, app.ReactorSelectionMixin):
 
 
     def parseArgs(self, *args):
-        self['tests'].update(args)
+        try:
+            self['test'] = args[0]
+        except:
+            raise usage.UsageError("No test filename specified!")
 
 
     def postOptions(self):
@@ -265,39 +278,6 @@ class Options(usage.Options, app.ReactorSelectionMixin):
                                        "--nopm ")
             failure.DO_POST_MORTEM = False
 
-
-def _initialDebugSetup(config):
-    # do this part of debug setup first for easy debugging of import failures
-    if config['debug']:
-        failure.startDebugMode()
-    if config['debug'] or config['debug-stacktraces']:
-        defer.setDebugging(True)
-
-
-def _getSuitesAndInputs(config):
-    #loader = irunner.TestLoader()
-    loader = runner.NetTestLoader()
-    recurse = not config['no-recurse']
-    print "loadByNames %s" % config['tests']
-    inputs, suites = loader.loadByNamesWithInput(config['tests'], recurse)
-    return inputs, suites
-
-def _makeRunner(config):
-    mode = None
-    if config['debug']:
-        mode = runner.OONIRunner.DEBUG
-    print "using %s" % config['reporter']
-    return runner.OONIRunner(config['reporter'],
-                              reportfile=config["reportfile"],
-                              mode=mode,
-                              logfile=config['logfile'],
-                              tracebackFormat=config['tbformat'],
-                              realTimeErrors=config['rterrors'],
-                              uncleanWarnings=config['unclean-warnings'],
-                              workingDirectory=config['temp-directory'],
-                              forceGarbageCollection=config['force-gc'])
-
-
 def run():
     if len(sys.argv) == 1:
         sys.argv.append("--help")
@@ -307,10 +287,10 @@ def run():
     except usage.error, ue:
         raise SystemExit, "%s: %s" % (sys.argv[0], ue)
 
-    _initialDebugSetup(config)
-    trialRunner = _makeRunner(config)
-    inputs, testSuites = _getSuitesAndInputs(config)
-    log.startLogging(sys.stdout)
-    for i, suite in enumerate(testSuites):
-        test_result = trialRunner.run(suite, inputs[i])
+    file_name = os.path.abspath('nettests/simpletest.py')
+    classes = runner.findTestClassesFromFile(config['test'])
+    casesList, options = runner.loadTestsAndOptions(classes)
+    for idx, cases in enumerate(casesList):
+        orunner = runner.ORunner(cases, options[idx])
+        orunner.run()
 
diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py
index 2b1e87c..e4ee187 100644
--- a/ooni/plugoo/tests.py
+++ b/ooni/plugoo/tests.py
@@ -133,6 +133,7 @@ class OONITest(object):
         @param args: the asset(s) lines that we are working on.
         """
         self.start_time = date.now()
+        print "FOWID"
         log.msg("Starting test %s" % self.__class__)
         return self._do_experiment(args)
 
diff --git a/ooni/runner.py b/ooni/runner.py
index 604942d..ddde83b 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -13,9 +13,16 @@ from twisted.trial.runner import filenameToModule, _importFromFile
 
 from ooni.reporter import ReporterFactory
 from ooni.input import InputUnitFactory
+from ooni.nettest import InputTestSuite
 from ooni import nettest
 from ooni.plugoo import tests as oonitests
 
+def isTestCase(thing):
+    try:
+        return issubclass(thing, unittest.TestCase)
+    except TypeError:
+        return False
+
 def isLegacyTest(obj):
     """
     Returns True if the test in question is written using the OONITest legacy
@@ -23,7 +30,10 @@ def isLegacyTest(obj):
     We do this for backward compatibility of the OONIProbe API.
     """
     try:
-        return issubclass(obj, oonitests.OONITest)
+        if issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest:
+                return True
+        else:
+            return False
     except TypeError:
         return False
 
@@ -36,267 +46,89 @@ def adaptLegacyTest(obj, inputs=[None]):
     older test cases compatible with the new OONI.
     """
     class LegacyOONITest(nettest.TestCase):
-        inputs = [1]
+        inputs = [None]
         original_test = obj
 
+        @defer.inlineCallbacks
         def test_start_legacy_test(self):
             print "bla bla bla"
-            my_test = self.original_test()
-            print my_test
-            print "foobat"
-            my_test.startTest(self.input)
-            print "HHAHAHA"
+            print self.original_test
+            my_test = self.original_test(None, None, None)
+            yield my_test.startTest(self.input)
 
     return LegacyOONITest
 
 
-class LoggedSuite(nettest.TestSuite):
-    """
-    Any errors logged in this suite will be reported to the L{TestResult}
-    object.
-    """
-
-    def run(self, result):
-        """
-        Run the suite, storing all errors in C{result}. If an error is logged
-        while no tests are running, then it will be added as an error to
-        C{result}.
-
-        @param result: A L{TestResult} object.
-        """
-        observer = unittest._logObserver
-        observer._add()
-        super(LoggedSuite, self).run(result)
-        observer._remove()
-        for error in observer.getErrors():
-            result.addError(TestHolder(NOT_IN_TEST), error)
-        observer.flushErrors()
+def findTestClassesFromFile(filename):
+    classes = []
 
+    print "FILENAME %s" % filename
+    module = filenameToModule(filename)
+    for name, val in inspect.getmembers(module):
+        if isTestCase(val):
+            classes.append(val)
+        elif isLegacyTest(val):
+            classes.append(adaptLegacyTest(val))
+    return classes
 
-class OONISuite(nettest.TestSuite):
-    """
-    Suite to wrap around every single test in a C{trial} run. Used internally
-    by OONI to set up things necessary for OONI tests to work, regardless of
-    what context they are run in.
-    """
+def makeTestCases(klass, tests, methodPrefix):
+    cases = []
+    for test in tests:
+        cases.append(klass(methodPrefix+test))
+    return cases
 
-    def __init__(self, tests=()):
-        suite = LoggedSuite(tests)
-        super(OONISuite, self).__init__([suite])
-
-    def _bail(self):
-        from twisted.internet import reactor
-        d = defer.Deferred()
-        reactor.addSystemEventTrigger('after', 'shutdown',
-                                      lambda: d.callback(None))
-        reactor.fireSystemEvent('shutdown') # radix's suggestion
-        # As long as TestCase does crap stuff with the reactor we need to
-        # manually shutdown the reactor here, and that requires util.wait
-        # :(
-        # so that the shutdown event completes
-        nettest.TestCase('mktemp')._wait(d)
-
-    def run(self, result):
-        try:
-            nettest.TestSuite.run(self, result)
-        finally:
-            self._bail()
-
-
-class NetTestLoader(TestLoader):
-    """
-    Reponsible for finding the modules that can work as tests and running them.
-    If we detect that a certain test is written using the legacy OONI API we
-    will wrap it around a next gen class to make it work here too.
-
-    XXX This class needs to be cleaned up a *lot* of all the things we actually
-    don't need.
-    """
+def loadTestsAndOptions(classes):
     methodPrefix = 'test'
-    modulePrefix = 'test_'
-
-    def __init__(self):
-        self.suiteFactory = nettest.TestSuite
-        self._importErrors = []
-
-    def findTestClasses(self, module):
-        classes = []
-        for name, val in inspect.getmembers(module):
-            if isTestCase(val):
-                classes.append(val)
-            # This is here to allow backward compatibility with legacy OONI
-            # tests.
-            elif isLegacyTest(val):
-                print "adapting! %s" % val
-                val = adaptLegacyTest(val)
-                classes.append(val)
-        return classes
-
-    def loadClass(self, klass):
-        """
-        Given a class which contains test cases, return a sorted list of
-        C{TestCase} instances.
-        """
-        if not (isinstance(klass, type) or isinstance(klass, types.ClassType)):
-            raise TypeError("%r is not a class" % (klass,))
-        if not isTestCase(klass):
-            raise ValueError("%r is not a test case" % (klass,))
-        names = self.getTestCaseNames(klass)
-        tests = []
-        for name in names:
-            tests.append(self._makeCase(klass, self.methodPrefix+name))
-
-        suite = self.suiteFactory(tests)
-        print "**+*"
-        print tests
-        print "**+*"
-
-        return suite
-    loadTestsFromTestCase = loadClass
-
-    def findAllInputs(self, thing):
-        testClasses = self.findTestClasses(thing)
-        # XXX will there ever be more than 1 test class with inputs?
-        for klass in testClasses:
-            try:
-                inputs = klass.inputs
-            except:
-                pass
-        return inputs
-
-    def loadByNamesWithInput(self, names, recurse=False):
-        """
-        Construct a OONITestSuite containing all the tests found in 'names', where
-        names is a list of fully qualified python names and/or filenames. The
-        suite returned will have no duplicate tests, even if the same object
-        is named twice.
-
-        This test suite will have set the attribute inputs to the inputs found
-        inside of the tests.
-        """
-        inputs = []
-        things = []
-        errors = []
-        for name in names:
-            try:
-                thing = self.findByName(name)
-                things.append(thing)
-            except:
-                errors.append(ErrorHolder(name, failure.Failure()))
-        suites = []
-        for thing in self._uniqueTests(things):
-            inputs.append(self.findAllInputs(thing))
-            suite = self.loadAnything(thing, recurse)
-            suites.append(suite)
-
-        suites.extend(errors)
-        return inputs, suites
-
-class OONIRunner(object):
-    """
-    A specialised runner that is used by the ooniprobe frontend to run tests.
-    Heavily inspired by the trial TrialRunner class.
-    """
-
-    DEBUG = 'debug'
-    DRY_RUN = 'dry-run'
-
-    def _getDebugger(self):
-        dbg = pdb.Pdb()
+    suiteFactory = InputTestSuite
+    options = []
+    testCases = []
+    for klass in classes:
         try:
-            import readline
-        except ImportError:
-            print "readline module not available"
-            sys.exc_clear()
-        for path in ('.pdbrc', 'pdbrc'):
-            if os.path.exists(path):
-                try:
-                    rcFile = file(path, 'r')
-                except IOError:
-                    sys.exc_clear()
-                else:
-                    dbg.rcLines.extend(rcFile.readlines())
-        return dbg
-
-
-    def _setUpTestdir(self):
-        self._tearDownLogFile()
-        currentDir = os.getcwd()
-        base = filepath.FilePath(self.workingDirectory)
-        testdir, self._testDirLock = util._unusedTestDirectory(base)
-        os.chdir(testdir.path)
-        return currentDir
-
-
-    def _tearDownTestdir(self, oldDir):
-        os.chdir(oldDir)
-        self._testDirLock.unlock()
-
-
-    _log = log
-    def _makeResult(self):
-        reporter = self.reporterFactory(self.stream, self.tbformat,
-                                        self.rterrors, self._log)
-        if self.uncleanWarnings:
-            reporter = UncleanWarningsReporterWrapper(reporter)
-        return reporter
-
-    def __init__(self, reporterFactory,
-                 reportfile="report.yaml",
-                 mode=None,
-                 logfile='test.log',
-                 stream=sys.stdout,
-                 profile=False,
-                 tracebackFormat='default',
-                 realTimeErrors=False,
-                 uncleanWarnings=False,
-                 workingDirectory=None,
-                 forceGarbageCollection=False):
-        self.reporterFactory = reporterFactory
-        self._reportfile = reportfile
-        self.logfile = logfile
-        self.mode = mode
-        self.stream = stream
-        self.tbformat = tracebackFormat
-        self.rterrors = realTimeErrors
-        self.uncleanWarnings = uncleanWarnings
-        self._result = None
-        self.workingDirectory = workingDirectory or '_trial_temp'
-        self._logFileObserver = None
-        self._logFileObject = None
-        self._forceGarbageCollection = forceGarbageCollection
-        if profile:
-            self.run = util.profiled(self.run, 'profile.data')
-
-    def _tearDownLogFile(self):
-        if self._logFileObserver is not None:
-            log.removeObserver(self._logFileObserver.emit)
-            self._logFileObserver = None
-        if self._logFileObject is not None:
-            self._logFileObject.close()
-            self._logFileObject = None
-
-    def _setUpLogFile(self):
-        self._tearDownLogFile()
-        if self.logfile == '-':
-            logFile = sys.stdout
+            k = klass()
+            options.append(k.getOptions())
+        except AttributeError:
+            options.append([])
+
+        tests = reflect.prefixedMethodNames(klass, methodPrefix)
+        if tests:
+            cases = makeTestCases(klass, tests, methodPrefix)
+            testCases.append(cases)
         else:
-            logFile = file(self.logfile, 'a')
-        self._logFileObject = logFile
-        self._logFileObserver = log.FileLogObserver(logFile)
-        log.startLoggingWithObserver(self._logFileObserver.emit, 0)
+            options.pop()
+
+    return testCases, options
+
+class ORunner(object):
+    def __init__(self, cases, options=None):
+        self.baseSuite = InputTestSuite
+        self.cases = cases
+        self.options = options
+        self.inputs = options['inputs']
+        self.reporterFactory = ReporterFactory(open('foo.log', 'a+'),
+                testSuite=self.baseSuite(self.cases))
+
+    def runWithInputUnit(self, inputUnit):
+        idx = 0
+        result = self.reporterFactory.create()
+        for input in inputUnit:
+            suite = self.baseSuite(self.cases)
+            suite.input = input
+            suite(result, idx)
+
+            # XXX refactor all of this index bullshit to avoid having to pass
+            # this index around. Probably what I want to do is go and make
+            # changes to report to support the concept of having multiple runs
+            # of the same test.
+            # We currently need to do this addition in order to get the number
+            # of times the test cases that have run inside of the test suite.
+            idx += (suite._idx - idx)
+
+        result.done()
+
+    def run(self):
+        self.reporterFactory.writeHeader()
+
+        for inputUnit in InputUnitFactory(self.inputs):
+            self.runWithInputUnit(inputUnit)
 
-    def run(self, tests, inputs=[None]):
-        """
-        Run the test or suite and return a result object.
-        """
-        reporterFactory = ReporterFactory(open(self._reportfile, 'a+'),
-                testSuite=tests)
-        reporterFactory.writeHeader()
-        for inputUnit in InputUnitFactory(inputs):
-            testSuiteFactory = nettest.TestSuiteFactory(inputUnit, tests, nettest.TestSuite)
-            testUnitReport = reporterFactory.create()
-            for suite in testSuiteFactory:
-                suite(testUnitReport)
-            testUnitReport.done()
 





More information about the tor-commits mailing list