commit 4e52ebe138393a3d942460890047b24ff467fdbc Author: Isis Lovecruft isis@torproject.org Date: Mon Dec 3 17:47:32 2012 +0000
Added timeout and abort for test inputs, methods, and classes. --- nettests/bridge_reachability/tcpsyn.py | 2 +- ooni/nettest.py | 115 ++++++++++++++++++++++++++----- ooni/runner.py | 103 ++++++++++++++++++++++++---- 3 files changed, 186 insertions(+), 34 deletions(-)
diff --git a/nettests/bridge_reachability/tcpsyn.py b/nettests/bridge_reachability/tcpsyn.py index 39c882b..f92ab09 100644 --- a/nettests/bridge_reachability/tcpsyn.py +++ b/nettests/bridge_reachability/tcpsyn.py @@ -63,7 +63,7 @@ class TCPFlagTest(nettest.NetTestCase):
#destinations = {}
- @log.catcher + @log.catch def setUp(self, *a, **kw): """Configure commandline parameters for TCPSynTest.""" self.report = {} diff --git a/ooni/nettest.py b/ooni/nettest.py index a868ac4..139b4f4 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -13,12 +13,26 @@ import os import itertools import traceback
-from twisted.trial import unittest, itrial, util +from twisted.trial import unittest, itrial +from twisted.trial import util as txtrutil from twisted.internet import defer, utils from twisted.python import usage
+from ooni import runner from ooni.utils import log
+# This needs to be here so that NetTestCase.abort() can call it, since we +# cannot import runner because runner imports NetTestCase. +def isTestCase(obj): + """ + Return True if obj is a subclass of NetTestCase, false if otherwise. + """ + try: + return issubclass(obj, NetTestCase) + except TypeError: + return False + + class NetTestCase(object): """ This is the base of the OONI nettest universe. When you write a nettest @@ -61,21 +75,28 @@ class NetTestCase(object):
* requiresRoot: set to True if the test must be run as root.
- * optFlags: is assigned a list of lists. Each list represents a flag parameter, as so: + * optFlags: + is assigned a list of lists. Each list represents a flag + parameter, as so:
- optFlags = [['verbose', 'v', 'Makes it tell you what it doing.'], | ['quiet', 'q', 'Be vewy vewy quiet.']] + optFlags = [ + ['verbose', 'v', 'Makes it tell you what it doing.'], + ['quiet', 'q', 'Be vewy vewy quiet.']]
As you can see, the first item is the long option name (prefixed with '--' on the command line), followed by the short option name (prefixed with '-'), and the description. The description is used for the built-in handling of the --help switch, which prints a usage summary.
+ * optParameters: + is much the same, except the list also contains a default value:
- * optParameters: is much the same, except the list also contains a default value: - - | optParameters = [['outfile', 'O', 'outfile.log', 'Description...']] + optParameters = [ + ['outfile', 'O', 'outfile.log', 'Description...']]
- * usageOptions: a subclass of twisted.python.usage.Options for more advanced command line arguments fun. + * usageOptions: + a subclass of twisted.python.usage.Options for more advanced command + line arguments fun.
* requiredOptions: a list containing the name of the options that are required for proper running of a test. @@ -97,22 +118,18 @@ class NetTestCase(object):
optFlags = None optParameters = None - usageOptions = None requiredOptions = [] requiresRoot = False
localOptions = {} + def _setUp(self): - """ - This is the internal setup method to be overwritten by templates. - """ + """This is the internal setup method to be overwritten by templates.""" pass
def setUp(self): - """ - Place here your logic to be executed when the test is being setup. - """ + """Place your logic to be executed when the test is being setup here.""" pass
def inputProcessor(self, filename=None): @@ -166,8 +183,68 @@ class NetTestCase(object): def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs)
- def __test_done__(self): - up = inspect.stack() - parent = up[1] - # XXX call oreporter.allDone() from parent stack frame - raise NotImplemented + def _getSkip(self): + return txtrutil.acquireAttribute(self._parents, 'skip', None) + + #def _getSkipReason(self, method, skip): + # return super(TestCase, self)._getSkipReason(self, method, skip) + + def _getTimeout(self): + """ + Returns the timeout value set on this test. Check on the instance + first, the the class, then the module, then package. As soon as it + finds something with a timeout attribute, returns that. Returns + twisted.trial.util.DEFAULT_TIMEOUT_DURATION if it cannot find + anything. See TestCase docstring for more details. + """ + testMethod = getattr(self, methodName) + self._parents = [testMethod, self] + self._parents.extend(txtrutil.getPythonContainers(testMethod)) + timeout = txtrutil.acquireAttribute(self._parents, 'timeout', + txtrutil.DEFAULT_TIMEOUT_DURATION) + try: + return float(timeout) + except (ValueError, TypeError): + warnings.warn("'timeout' attribute needs to be a number.", + category=DeprecationWarning) + return txtrutil.DEFAULT_TIMEOUT_DURATION + + def _abort(self, reason, obj=None): + """ + Abort running an input, test_method, or test_class. If called with only + one argument, assume we're going to ignore the current input. Otherwise, + the name of the method or class in relation to the test_instance, + i.e. "self" should be given as value for the keyword argument "obj". + + XXX call oreporter.allDone() from parent stack frame + """ + reason = str(reason) # XXX should probably coerce + raise SkipTest("%s\n%s" % (str(reason), str(self.input)) ) + + def _abortMethod(self, reason, method): + if inspect.ismethod(method): + abort = getattr(self.__class__, method, False) + log.debug("Aborting remaining inputs for %s" % str(abort.func_name)) + setattr(abort, 'skip', reason) + else: + log.debug("abortMethod(): could not find method %s" % str(method)) + + @log.catch + def _abortClass(self, reason, cls): + if not inspect.isclass(obj) or not runner.isTestCase(obj): + log.debug("_abortClass() could not find class %s" % str(cls)) + return + abort = getattr(obj, '__class__', self.__class__) + log.debug("Aborting %s test" % str(abort.name)) + setattr(abort, 'skip', reason) + + def abortCurrentInput(self, reason): + """ + Abort the current input. + + @param reason: A string explaining why this test is being skipped. + """ + return self._abort(reason) + + def abortInput(self, reason): + return self._abort(reason) diff --git a/ooni/runner.py b/ooni/runner.py index 8d750ae..b6de21e 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -15,13 +15,14 @@ import inspect import traceback import itertools
-from twisted.python import reflect, usage +from twisted.python import reflect, usage, failure from twisted.internet import defer from twisted.trial.runner import filenameToModule +from twisted.trial import util as txtrutil from twisted.internet import reactor, threads
from ooni.inputunit import InputUnitFactory -from ooni.nettest import NetTestCase +from ooni.nettest import NetTestCase, isTestCase
from ooni import reporter
@@ -92,12 +93,6 @@ def processTest(obj, cmd_line_options):
return obj
-def isTestCase(obj): - try: - return issubclass(obj, NetTestCase) - except TypeError: - return False - def findTestClassesFromConfig(cmd_line_options): """ Takes as input the command line config parameters and returns the test @@ -112,7 +107,6 @@ def findTestClassesFromConfig(cmd_line_options): A list of class objects found in a file or module given on the commandline. """ - filename = cmd_line_options['test'] classes = []
@@ -150,12 +144,64 @@ def loadTestsAndOptions(classes, cmd_line_options):
return test_cases, options
+def abortTestRun(test_class, warn_err_fail, test_input, oreporter): + """ + Abort the entire test, and record the error, failure, or warning for why + it could not be completed. + """ + log.warn("Aborting remaining tests for %s" % test_name) + +def abortTestWasCalled(abort_reason, abort_what, test_class, test_instance, + test_method, test_input, oreporter): + """ + XXX + """ + if not abort_what in ['class', 'method', 'input']: + log.warn("__test_abort__() must specify 'class', 'method', or 'input'") + abort_what = 'input' + + if not isinstance(abort_reason, Exception): + abort_reason = Exception(str(abort_reason)) + if abort_what == 'input': + log.msg("%s test requested to abort for input: %s" + % (test_instance.name, test_input)) + d = maybeDeferred() + + if hasattr(test_instance, "abort_all"): + log.msg("%s test requested to abort all remaining inputs" + % test_instance.name) + else: + d = defer.Deferred() + d.cancel() + d = abortTestRun(test_class, reason, test_input, oreporter) + + def runTestWithInput(test_class, test_method, test_input, oreporter): """ Runs a single testcase from a NetTestCase with one input. """ log.debug("Running %s with %s" % (test_method, test_input))
+ def test_abort_single_input(reason, test_instance, test_name): + pass + + def test_timeout(d): + err = defer.TimeoutError("%s test for %s timed out after %s seconds" + % (test_name, test_instance.input, + test_instance.timeout)) + fail = failure.Failure(err) + try: + d.errback(fail) + except defer.AlreadyCalledError: + # if the deferred has already been called but the *back chain is + # still unfinished, crash the reactor and report the timeout + reactor.crash() + test_instance._timedOut = True # see test_instance._wait + # XXX result is TestResult utils? + RESULT.addExpectedFailure(test_instance, fail) + test_timeout = utils.suppressWarnings( + test_timeout, util.suppress(category=DeprecationWarning)) + def test_done(result, test_instance, test_name): log.debug("runTestWithInput: concluded %s" % test_name) return oreporter.testDone(test_instance, test_name) @@ -169,16 +215,45 @@ def runTestWithInput(test_class, test_method, test_input, oreporter): log.debug("Processing %s" % test_instance.name) # use this to keep track of the test runtime test_instance._start_time = time.time() + test_instance.timeout = test_instance._getTimeout() # call setups on the test test_instance._setUp() test_instance.setUp() + test_ignored = util.acquireAttribute(test_instance._parents, 'skip', None) + test = getattr(test_instance, test_method)
- d = defer.maybeDeferred(test) - d.addCallback(test_done, test_instance, test_method) - d.addErrback(test_error, test_instance, test_method) - log.debug("returning %s input" % test_method) - return d + # check if we've aborted + test_skip = test.getSkip() + if test_skip is not None: + log.debug("%s.getSkip() returned %s" % (str(test_class), + str(test_skip)) ) + + abort_reason, abort_what = getattr(test_instance, 'abort', ('input', None)) + if abort_reason is not None: + do_abort = abortTestWasCalled(abort_reason, abort_what, test_class, + test_instance, test_method, test_input, + oreporter) + return defer.maybeDeferred(do_abort) + else: + d = defer.maybeDeferred(test) + + # register the timer with the reactor + call = reactor.callLater(test_timeout, test_timed_out, d) + d.addBoth(lambda x: call.active() and call.cancel() or x) + + # XXX check if test called test_abort... + d.addCallbacks(test_abort, + test_error, + callbackArgs=(test_instance, test_method), + errbackArgs=(test_instance, test_method) ) + d.addCallback(test_done, test_instance, test_method) + d.addErrback(test_error, test_instance, test_method) + log.debug("returning %s input" % test_method) + + ignored = d.getSkip() + + return d
def runTestWithInputUnit(test_class, test_method, input_unit, oreporter): """