[tor-commits] [ooni-probe/master] Added timeout and abort for test inputs, methods, and classes.
isis at torproject.org
isis at torproject.org
Tue Dec 18 05:53:46 UTC 2012
commit 4e52ebe138393a3d942460890047b24ff467fdbc
Author: Isis Lovecruft <isis at torproject.org>
Date: Mon Dec 3 17:47:32 2012 +0000
Added timeout and abort for test inputs, methods, and classes.
---
nettests/bridge_reachability/tcpsyn.py | 2 +-
ooni/nettest.py | 115 ++++++++++++++++++++++++++-----
ooni/runner.py | 103 ++++++++++++++++++++++++----
3 files changed, 186 insertions(+), 34 deletions(-)
diff --git a/nettests/bridge_reachability/tcpsyn.py b/nettests/bridge_reachability/tcpsyn.py
index 39c882b..f92ab09 100644
--- a/nettests/bridge_reachability/tcpsyn.py
+++ b/nettests/bridge_reachability/tcpsyn.py
@@ -63,7 +63,7 @@ class TCPFlagTest(nettest.NetTestCase):
#destinations = {}
- @log.catcher
+ @log.catch
def setUp(self, *a, **kw):
"""Configure commandline parameters for TCPSynTest."""
self.report = {}
diff --git a/ooni/nettest.py b/ooni/nettest.py
index a868ac4..139b4f4 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -13,12 +13,26 @@ import os
import itertools
import traceback
-from twisted.trial import unittest, itrial, util
+from twisted.trial import unittest, itrial
+from twisted.trial import util as txtrutil
from twisted.internet import defer, utils
from twisted.python import usage
+from ooni import runner
from ooni.utils import log
+# This needs to be here so that NetTestCase.abort() can call it, since we
+# cannot import runner because runner imports NetTestCase.
+def isTestCase(obj):
+ """
+ Return True if obj is a subclass of NetTestCase, false if otherwise.
+ """
+ try:
+ return issubclass(obj, NetTestCase)
+ except TypeError:
+ return False
+
+
class NetTestCase(object):
"""
This is the base of the OONI nettest universe. When you write a nettest
@@ -61,21 +75,28 @@ class NetTestCase(object):
* requiresRoot: set to True if the test must be run as root.
- * optFlags: is assigned a list of lists. Each list represents a flag parameter, as so:
+ * optFlags:
+ is assigned a list of lists. Each list represents a flag
+ parameter, as so:
- optFlags = [['verbose', 'v', 'Makes it tell you what it doing.'], | ['quiet', 'q', 'Be vewy vewy quiet.']]
+ optFlags = [
+ ['verbose', 'v', 'Makes it tell you what it doing.'],
+ ['quiet', 'q', 'Be vewy vewy quiet.']]
As you can see, the first item is the long option name (prefixed with
'--' on the command line), followed by the short option name (prefixed with
'-'), and the description. The description is used for the built-in handling of
the --help switch, which prints a usage summary.
+ * optParameters:
+ is much the same, except the list also contains a default value:
- * optParameters: is much the same, except the list also contains a default value:
-
- | optParameters = [['outfile', 'O', 'outfile.log', 'Description...']]
+ optParameters = [
+ ['outfile', 'O', 'outfile.log', 'Description...']]
- * usageOptions: a subclass of twisted.python.usage.Options for more advanced command line arguments fun.
+ * usageOptions:
+ a subclass of twisted.python.usage.Options for more advanced command
+ line arguments fun.
* requiredOptions: a list containing the name of the options that are
required for proper running of a test.
@@ -97,22 +118,18 @@ class NetTestCase(object):
optFlags = None
optParameters = None
-
usageOptions = None
requiredOptions = []
requiresRoot = False
localOptions = {}
+
def _setUp(self):
- """
- This is the internal setup method to be overwritten by templates.
- """
+ """This is the internal setup method to be overwritten by templates."""
pass
def setUp(self):
- """
- Place here your logic to be executed when the test is being setup.
- """
+ """Place your logic to be executed when the test is being setup here."""
pass
def inputProcessor(self, filename=None):
@@ -166,8 +183,68 @@ class NetTestCase(object):
def __repr__(self):
return "<%s inputs=%s>" % (self.__class__, self.inputs)
- def __test_done__(self):
- up = inspect.stack()
- parent = up[1]
- # XXX call oreporter.allDone() from parent stack frame
- raise NotImplemented
+ def _getSkip(self):
+ return txtrutil.acquireAttribute(self._parents, 'skip', None)
+
+ #def _getSkipReason(self, method, skip):
+ # return super(TestCase, self)._getSkipReason(self, method, skip)
+
+ def _getTimeout(self):
+ """
+ Returns the timeout value set on this test. Check on the instance
+ first, the the class, then the module, then package. As soon as it
+ finds something with a timeout attribute, returns that. Returns
+ twisted.trial.util.DEFAULT_TIMEOUT_DURATION if it cannot find
+ anything. See TestCase docstring for more details.
+ """
+ testMethod = getattr(self, methodName)
+ self._parents = [testMethod, self]
+ self._parents.extend(txtrutil.getPythonContainers(testMethod))
+ timeout = txtrutil.acquireAttribute(self._parents, 'timeout',
+ txtrutil.DEFAULT_TIMEOUT_DURATION)
+ try:
+ return float(timeout)
+ except (ValueError, TypeError):
+ warnings.warn("'timeout' attribute needs to be a number.",
+ category=DeprecationWarning)
+ return txtrutil.DEFAULT_TIMEOUT_DURATION
+
+ def _abort(self, reason, obj=None):
+ """
+ Abort running an input, test_method, or test_class. If called with only
+ one argument, assume we're going to ignore the current input. Otherwise,
+ the name of the method or class in relation to the test_instance,
+ i.e. "self" should be given as value for the keyword argument "obj".
+
+ XXX call oreporter.allDone() from parent stack frame
+ """
+ reason = str(reason) # XXX should probably coerce
+ raise SkipTest("%s\n%s" % (str(reason), str(self.input)) )
+
+ def _abortMethod(self, reason, method):
+ if inspect.ismethod(method):
+ abort = getattr(self.__class__, method, False)
+ log.debug("Aborting remaining inputs for %s" % str(abort.func_name))
+ setattr(abort, 'skip', reason)
+ else:
+ log.debug("abortMethod(): could not find method %s" % str(method))
+
+ @log.catch
+ def _abortClass(self, reason, cls):
+ if not inspect.isclass(obj) or not runner.isTestCase(obj):
+ log.debug("_abortClass() could not find class %s" % str(cls))
+ return
+ abort = getattr(obj, '__class__', self.__class__)
+ log.debug("Aborting %s test" % str(abort.name))
+ setattr(abort, 'skip', reason)
+
+ def abortCurrentInput(self, reason):
+ """
+ Abort the current input.
+
+ @param reason: A string explaining why this test is being skipped.
+ """
+ return self._abort(reason)
+
+ def abortInput(self, reason):
+ return self._abort(reason)
diff --git a/ooni/runner.py b/ooni/runner.py
index 8d750ae..b6de21e 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -15,13 +15,14 @@ import inspect
import traceback
import itertools
-from twisted.python import reflect, usage
+from twisted.python import reflect, usage, failure
from twisted.internet import defer
from twisted.trial.runner import filenameToModule
+from twisted.trial import util as txtrutil
from twisted.internet import reactor, threads
from ooni.inputunit import InputUnitFactory
-from ooni.nettest import NetTestCase
+from ooni.nettest import NetTestCase, isTestCase
from ooni import reporter
@@ -92,12 +93,6 @@ def processTest(obj, cmd_line_options):
return obj
-def isTestCase(obj):
- try:
- return issubclass(obj, NetTestCase)
- except TypeError:
- return False
-
def findTestClassesFromConfig(cmd_line_options):
"""
Takes as input the command line config parameters and returns the test
@@ -112,7 +107,6 @@ def findTestClassesFromConfig(cmd_line_options):
A list of class objects found in a file or module given on the
commandline.
"""
-
filename = cmd_line_options['test']
classes = []
@@ -150,12 +144,64 @@ def loadTestsAndOptions(classes, cmd_line_options):
return test_cases, options
+def abortTestRun(test_class, warn_err_fail, test_input, oreporter):
+ """
+ Abort the entire test, and record the error, failure, or warning for why
+ it could not be completed.
+ """
+ log.warn("Aborting remaining tests for %s" % test_name)
+
+def abortTestWasCalled(abort_reason, abort_what, test_class, test_instance,
+ test_method, test_input, oreporter):
+ """
+ XXX
+ """
+ if not abort_what in ['class', 'method', 'input']:
+ log.warn("__test_abort__() must specify 'class', 'method', or 'input'")
+ abort_what = 'input'
+
+ if not isinstance(abort_reason, Exception):
+ abort_reason = Exception(str(abort_reason))
+ if abort_what == 'input':
+ log.msg("%s test requested to abort for input: %s"
+ % (test_instance.name, test_input))
+ d = maybeDeferred()
+
+ if hasattr(test_instance, "abort_all"):
+ log.msg("%s test requested to abort all remaining inputs"
+ % test_instance.name)
+ else:
+ d = defer.Deferred()
+ d.cancel()
+ d = abortTestRun(test_class, reason, test_input, oreporter)
+
+
def runTestWithInput(test_class, test_method, test_input, oreporter):
"""
Runs a single testcase from a NetTestCase with one input.
"""
log.debug("Running %s with %s" % (test_method, test_input))
+ def test_abort_single_input(reason, test_instance, test_name):
+ pass
+
+ def test_timeout(d):
+ err = defer.TimeoutError("%s test for %s timed out after %s seconds"
+ % (test_name, test_instance.input,
+ test_instance.timeout))
+ fail = failure.Failure(err)
+ try:
+ d.errback(fail)
+ except defer.AlreadyCalledError:
+ # if the deferred has already been called but the *back chain is
+ # still unfinished, crash the reactor and report the timeout
+ reactor.crash()
+ test_instance._timedOut = True # see test_instance._wait
+ # XXX result is TestResult utils?
+ RESULT.addExpectedFailure(test_instance, fail)
+ test_timeout = utils.suppressWarnings(
+ test_timeout, util.suppress(category=DeprecationWarning))
+
def test_done(result, test_instance, test_name):
log.debug("runTestWithInput: concluded %s" % test_name)
return oreporter.testDone(test_instance, test_name)
@@ -169,16 +215,45 @@ def runTestWithInput(test_class, test_method, test_input, oreporter):
log.debug("Processing %s" % test_instance.name)
# use this to keep track of the test runtime
test_instance._start_time = time.time()
+ test_instance.timeout = test_instance._getTimeout()
# call setups on the test
test_instance._setUp()
test_instance.setUp()
+ test_ignored = util.acquireAttribute(test_instance._parents, 'skip', None)
+
test = getattr(test_instance, test_method)
- d = defer.maybeDeferred(test)
- d.addCallback(test_done, test_instance, test_method)
- d.addErrback(test_error, test_instance, test_method)
- log.debug("returning %s input" % test_method)
- return d
+ # check if we've aborted
+ test_skip = test.getSkip()
+ if test_skip is not None:
+ log.debug("%s.getSkip() returned %s" % (str(test_class),
+ str(test_skip)) )
+
+ abort_reason, abort_what = getattr(test_instance, 'abort', ('input', None))
+ if abort_reason is not None:
+ do_abort = abortTestWasCalled(abort_reason, abort_what, test_class,
+ test_instance, test_method, test_input,
+ oreporter)
+ return defer.maybeDeferred(do_abort)
+ else:
+ d = defer.maybeDeferred(test)
+
+ # register the timer with the reactor
+ call = reactor.callLater(test_timeout, test_timed_out, d)
+ d.addBoth(lambda x: call.active() and call.cancel() or x)
+
+ # XXX check if test called test_abort...
+ d.addCallbacks(test_abort,
+ test_error,
+ callbackArgs=(test_instance, test_method),
+ errbackArgs=(test_instance, test_method) )
+ d.addCallback(test_done, test_instance, test_method)
+ d.addErrback(test_error, test_instance, test_method)
+ log.debug("returning %s input" % test_method)
+
+ ignored = d.getSkip()
+
+ return d
def runTestWithInputUnit(test_class, test_method, input_unit, oreporter):
"""
More information about the tor-commits
mailing list