commit 9b2132cea553a7e723efa1fb3afcce425c745ca3 Author: Isis Lovecruft <isis@torproject.org> Date: Sun Nov 4 03:59:03 2012 +0000 * Updated the Makefile which is used for checking if OONI tests are working. * Changes to nettest and runner to make them call parent classes correctly. * Started working on adaptor class for nettest.TestCase. --- bin/Makefile | 20 ++-- nettests/core/echo.py | 1 + ooni/nettest.py | 272 +++++++++++++++++++++++++++++++++++++------------ ooni/runner.py | 240 +++++++++++++++++++++++++------------------- 4 files changed, 355 insertions(+), 178 deletions(-) diff --git a/bin/Makefile b/bin/Makefile index 307e81f..ee8976f 100644 --- a/bin/Makefile +++ b/bin/Makefile @@ -21,34 +21,34 @@ all: echot simplet captivet dnst httphostt squidt all_debug: echod simpled captived dnsd httphostd squidd simplet: - ../bin/ooniprobe ../nettests/simpletest.py -a ../ooni/assets/hostnames.txt + ../bin/ooniprobe ../nettests/simpletest.py -a ../lists/short_hostname_list.txt simpled: - python -m pdb ../bin/ooniprobe ../nettests/simpletest.py -a ../ooni/assets/hostnames.txt + python -m pdb ../bin/ooniprobe ../nettests/simpletest.py -a ../lists/short_hostname_list.txt echot: - ../bin/ooniprobe ../nettests/core/echo.py -f ../ooni/assets/hostnames.txt + ../bin/ooniprobe ../nettests/core/echo.py -f ../lists/short_hostname_list.txt echod: - python -m pdb ../bin/ooniprobe ../nettests/core/echo.py -f ../ooni/assets/hostnames.txt + python -m pdb ../bin/ooniprobe -B ../nettests/core/echo.py -f ../lists/short_hostname_list.txt captivet: - ../bin/ooniprobe ../nettests/core/captiveportal.py -f ../ooni/assets/hostnames.txt + ../bin/ooniprobe ../nettests/core/captiveportal.py -f ../lists/short_hostname_list.txt captived: - python -m pdb ../bin/ooniprobe --spew ../nettests/core/captiveportal.py -f ../ooni/assets/hostnames.txt + python -m pdb ../bin/ooniprobe --spew ../nettests/core/captiveportal.py -f ../lists/short_hostname_list.txt dnst: - ../bin/ooniprobe ../nettests/core/dnstamper.py -f ../ooni/assets/hostnames.txt + ../bin/ooniprobe ../nettests/core/dnstamper.py -f ../lists/short_hostname_list.txt dnsd: - python -m pdb ../bin/ooniprobe --spew ../nettests/core/dnstamper.py -f ../ooni/assets/hostnames.txt + python -m pdb ../bin/ooniprobe --spew ../nettests/core/dnstamper.py -f ../lists/short_hostname_list.txt squidt: - ../bin/ooniprobe ../nettests/core/squid.py -f ../ooni/assets/hostnames.txt + ../bin/ooniprobe ../nettests/core/squid.py -f ../lists/short_hostname_list.txt squidd: - python -m pdb ../bin/ooniprobe --spew ../nettests/core/squid.py -f ../ooni/assets/hostnames.txt + python -m pdb ../bin/ooniprobe --spew ../nettests/core/squid.py -f ../lists/short_hostname_list.txt #mvreports: # for $report in `find ../ -name "report*"`; do mv $report test-results #; done diff --git a/nettests/core/echo.py b/nettests/core/echo.py new file mode 120000 index 0000000..d9926cd --- /dev/null +++ b/nettests/core/echo.py @@ -0,0 +1 @@ +../../ooni/bridget/tests/echo.py \ No newline at end of file diff --git a/ooni/nettest.py b/ooni/nettest.py index 541d65b..e01c4f1 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -65,7 +65,206 @@ class InputTestSuite(pyunit.TestSuite): return result -class TestCase(unittest.TestCase): +class NetTestAdaptor(unittest.TestCase): + """ + XXX fill me in + """ + @staticmethod + def __copyattr__(obj, old, new=None, alt=None): + """ + Assign me to a new attribute name to have a copy of the old + attribute, if it exists. + + Example: + >>> self.sun = "black" + >>> self._watermelon = __copyattr__(self, "sun") + >>> self._clocknoise = __copyattr__(self, "sound") + >>> print self._watermelon + black + >>> print self._clocknoise + + :param old: + A string representing the name of the old attribute + :param new: + (Optional) A string to set as the new attribute name. + :param alt: + (Optional) An alternate value to return if the old + attribute is not found. + :return: + If :param:`old` is found, I return it's value. + + If :param:`old` is not found: + If :param:`alt` is given, I return :param:`alt`. + Else, I return None. + + If :param:`new` is set, I do not return anything, and + instead I set the new attribute's name to :param:`name` + and it's value to the value which I would have otherwise + returned. + """ + if not new: + if not alt: + if hasattr(obj, old): + return getattr(obj, old) + return + else: + if hasattr(obj, old): + return getattr(obj, old) + return alt + else: + if not alt: + if hasattr(obj, old): + _copy = getattr(obj, old) + else: + copy = None + setattr(obj, new, _copy) + else: + if hasattr(obj, old): + _copy = getattr(obj, old) + else: + _copy = alt + setattr(obj, new, _copy) + + ## Using setattr in __init__ for now: + #def copyattr(self, *args, **kwargs): + # if len(args) >= 1: + # _copy = partial(__copyattr__, args[0]) + # if len(args) == 2: + # return _copy(new=args[1]) + # elif len(args) == 3: + # return _copy(new=args[1], alt=args[2]) + # elif kwargs: + # return _copy(kwargs) + # else: + # return + + def __init__(self, *args, **kwargs): + """ + If you override me, you must call + + ``super(NetTestCase, self).__init__(*args, **kwargs)`` + + at the beginning of your __init__ method. Keyword arguments passed to + the above statement become attributes of the adaptor, and can be used + to alter the logic of input handling and parent class instantiation. + Therefore, You probably do not need to pass me any keyword arguments + when calling me, i.e. using ``(*args, **kwargs)`` will work just fine. + """ + log.debug("NetTestAdapter: created") + if kwargs: + if 'methodName' in kwargs: + log.debug("NetTestAdaptor: found 'methodName' in kwargs") + log.debug("NetTestAdaptor: calling unittest.TestCase.__init()") + super( NetTestAdaptor, self ).__init__( + methodName=kwargs['methodName'] ) + else: + log.debug("NetTestAdaptor: calling unittest.TestCase.__init()") + super( NetTestAdaptor, self ).__init__( ) + + for key, value in kwargs.items(): ## Let subclasses define their + if key != 'methodName': ## instantiation without + if not hasattr(self, key): ## overriding parent classes + log.debug("NetTestAdaptor: calling setattr(self,%s,%s)" + % (key, value) ) + setattr(self, key, value) + + #setattr(self, "copyattr", __copy_attr__) + + ## Internal attribute copies: + #self._input_parser = copyattr("inputParser", alt=__input_parser__) + #self._nettest_name = copyattr("name", alt="NetTestAdaptor")) + + ## Set our inputs to the parsed and processed inputs: + #self.inputs = __get_inputs__() + + ## xxx do we need: + if self.parsed_inputs: + self.inputs = self.parsed_inputs + else: + log.debug("Unable to find parsed inputs") + + @staticmethod + def __input_parser__(one_input): return one_input + + @classmethod + def setUpClass(cls): + """ + Create a TestCase instance. This function is equivalent to '__init__'. + To add futher setup steps before a set of tests in a TestCase instance + run, create a function called 'setUp'. + + Class attributes, such as `report`, `optParameters`, `name`, and + `author` should be overriden statically as class attributes in any + subclass of :class:`ooni.nettest.TestCase`, so that the calling + functions in ooni.runner can handle them correctly. + """ + cls._raw_inputs = __copyattr__(cls, "inputs") + cls._input_file = __copyattr__(cls, "inputFile") + cls._input_parser = __copyattr__(cls, "inputParser", alt=__input_parser__) + cls._nettest_name = __copyattr__(cls, "name", alt="NetTestAdaptor") + cls.parsed_inputs = __get_inputs__(cls) + ## XXX we should handle options generation here + + @classmethod + def __get_inputs__(cls): + """ + I am called from the ooni.runner and you probably should not override + me. I gather the internal inputs from :class:`NetTestCase` attributes + and pass them through :meth:`NetTestCase.inputParser`. If you are + looking for a way to parse inputs from inputFile, see + :meth:`inputParser`. If :class:`NetTestCase` has an attribute + :attr:`inputFile`, I also handle opening that file, striping each line + of whitespace, and then sending the line to + :meth:`NetTestCase.inputParser`. + + All inputs which I find, both statically set inputs and those returned + from processing an inputFile, I add to a list :ivar:`parsed`, after + parsing them. I return :ivar:`parsed`: + + :ivar parsed: + A list of parsed inputs. + :return: + :ivar:`parsed`. + """ + parsed = [] + + if cls._raw_inputs: + if isinstance(cls._raw_inputs, (list, tuple,)): + if len(cls._raw_inputs) > 0: + if len(cls._raw_inputs) == 1 and cls._raw_inputs[0] is None: + pass ## don't burn cycles on testing null inputs + else: + log.msg("Received direct inputs:\n%s" % cls._raw_inputs) + parsed.extend([cls._input_parser(x) for x in cls._raw_inputs]) + elif isinstance(cls._raw_inputs, str): + separated = cls._raw_inputs.translate(None, ',') ## space delineates + inputlist = separated.split(' ') + parsed.extend([cls._input_parser(x) for x in inputlist]) + else: + log.debug("inputs not string or list; type: %s" + % type(cls._raw_inputs)) + + if cls._input_file: + try: + log.debug("Opening input file") + fp = open(cls._input_file) + except: + log.debug("Couldn't open input file") + else: + log.debug("Running input file processor") + lines = [line.strip() for line in fp.readlines()] + fp.close() + + ## add to what we've already parsed, if any: + log.debug("Parsing lines from input file") + parsed.extend([cls._input_parser(ln) for ln in lines]) + else: + log.debug("%s specified that it doesn't need inputFile." + % cls._nettest_name) + + return parsed + +class NetTestCase(NetTestAdaptor): """ This is the monad of the OONI nettest universe. When you write a nettest you will subclass this object. @@ -109,7 +308,7 @@ class TestCase(unittest.TestCase): """ name = "I Did Not Change The Name" author = "Jane Doe <foo@example.com>" - version = "0" + version = "0.0.0" inputFile = None inputs = [None] @@ -118,28 +317,13 @@ class TestCase(unittest.TestCase): report['errors'] = [] optParameters = None + optFlags = None + subCommands = None requiresRoot = False - def setUpClass(self, *args, **kwargs): - """ - Create a TestCase instance. This function is equivalent to '__init__'. - To add futher setup steps before a set of tests in a TestCase instance - run, create a function called 'setUp'. - - Class attributes, such as `report`, `optParameters`, `name`, and - `author` should be overriden statically as class attributes in any - subclass of :class:`ooni.nettest.TestCase`, so that the calling - functions in ooni.runner can handle them correctly. - """ - if kwargs and 'methodName' in kwargs: - return super( TestCase, self ).setUpClass( - methodName=kwargs['methodName'] ) - return super( TestCase, self ).setUpClass( ) - - #for key, value in kwargs.items(): - # setattr(self.__class__, key, value) - # - #self.inputs = self.getInputs() + @classmethod + def setUpClass(cls): + pass def deferSetUp(self, ignored, result): """ @@ -157,53 +341,11 @@ class TestCase(unittest.TestCase): log.debug("Not first run. Running test setup directly") return unittest.TestCase.deferSetUp(self, ignored, result) - @staticmethod - def inputParser(inputs): + def inputParser(self, inputs): """Replace me with a custom function for parsing inputs.""" log.debug("Running custom input processor") return inputs - def _getInputs(self): - """ - I am called from the ooni.runner and you probably should not override - me. I gather the internal inputs from an instantiated test class and - pass them to the rest of the runner. - - If you are looking for a way to parse inputs from inputFile, see - :meth:`inputParser`. - I open :attr:inputFile if there is one, and return inputs one by one - after stripping them of whitespace and running them through the parser - :meth:`inputParser`. - """ - - ## don't burn cycles on testing null inputs: - if len(self.inputs) == 1 and self.inputs[0] == None: - self.inputs = [] - processor = [] - else: - log.msg("Received direct inputs:\n%s" % self.inputs) - processor = [i for i in self.inputParser(self.inputs)] - - if self.inputFile: - try: - fp = open(self.inputFile) - except Exception, e: - log.err(e) - fp.close() - else: - log.debug("Running input file processor") - lines = [line.strip() for line in fp.readlines()] - fp.close() - parsed = [self.inputParser(ln) for ln in lines] - both = itertools.chain(processor, parsed) - elif self.inputFile is False: - log.debug("%s specified that it doesn't need inputFile." - % self.name) - both = processor - else: - both = processor - return both - def getOptions(self): ''' diff --git a/ooni/runner.py b/ooni/runner.py index ae086c1..cf3a7ee 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -22,7 +22,6 @@ from ooni.nettest import InputTestSuite, TestCase from ooni.plugoo import tests as oonitests from ooni.reporter import ReporterFactory from ooni.utils import log, date -from ooni.utils.assertions import isClass from ooni.utils.legacy import LegacyOONITest from ooni.utils.legacy import start_legacy_test, adapt_legacy_test @@ -127,42 +126,48 @@ def processTestOptions(cls, config): A configured and instantiated :class:`twisted.python.usage.Options` class. """ - if cls.optParameters or cls.inputFile: - if not cls.optParameters: - cls.optParameters = [] + #if cls.optParameters or cls.inputFile: + if not cls.optParameters: + cls.optParameters = [] - if cls.inputFile: - cls.optParameters.append(cls.inputFile) + if cls.inputFile: + cls.optParameters.append(cls.inputFile) - if not hasattr(cls, subCommands): - cls.subCommands = [] + log.debug("CLS IS %s" % cls) + log.debug("CLS OPTPARAM IS %s" % cls.optParameters) - class Options(usage.Options): - optParameters = cls.optParameters - parseArgs = lambda a: cls.subCommands.append(a) + #if not hasattr(cls, subCommands): + # cls.subCommands = [] + + if not cls.subCommands: + cls.subCommands = [] + + class Options(usage.Options): + optParameters = cls.optParameters + parseArgs = lambda a: cls.subCommands.append(a) - opts = Options() - opts.parseOptions(config['subArgs']) - cls.localOptions = opts + opts = Options() + opts.parseOptions(config['subArgs']) + cls.localOptions = opts - if cls.inputFile: - cls.inputFile = opts[cls.inputFile[0]] - """ + if cls.inputFile: + cls.inputFile = opts[cls.inputFile[0]] + """ + try: + log.debug("%s: trying %s.localoptions.getOptions()..." + % (__name__, cls.name)) try: - log.debug("%s: trying %s.localoptions.getOptions()..." - % (__name__, cls.name)) - try: - assert hasattr(cls, 'getOptions') - except AssertionError, ae: - options = opts.opt_help() - raise Exception, "Cannot find %s.getOptions()" % cls.name - else: - options = cls.getOptions() - except usage.UsageError: + assert hasattr(cls, 'getOptions') + except AssertionError, ae: options = opts.opt_help() + raise Exception, "Cannot find %s.getOptions()" % cls.name else: - """ - return cls.localOptions + options = cls.getOptions() + except usage.UsageError: + options = opts.opt_help() + else: + """ + return cls.localOptions def loadTestsAndOptions(classes, config): """ @@ -178,90 +183,119 @@ def loadTestsAndOptions(classes, config): DEPRECATED = LegacyOONITest for klass in classes: - if isinstance(klass, DEPRECATED) \ - and not issubclass(klass, TestCase): - log.msg("Processing cases and options for legacy test %s" - % ( klass.shortName if hasattr(klass, shortName) - else 'oonitest' )) - if hasattr(klass, description): - log.msg("%s" % klass.description) - - subcmds = [] - if hasattr(klass, options): ## an unitiated Legacy test - log.debug("%s.options found: %s " % (klass, klass.options)) - try: - assert isclass(klass.options), \ - "%s is not class" % klass.qoptions - except AssertionError, ae: - log.debug(ae) - else: - ok = klass.options - ok.parseArgs = lambda x: subcmds.append(x) - try: - opts = ok() - opts.parseOptions(config['subArgs']) - except Exception, e: - log.debug(e) - opts = {} - finally: - opts.append(opts) - - if hasattr(klass, local_options): ## we've been initialized already - log.debug("%s.local_options found" % klass) + if isinstance(klass, DEPRECATED): + #not issubclass(klass, TestCase): + try: + cases, opts = processLegacyTest(klass, config) + if cases: + log.debug("Processing cases: %s" % str(cases)) + return [], [] + test_cases.append(cases) + except Exception, e: + log.err(e) + else: try: - assert klass.local_options is not None opts = klass.local_options + option.append(opts) except AttributeError, ae: - opts = {}; log.debug(ae) - finally: - log.debug("type(opts) = %s" % type(opts)) - options.append(opts) - try: - cases = start_legacy_test(klass) - #if cases: ## why are these empty lists here? - # return [], [] ## were nettests having issues due - except Exception, e: ## to legacy tests? - cases = []; log.err(e) - finally: - log.debug(str(cases)) - test_cases.append(cases) + options.append([]) + log.err(ae) elif issubclass(klass, TestCase): - log.debug("Processing cases and options for OONI %s test" - % ( klass.name if hasattr( klass, 'name' ) \ - else TestCase.name )) try: - tests = reflect.prefixedMethodNames(klass, method_prefix) + cases, opts = processNetTest(klass, config, method_prefix) except Exception, e: - tests = []; log.debug(e) + log.err(e) else: - try: - cases = makeTestCases(klass, tests, method_prefix) - except Exception, e: - cases = []; log.err(e) - log.debug("loadTestsAndOptions(): test %s found cases=%s" - % (tests, cases)) + test_cases.append(cases) + options.append(opts) - if hasattr(klass, 'optParameters') or hasattr(klass, 'inputFile'): - try: - opt = processTestOptions(klass, config) - print opt, config - except: - opt = {} - finally: - instance = klass() - try: - inputs = instance._getInputs() - except Exception, e: - inputs = []; log.err(e) - else: - if opt and opt is not None: - opt.update({'inputs':inputs}) - else: pass - finally: options.append(opt) - log.debug("loadTestsAndOptions(): type(opt)=%s" % type(opt)) return test_cases, options +def processNetTest(klass, config, method_prefix): + try: + log.debug("Processing cases and options for OONI %s test" + % (klass.name if hasattr(klass, 'name') else 'Network Test')) + + tests = reflect.prefixedMethodNames(klass, method_prefix) + if tests: + cases = makeTestCases(klass, tests, method_prefix) + log.debug("loadTestsAndOptions(): test %s found cases=%s"% (tests, cases)) + try: + k = klass() + opts = processTestOptions(k, config) + except Exception, e: + opts = [] + log.err(e) + else: + cases = [] + except Exception, e: + log.err(e) + + return cases, opts + +''' + if hasattr(klass, 'optParameters') or hasattr(klass, 'inputFile'): + try: + opts = processTestOptions(klass, config) + except: + opts = [] + finally: + try: + k = klass() + inputs = k._getInputs() + except Exception, e: + inputs = [] + log.err(e) + else: + if opts and len(inputs) != 0: + opts.append(['inputs', '', inputs, "cmdline inputs"]) + log.debug("loadTestsAndOptions(): inputs=%s" % inputs) +''' + +def processLegacyTest(klass, config): + log.msg("Processing cases and options for legacy test %s" + % ( klass.shortName if hasattr(klass, shortName) else 'oonitest' )) + if hasattr(klass, description): + log.msg("%s" % klass.description) + + subcmds = [] + if hasattr(klass, options): ## an unitiated Legacy test + log.debug("%s.options found: %s " % (klass, klass.options)) + try: + assert isclass(klass.options), "legacytest.options is not class" + except AssertionError, ae: + log.debug(ae) + else: + ok = klass.options + ok.parseArgs = lambda x: subcmds.append(x) + try: + opts = ok() + opts.parseOptions(config['subArgs']) + except Exception, e: + log.err(e) + opts = {} + + elif hasattr(klass, local_options): ## we've been initialized already + log.debug("%s.local_options found" % klass) + try: + assert klass.local_options is not None + opts = klass.local_options + except AttributeError, ae: + opts = {}; log.err(ae) + + try: + cases = start_legacy_test(klass) + ## XXX we need to get these results into the reporter + if cases: + return [], [] + except Exception, e: + cases = []; log.err(e) + finally: + log.debug(str(cases)) + + return cases, opts + class ORunner(object): """ This is a specialized runner used by the ooniprobe command line tool. @@ -277,8 +311,8 @@ class ORunner(object): try: assert len(options) != 0, "Length of options is zero!" except AssertionError, ae: - self.inputs = [] log.err(ae) + self.inputs = [] else: try: first = options.pop(0)