commit b391f1dc286d318178d0c0b622748b7b604d45d0 Author: seamus tuohy code@seamustuohy.com Date: Sun Jun 5 13:01:24 2016 -0400
Make inputs a shared iterator across all NetTestCase measurements
This commit makes the inputs iterator shared by all the measurement tests within a NetTestCase. Per issue 503, In order to make bisection style testing functional the individual tests need to be able to pass data to the main test_case.inputs generator that seeds each measurement. This provides opportunities for each measurement test in a NetTestCase to communicate with the iterator to do things such as adding additional values to be passed to measurement tests as a later input. --- docs/source/writing_tests.rst | 61 ++++++++++++++++++++++- ooni/nettest.py | 10 ++-- ooni/tests/test_nettest.py | 109 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 6 deletions(-)
diff --git a/docs/source/writing_tests.rst b/docs/source/writing_tests.rst index 2132010..aaa4adf 100644 --- a/docs/source/writing_tests.rst +++ b/docs/source/writing_tests.rst @@ -57,14 +57,71 @@ this:: yield x.strip() fp.close()
-For example, if you wanted to modify inputProcessor to read enteries from a CSV file, you could use:: - +For example, if you wanted to modify inputProcessor to read entries from a CSV file, you could use:: + def inputProcessor(self, filename): with open(filename) as csvFile: reader = DictReader(csvFile) for entry in reader: yield entry
+ +The ``inputs`` iterator is unique per ``NetTestCase`` and shared by all its measurement tests. This provides opportunities for each measurement test in a ``NetTestCase`` to communicate with the iterator to do things such as adding additional values to be passed to measurement tests as a later ``input``. + +.. note :: Deleting/removing the current item from the ``inputs`` iterator will not stop other measurement tests from operating on that ``input``. When a ``NetTestCase`` is run a single ``input`` is taken from the ``inputs`` iterator by the OONI test loader and run against all of the individual measurement tests within that ``NetTestCase``. Removing an ``input`` from the ``inputs`` iterator during a measurement will not stop that input from being called on all other measurement tests within the NetTestCase. + +Here is one example of how you can take advantage of shared ``inputs`` iterator when writing your own OONI tests. If you have a list of urls and you want to make sure that you always test the HTTP equivalent of any HTTPS urls provided you can ``send`` values back to a custom generator in your ``postProcessor``. + +To do this the first thing you would need to create is a URL generator that can accept values sent to it. + +:: + class UrlGeneratorWithSend(object): + def __init__(self): + """Create initial list and set generator state.""" + self.urls = ["http://www.torproject.org", + "https://ooni.torproject.org"] + self.current = 0 + + def __iter__(self): + return self + + def __next__(self): + try: + cur = self.urls[self.current] + self.current += 1 + return cur + except IndexError: + raise StopIteration + + # Python 2 & 3 generator compatibility + next = __next__ + + def send(self, returned): + """Appends a value to self.urls when activated""" + if returned is not None: + print("Value {0} sent to generator".format(returned)) + self.urls.append(returned) + +With this generator created you can now assign it as the ``inputs`` to a ``NetTestCase`` and ``send`` values back to it. + +:: + class TestUrlList(nettest.NetTestCase): + + # Adding custom generator here + inputs = UrlGeneratorWithSend() + + def postProcessor(self, measurements): + """If any HTTPS url's are passed send back an HTTP url.""" + if re.match("^https", self.input): + http_version = re.sub("https", "http", self.input, 1) + self.inputs.send(http_version) + return self.report + + def test_url(self): + self.report['tested'] = [self.input] + return defer.succeed(1) + + Setup and command line passing ------------------------------
diff --git a/ooni/nettest.py b/ooni/nettest.py index 8b9556e..35fc2ff 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -585,10 +585,13 @@ class NetTest(object): """
for test_class, test_methods in self.testCases: - # load the input processor as late as possible - for input in test_class.inputs: + # load a singular input processor for all instances + all_inputs = test_class.inputs + for test_input in all_inputs: measurements = [] test_instance = test_class() + # Set each instances inputs to a singular input processor + test_instance.inputs = all_inputs test_instance._setUp() test_instance.summary = self.summary for method in test_methods: @@ -596,7 +599,7 @@ class NetTest(object): measurement = self.makeMeasurement( test_instance, method, - input) + test_input) measurements.append(measurement.done) self.state.taskCreated() yield measurement @@ -721,7 +724,6 @@ class NetTestCase(object): It gets called once for every input. """ self.report = {} - self.inputs = None
def requirements(self): """ diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index da4969f..55134d3 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -1,4 +1,5 @@ import os +import yaml from tempfile import mkstemp
from twisted.trial import unittest @@ -147,6 +148,65 @@ class HTTPBasedTest(httpt.HTTPTest): use_tor=False) """
+generator_net_test = """ +from twisted.python import usage +from ooni.nettest import NetTestCase + +class UsageOptions(usage.Options): + optParameters = [['spam', 's', None, 'ham']] + +def input_generator(): + # Generates a list of numbers + # The first value sent back is appended to the list. + received = False + numbers = [i for i in range(10)] + while numbers: + i = numbers.pop() + result = yield i + # Place sent value back in numbers + if result is not None and received is False: + numbers.append(result) + received = True + yield i + +class TestSendGen(NetTestCase): + usageOptions = UsageOptions + inputs = input_generator() + + def test_input_sent_to_generator(self): + # Sends a single value back to the generator + if self.input == 5: + self.inputs.send(self.input) +""" + +generator_id_net_test = """ +from twisted.python import usage +from ooni.nettest import NetTestCase + +class UsageOptions(usage.Options): + optParameters = [['spam', 's', None, 'ham']] + +class DummyTestCaseA(NetTestCase): + + usageOptions = UsageOptions + + def test_a(self): + self.report.setdefault("results", []).append(id(self.inputs)) + + def test_b(self): + self.report.setdefault("results", []).append(id(self.inputs)) + + def test_c(self): + self.report.setdefault("results", []).append(id(self.inputs)) + +class DummyTestCaseB(NetTestCase): + + usageOptions = UsageOptions + + def test_a(self): + self.report.setdefault("results", []).append(id(self.inputs)) +""" + dummyInputs = range(1) dummyArgs = ('--spam', 'notham') dummyOptions = {'spam': 'notham'} @@ -308,6 +368,55 @@ class TestNetTest(unittest.TestCase): for test_class, methods in ntl.getTestCases(): self.assertTrue(test_class.requiresRoot)
+ def test_singular_input_processor(self): + """ + Verify that all measurements use the same object as their input processor. + """ + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(generator_id_net_test) + ntl.checkOptions() + + director = Director() + self.filename = 'dummy_report.yamloo' + d = director.startNetTest(ntl, self.filename) + + @d.addCallback + def complete(result): + with open(self.filename) as report_file: + all_report_entries = yaml.safe_load_all(report_file) + header = all_report_entries.next() + results_case_a = all_report_entries.next() + aa_test, ab_test, ac_test = results_case_a.get('results', []) + results_case_b = all_report_entries.next() + ba_test = results_case_b.get('results', [])[0] + # Within a NetTestCase an inputs object will be consistent + self.assertEqual(aa_test, ab_test, ac_test) + # An inputs object will be different between different NetTestCases + self.assertNotEqual(aa_test, ba_test) + + return d + + def test_send_to_inputs_generator(self): + """ + Verify that a net test can send information back into an inputs generator. + """ + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(generator_net_test) + ntl.checkOptions() + + director = Director() + self.filename = 'dummy_report.yamloo' + d = director.startNetTest(ntl, self.filename) + + @d.addCallback + def complete(result): + with open(self.filename) as report_file: + all_report_entries = yaml.safe_load_all(report_file) + header = all_report_entries.next() + results = [x['input'] for x in all_report_entries] + self.assertEqual(results, [9, 8, 7, 6, 5, 5, 3, 2, 1, 0]) + + return d
class TestNettestTimeout(ConfigTestCase):