[tor-commits] [ooni-probe/master] Make inputs a shared iterator across all NetTestCase measurements
art at torproject.org
art at torproject.org
Tue Jun 21 12:14:12 UTC 2016
commit b391f1dc286d318178d0c0b622748b7b604d45d0
Author: seamus tuohy <code at seamustuohy.com>
Date: Sun Jun 5 13:01:24 2016 -0400
Make inputs a shared iterator across all NetTestCase measurements
This commit makes the inputs iterator shared by all the measurement
tests within a NetTestCase. Per issue 503, In order to make bisection
style testing functional the individual tests need to be able to pass
data to the main test_case.inputs generator that seeds each
measurement. This provides opportunities for each measurement test in
a NetTestCase to communicate with the iterator to do things such as
adding additional values to be passed to measurement tests as a later
input.
---
docs/source/writing_tests.rst | 61 ++++++++++++++++++++++-
ooni/nettest.py | 10 ++--
ooni/tests/test_nettest.py | 109 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 174 insertions(+), 6 deletions(-)
diff --git a/docs/source/writing_tests.rst b/docs/source/writing_tests.rst
index 2132010..aaa4adf 100644
--- a/docs/source/writing_tests.rst
+++ b/docs/source/writing_tests.rst
@@ -57,14 +57,71 @@ this::
yield x.strip()
fp.close()
-For example, if you wanted to modify inputProcessor to read enteries from a CSV file, you could use::
-
+For example, if you wanted to modify inputProcessor to read entries from a CSV file, you could use::
+
def inputProcessor(self, filename):
with open(filename) as csvFile:
reader = DictReader(csvFile)
for entry in reader:
yield entry
+
+The ``inputs`` iterator is unique per ``NetTestCase`` and shared by all its measurement tests. This provides opportunities for each measurement test in a ``NetTestCase`` to communicate with the iterator to do things such as adding additional values to be passed to measurement tests as a later ``input``.
+
+.. note :: Deleting/removing the current item from the ``inputs`` iterator will not stop other measurement tests from operating on that ``input``. When a ``NetTestCase`` is run a single ``input`` is taken from the ``inputs`` iterator by the OONI test loader and run against all of the individual measurement tests within that ``NetTestCase``. Removing an ``input`` from the ``inputs`` iterator during a measurement will not stop that input from being called on all other measurement tests within the NetTestCase.
+
+Here is one example of how you can take advantage of shared ``inputs`` iterator when writing your own OONI tests. If you have a list of urls and you want to make sure that you always test the HTTP equivalent of any HTTPS urls provided you can ``send`` values back to a custom generator in your ``postProcessor``.
+
+To do this the first thing you would need to create is a URL generator that can accept values sent to it.
+
+::
+ class UrlGeneratorWithSend(object):
+ def __init__(self):
+ """Create initial list and set generator state."""
+ self.urls = ["http://www.torproject.org",
+ "https://ooni.torproject.org"]
+ self.current = 0
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ try:
+ cur = self.urls[self.current]
+ self.current += 1
+ return cur
+ except IndexError:
+ raise StopIteration
+
+ # Python 2 & 3 generator compatibility
+ next = __next__
+
+ def send(self, returned):
+ """Appends a value to self.urls when activated"""
+ if returned is not None:
+ print("Value {0} sent to generator".format(returned))
+ self.urls.append(returned)
+
+With this generator created you can now assign it as the ``inputs`` to a ``NetTestCase`` and ``send`` values back to it.
+
+::
+ class TestUrlList(nettest.NetTestCase):
+
+ # Adding custom generator here
+ inputs = UrlGeneratorWithSend()
+
+ def postProcessor(self, measurements):
+ """If any HTTPS url's are passed send back an HTTP url."""
+ if re.match("^https", self.input):
+ http_version = re.sub("https", "http", self.input, 1)
+ self.inputs.send(http_version)
+ return self.report
+
+ def test_url(self):
+ self.report['tested'] = [self.input]
+ return defer.succeed(1)
+
+
Setup and command line passing
------------------------------
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 8b9556e..35fc2ff 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -585,10 +585,13 @@ class NetTest(object):
"""
for test_class, test_methods in self.testCases:
- # load the input processor as late as possible
- for input in test_class.inputs:
+ # load a singular input processor for all instances
+ all_inputs = test_class.inputs
+ for test_input in all_inputs:
measurements = []
test_instance = test_class()
+ # Set each instances inputs to a singular input processor
+ test_instance.inputs = all_inputs
test_instance._setUp()
test_instance.summary = self.summary
for method in test_methods:
@@ -596,7 +599,7 @@ class NetTest(object):
measurement = self.makeMeasurement(
test_instance,
method,
- input)
+ test_input)
measurements.append(measurement.done)
self.state.taskCreated()
yield measurement
@@ -721,7 +724,6 @@ class NetTestCase(object):
It gets called once for every input.
"""
self.report = {}
- self.inputs = None
def requirements(self):
"""
diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py
index da4969f..55134d3 100644
--- a/ooni/tests/test_nettest.py
+++ b/ooni/tests/test_nettest.py
@@ -1,4 +1,5 @@
import os
+import yaml
from tempfile import mkstemp
from twisted.trial import unittest
@@ -147,6 +148,65 @@ class HTTPBasedTest(httpt.HTTPTest):
use_tor=False)
"""
+generator_net_test = """
+from twisted.python import usage
+from ooni.nettest import NetTestCase
+
+class UsageOptions(usage.Options):
+ optParameters = [['spam', 's', None, 'ham']]
+
+def input_generator():
+ # Generates a list of numbers
+ # The first value sent back is appended to the list.
+ received = False
+ numbers = [i for i in range(10)]
+ while numbers:
+ i = numbers.pop()
+ result = yield i
+ # Place sent value back in numbers
+ if result is not None and received is False:
+ numbers.append(result)
+ received = True
+ yield i
+
+class TestSendGen(NetTestCase):
+ usageOptions = UsageOptions
+ inputs = input_generator()
+
+ def test_input_sent_to_generator(self):
+ # Sends a single value back to the generator
+ if self.input == 5:
+ self.inputs.send(self.input)
+"""
+
+generator_id_net_test = """
+from twisted.python import usage
+from ooni.nettest import NetTestCase
+
+class UsageOptions(usage.Options):
+ optParameters = [['spam', 's', None, 'ham']]
+
+class DummyTestCaseA(NetTestCase):
+
+ usageOptions = UsageOptions
+
+ def test_a(self):
+ self.report.setdefault("results", []).append(id(self.inputs))
+
+ def test_b(self):
+ self.report.setdefault("results", []).append(id(self.inputs))
+
+ def test_c(self):
+ self.report.setdefault("results", []).append(id(self.inputs))
+
+class DummyTestCaseB(NetTestCase):
+
+ usageOptions = UsageOptions
+
+ def test_a(self):
+ self.report.setdefault("results", []).append(id(self.inputs))
+"""
+
dummyInputs = range(1)
dummyArgs = ('--spam', 'notham')
dummyOptions = {'spam': 'notham'}
@@ -308,6 +368,55 @@ class TestNetTest(unittest.TestCase):
for test_class, methods in ntl.getTestCases():
self.assertTrue(test_class.requiresRoot)
+ def test_singular_input_processor(self):
+ """
+ Verify that all measurements use the same object as their input processor.
+ """
+ ntl = NetTestLoader(dummyArgs)
+ ntl.loadNetTestString(generator_id_net_test)
+ ntl.checkOptions()
+
+ director = Director()
+ self.filename = 'dummy_report.yamloo'
+ d = director.startNetTest(ntl, self.filename)
+
+ @d.addCallback
+ def complete(result):
+ with open(self.filename) as report_file:
+ all_report_entries = yaml.safe_load_all(report_file)
+ header = all_report_entries.next()
+ results_case_a = all_report_entries.next()
+ aa_test, ab_test, ac_test = results_case_a.get('results', [])
+ results_case_b = all_report_entries.next()
+ ba_test = results_case_b.get('results', [])[0]
+ # Within a NetTestCase an inputs object will be consistent
+ self.assertEqual(aa_test, ab_test, ac_test)
+ # An inputs object will be different between different NetTestCases
+ self.assertNotEqual(aa_test, ba_test)
+
+ return d
+
+ def test_send_to_inputs_generator(self):
+ """
+ Verify that a net test can send information back into an inputs generator.
+ """
+ ntl = NetTestLoader(dummyArgs)
+ ntl.loadNetTestString(generator_net_test)
+ ntl.checkOptions()
+
+ director = Director()
+ self.filename = 'dummy_report.yamloo'
+ d = director.startNetTest(ntl, self.filename)
+
+ @d.addCallback
+ def complete(result):
+ with open(self.filename) as report_file:
+ all_report_entries = yaml.safe_load_all(report_file)
+ header = all_report_entries.next()
+ results = [x['input'] for x in all_report_entries]
+ self.assertEqual(results, [9, 8, 7, 6, 5, 5, 3, 2, 1, 0])
+
+ return d
class TestNettestTimeout(ConfigTestCase):
More information about the tor-commits
mailing list