March 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Work with aagbsn to make the TaskManager pass all tests
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 8fc8056fa1796c3f5d096d98076aa9d0bbbe47d9 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 12 16:56:47 2013 +0100 Work with aagbsn to make the TaskManager pass all tests Tests will retry to run until the retry limit has been reached Test failures are properly kept track of --- ooni/managers.py | 35 +++++----- ooni/tasks.py | 19 ++++-- tests/test_managers.py | 174 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 205 insertions(+), 23 deletions(-) diff --git a/ooni/managers.py b/ooni/managers.py index 26ac3f9..be11473 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -18,14 +18,11 @@ class TaskManager(object): failures = [] concurrency = 10 + completedTasks = 0 + _tasks = iter(()) _active_tasks = [] - def _tasksDone(self): - self.tasksDone.callback(None) - - self.tasksDone = defer.Deferred() - def _failed(self, failure, task): """ The has failed to complete, we append it to the end of the task chain @@ -34,14 +31,16 @@ class TaskManager(object): self._active_tasks.remove(task) self.failures.append((failure, task)) - if task.failures < self.retries: + if task.failures <= self.retries: self._tasks = itertools.chain(self._tasks, makeIterable(task)) - - self._fillSlots() + else: + task.done.callback((failure, task)) self.failed(failure, task) + self._fillSlots() + def _fillSlots(self): """ Called on test completion and schedules measurements to be run for the @@ -52,10 +51,9 @@ class TaskManager(object): task = self._tasks.next() self._run(task) except StopIteration: - self._tasksDone() break - def _suceeded(self, result, task): + def _succeeded(self, result, task): """ We have successfully completed a measurement. """ @@ -64,7 +62,9 @@ class TaskManager(object): self._fillSlots() - self.suceeded(result, task) + task.done.callback(result) + + self.succeeded(result, task) def _run(self, task): """ @@ -73,9 +73,9 @@ class TaskManager(object): """ self._active_tasks.append(task) - d = task.run() - d.addCallback(self.succeeded, task) - d.addErrback(self.failed, task) + d = task.start() + d.addCallback(self._succeeded, task) + d.addErrback(self._failed, task) @property def failedMeasurements(self): @@ -93,13 +93,14 @@ class TaskManager(object): Takes as argument a single task or a task iterable and appends it to the task generator queue. """ - iterable = makeIterable(task_or_task_iterator) self._tasks = itertools.chain(self._tasks, iterable) self._fillSlots() def start(self): + self.failures = [] + self.tasksDone = defer.Deferred() self._fillSlots() @@ -135,7 +136,7 @@ class MeasurementsManager(TaskManager): director = None - def suceeded(self, result, measurement): + def succeeded(self, result, measurement): pass def failed(self, failure, measurement): @@ -192,7 +193,7 @@ class ReportEntryManager(object): def initializeTaskList(self): pass - def suceeded(self, result, measurement): + def succeeded(self, result, measurement): pass def failed(self, failure, measurement): diff --git a/ooni/tasks.py b/ooni/tasks.py index 909f3e5..55f09bd 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -1,24 +1,31 @@ +from twisted.internet import defer + class BaseTask(object): _timer = None def __init__(self): self.running = False self.failures = 0 + # This is a deferred that gets called when a test has reached it's + # final status, this means: all retries have been attempted or the test + # has successfully executed. + self.done = defer.Deferred() def _failed(self, failure): self.failures += 1 self.failed(failure) - return + return failure - def _run(self): + def _succeeded(self, result): + self.succeeded(result) + return result + + def start(self): d = self.run() d.addErrback(self._failed) d.addCallback(self._succeeded) return d - def _succeeded(self, result): - self.succeeded(result) - def succeeded(self, result): """ Place here the logic to handle a successful execution of the task. @@ -113,7 +120,7 @@ class Measurement(TaskWithTimeout): d.addErrback(self.failure) return d -class ReportEntry(TimedOutTask): +class ReportEntry(TaskWithTimeout): def __init__(self, reporter, measurement): self.reporter = reporter self.measurement = measurement diff --git a/tests/test_managers.py b/tests/test_managers.py new file mode 100644 index 0000000..59eba5f --- /dev/null +++ b/tests/test_managers.py @@ -0,0 +1,174 @@ +from twisted.trial import unittest +from twisted.python import failure +from twisted.internet import defer + +from ooni.tasks import BaseTask +from ooni.managers import TaskManager + + +mockFailure = failure.Failure(Exception('mock')) + +class MockSuccessTask(BaseTask): + def run(self): + return defer.succeed(42) + +class MockFailTask(BaseTask): + def run(self): + return defer.fail(mockFailure) + +class MockFailOnceTask(BaseTask): + def run(self): + if self.failures >= 1: + return defer.succeed(42) + else: + return defer.fail(mockFailure) + +class MockTaskManager(TaskManager): + def __init__(self): + self.successes = [] + + def failed(self, failure, task): + # print "TASK" + # print task + # print "FAILURES (%s)" % task.failures + # print failure + pass + + def succeeded(self, result, task): + self.successes.append((result, task)) + +class TestTaskManager(unittest.TestCase): + def setUp(self): + self.taskManager = MockTaskManager() + self.taskManager.concurrency = 10 + self.taskManager.retries = 2 + + self.taskManager.start() + + def tearDown(self): + pass + + def test_schedule_successful_one_task(self): + mock_task = MockSuccessTask() + self.taskManager.schedule(mock_task) + + @mock_task.done.addCallback + def done(res): + self.assertEqual(self.taskManager.successes, + [(42, mock_task)]) + return mock_task.done + + def test_schedule_failing_one_task(self): + mock_task = MockFailTask() + self.taskManager.schedule(mock_task) + + @mock_task.done.addCallback + def done(failure): + self.assertEqual(len(self.taskManager.failures), 3) + + self.assertEqual(failure, (mockFailure, mock_task)) + + return mock_task.done + + def test_schedule_successful_ten_tasks(self): + all_done = [] + for x in range(10): + mock_task = MockSuccessTask() + all_done.append(mock_task.done) + self.taskManager.schedule(mock_task) + + d = defer.DeferredList(all_done) + @d.addCallback + def done(res): + for task_result, task_instance in self.taskManager.successes: + self.assertEqual(task_result, 42) + self.assertIsInstance(task_instance, MockSuccessTask) + + return d + + def test_schedule_failing_ten_tasks(self): + all_done = [] + for x in range(10): + mock_task = MockFailTask() + all_done.append(mock_task.done) + self.taskManager.schedule(mock_task) + + d = defer.DeferredList(all_done) + @d.addCallback + def done(res): + # 10*2 because 2 is the number of retries + self.assertEqual(len(self.taskManager.failures), 10*3) + for task_result, task_instance in self.taskManager.failures: + self.assertEqual(task_result, mockFailure) + self.assertIsInstance(task_instance, MockFailTask) + + return d + + def test_schedule_successful_27_tasks(self): + all_done = [] + for x in range(27): + mock_task = MockSuccessTask() + all_done.append(mock_task.done) + self.taskManager.schedule(mock_task) + + d = defer.DeferredList(all_done) + @d.addCallback + def done(res): + for task_result, task_instance in self.taskManager.successes: + self.assertEqual(task_result, 42) + self.assertIsInstance(task_instance, MockSuccessTask) + + return d + + def test_schedule_failing_27_tasks(self): + all_done = [] + for x in range(27): + mock_task = MockFailTask() + all_done.append(mock_task.done) + self.taskManager.schedule(mock_task) + + d = defer.DeferredList(all_done) + @d.addCallback + def done(res): + # 10*2 because 2 is the number of retries + self.assertEqual(len(self.taskManager.failures), 27*3) + for task_result, task_instance in self.taskManager.failures: + self.assertEqual(task_result, mockFailure) + self.assertIsInstance(task_instance, MockFailTask) + + return d + + + def test_task_retry_and_succeed(self): + mock_task = MockFailOnceTask() + self.taskManager.schedule(mock_task) + + @mock_task.done.addCallback + def done(res): + self.assertEqual(len(self.taskManager.failures), 1) + + self.assertEqual(self.taskManager.failures, + [(mockFailure, mock_task)]) + self.assertEqual(self.taskManager.successes, + [(42, mock_task)]) + + return mock_task.done + + def test_task_retry_and_succeed_56_tasks(self): + all_done = [] + for x in range(56): + mock_task = MockFailOnceTask() + all_done.append(mock_task.done) + self.taskManager.schedule(mock_task) + + d = defer.DeferredList(all_done) + @d.addCallback + def done(res): + self.assertEqual(len(self.taskManager.failures), 56) + + for task_result, task_instance in self.taskManager.successes: + self.assertEqual(task_result, 42) + self.assertIsInstance(task_instance, MockFailOnceTask) + + return d +

1 0

[ooni-probe/master] Delete measurements module
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit ca24885eeea83c023e79eecedb139db21cbc1cd9 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 12 18:30:50 2013 +0100 Delete measurements module --- ooni/measurements.py | 161 -------------------------------------------------- 1 files changed, 0 insertions(+), 161 deletions(-) diff --git a/ooni/measurements.py b/ooni/measurements.py deleted file mode 100644 index 86d6f84..0000000 --- a/ooni/measurements.py +++ /dev/null @@ -1,161 +0,0 @@ -from .tasks import TaskWithTimeout -from inspect import getmembers -from ooni.nettest import NetTestCase -from ooni.ratelimiting import StaticRateLimiter -from twisted.python.reflect import prefixedMethodNames -from twisted.trial.runner import filenameToModule -from StringIO import StringIO -from os.path import isfile - -class Measurement(TaskWithTimeout): - def __init__(self, test_class, test_method, test_input, net_test): - """ - test_class: - is the class, subclass of NetTestCase, of the test to be run - - test_method: - is a string representing the test method to be called to perform - this measurement - - test_input: - is the input to the test - """ - self.test_instance = test_class() - self.test_instance.input = test_input - self.test_instance.report = {} - self.test_instance._start_time = time.time() - self.test_instance._setUp() - self.test_instance.setUp() - self.test = getattr(self.test_instance, test_method) - - def succeeded(self): - self.net_test.measurementSuccess() - - def failed(self): - self.net_test.measurementFailed() - - def timedOut(self): - self.net_test.measurementTimeOut() - - def run(self): - d = defer.maybeDeferred(self.test) - d.addCallback(self.succeeded) - d.addErrback(self.failed) - return d - -class NetTest(object): - director = None - method_prefix = 'test' - - def __init__(self, net_test_file, inputs, options, report): - """ - net_test_file: - is a file object containing the test to be run. - - inputs: - is a generator containing the inputs to the net test. - - options: - is a dict containing the options to be passed to the net test. - """ - self.test_cases = self.loadNetTest(net_test_file) - self.inputs = inputs - self.options = options - - def loadNetTest(self, net_test_object): - """ - Creates all the necessary test_cases (a list of tuples containing the - NetTestCase (test_class, test_method)) - - example: - [(test_classA, test_method1), - (test_classA, test_method2), - (test_classA, test_method3), - (test_classA, test_method4), - (test_classA, test_method5), - - (test_classB, test_method1), - (test_classB, test_method2)] - - Note: the inputs must be valid for test_classA and test_classB. - - net_test_object: - is a file like object that will be used to generate the test_cases. - """ - try: - if isfile(net_test_object): - return self._loadNetTestFile(net_test_object) - except TypeError: - if isinstance(net_test_object, StringIO) or \ - isinstance(net_test_object, str): - return self._loadNetTestString(net_test_object) - - def _loadNetTestString(self, net_test_string): - """ - Load NetTest from a string - """ - ns = {} - test_cases = [] - exec net_test_string.read() in ns - for item in ns.itervalues(): - test_cases.extend(self._get_test_methods(item)) - return test_cases - - def _loadNetTestFile(self, net_test_file): - """ - Load NetTest from a file - """ - test_cases = [] - module = filenameToModule(net_test_file) - for __, item in getmembers(module): - test_cases.extend(self._get_test_methods(item)) - return test_cases - - def _get_test_methods(self, item): - """ - Look for test_ methods in subclasses of NetTestCase - """ - test_cases = [] - try: - assert issubclass(item, NetTestCase) - methods = prefixedMethodNames(item, self.method_prefix) - for method in methods: - test_cases.append((item, self.method_prefix + method)) - except (TypeError, AssertionError): - pass - return test_cases - - def timedOut(self, measurement): - """ - This gets called when a measurement has timed out. This may or may not - trigger a retry inside of MeasurementsTracker. - """ - self.director.measurementTimedOut(measurement) - - def failed(self, measurement, failure): - """ - This gets called when a measurement has failed in the sense that all - the retries have failed at successfully running the test. - This means that it's a definitive failure and we will no longer make - any attempts at re-running the measurement. - """ - self.director.measurementFailed(measurement, failure) - - def succeeded(self, measurement): - """ - This gets called when a measurement has failed. - """ - self.report.write(measurement) - - def generateMeasurements(self): - """ - This is a generator that yields measurements and sets their timeout - value and their netTest attribute. - """ - for test_input in self.inputs: - for test_class, test_method in self.test_cases: - measurement = Measurement(test_class, test_method, test_input) - measurement.netTest = self - measurement.timeout = self.rateLimiter.timeout - yield measurement -

1 0

[ooni-probe/master] Rename test_manager to test_nettest
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 84965ad6fb2235119012c502c555850eb967d342 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 12 18:49:32 2013 +0100 Rename test_manager to test_nettest Remove duplicated tests --- tests/test_manager.py | 167 ------------------------------------------------- tests/test_nettest.py | 92 +++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 167 deletions(-) diff --git a/tests/test_manager.py b/tests/test_manager.py deleted file mode 100644 index 323aa1f..0000000 --- a/tests/test_manager.py +++ /dev/null @@ -1,167 +0,0 @@ -from ooni.measurements import Measurement, NetTest -from ooni.managers import Director, MeasurementsManager -from ooni.tasks import BaseTask, TaskWithTimeout -from os import unlink -from StringIO import StringIO -from tempfile import TemporaryFile, mkstemp -from twisted.trial import unittest -from twisted.internet import defer, reactor - -net_test_string = """ -from twisted.python import usage -from ooni.nettest import NetTestCase - -class UsageOptions(usage.Options): - optParameters = [['spam', 's', 'ham']] - -class DummyTestCase(NetTestCase): - usageOptions = UsageOptions - - def test_a(self): - self.report['bar'] = 'bar' - - def test_b(self): - self.report['foo'] = 'foo' -""" - -dummyInputs = range(1) -dummyOptions = {'spam': 'notham'} - -#dummyNetTest = NetTest(net_test_file, inputs, options) - -class DummyMeasurement(BaseTask): - def run(self): - f = open('foo.txt', 'w') - f.write('testing') - f.close() - - return defer.succeed(self) - -class DummyMeasurementFailOnce(BaseTask): - def run(self): - f = open('dummyTaskFailOnce.txt', 'w') - f.write('fail') - f.close() - if self.failure >= 1: - return defer.succeed() - else: - return defer.fail() - -class DummyNetTest(NetTest): - def __init__(self, num_measurements=1): - NetTest.__init__(self, StringIO(net_test_string), dummyInputs, dummyOptions) - self.num_measurements = num_measurements - def generateMeasurements(self): - for i in range(self.num_measurements): - yield DummyMeasurement() - -class DummyDirector(object): - def __init__(self): - pass - -class DummyReporter(object): - def __init__(self): - pass - def write(self, result): - pass - -class TestNetTest(unittest.TestCase): - def test_load_net_test_from_file(self): - """ - Given a file like object verify that the net test cases are properly - generated. - """ - __, net_test_file = mkstemp() - with open(net_test_file, 'w') as f: - f.write(net_test_string) - f.close() - - net_test_from_string = NetTest(StringIO(net_test_string), - dummyInputs, dummyOptions, DummyReporter()) - net_test_from_file = NetTest(net_test_file, dummyInputs, - dummyOptions, DummyReporter()) - - # XXX: the returned classes are not the same because the - # module path is not correct, so the test below fails. - # TODO: figure out how to verify that the instantiated - # classes are done so properly. - - #self.assertEqual(net_test_from_string.test_cases, - # net_test_from_file.test_cases) - unlink(net_test_file) - -class TestMeasurementsManager(unittest.TestCase): - def setUp(self): - self.mock_mm = MeasurementsManager() - self.mock_mm.director = DummyDirector() - - def test_schedule_measurement(self): - # testing schedule() - # run a single measurement - measurement = DummyMeasurement() - self.mock_mm.concurrency = 1 - self.mock_mm.start() - self.mock_mm.schedule(measurement) - d = self.mock_mm.run(measurement) - def f(x): - self.assertEqual(self.mock_mm.completedMeasurements, 1) - d.addCallback(f) - return d - - def test_all_slots_full(self): - """ - Test case where active_measurements is full - """ - self.mock_mm.concurrency = 1 - self.mock_mm.start() - - dl = [] - for x in xrange(2): - measurement = DummyMeasurement() - dl.append(self.mock_mm.schedule(measurement)) - - def f(z): print self.mock_mm.completedMeasurements - - assert dl is not None - d = defer.DeferredList(dl) - assert d is not None - d.addCallback(f) - return d - - def test_populate_active_measurements(self): - """ - Test that populates the full set of active measurements - """ - pass - - def test_fail_and_reschedule(self): - """docstring for test_fail_and_reschedule""" - pass - - def test_fail_timeout_and_reschedule(self): - pass - - def test_all_completed(self): - """ - all inputs have been consumed and all tests have a final status. - """ - pass - - def test_measurements_exhausted(self): - """ - inputs consumed, but still have running measurements - """ - pass - -class TestDirector(unittest.TestCase): - def setUp(self): - self.director = Director() - - def test_successful_measurement(self): - pass - - def test_failing_measurment(self): - pass - - def test_retry_twice_measurement(self): - pass diff --git a/tests/test_nettest.py b/tests/test_nettest.py new file mode 100644 index 0000000..e23e612 --- /dev/null +++ b/tests/test_nettest.py @@ -0,0 +1,92 @@ +from ooni.measurements import Measurement, NetTest +from ooni.managers import Director, MeasurementsManager +from ooni.tasks import BaseTask, TaskWithTimeout +from os import unlink +from StringIO import StringIO +from tempfile import TemporaryFile, mkstemp +from twisted.trial import unittest +from twisted.internet import defer, reactor + +net_test_string = """ +from twisted.python import usage +from ooni.nettest import NetTestCase + +class UsageOptions(usage.Options): + optParameters = [['spam', 's', 'ham']] + +class DummyTestCase(NetTestCase): + usageOptions = UsageOptions + + def test_a(self): + self.report['bar'] = 'bar' + + def test_b(self): + self.report['foo'] = 'foo' +""" + +dummyInputs = range(1) +dummyOptions = {'spam': 'notham'} + +#dummyNetTest = NetTest(net_test_file, inputs, options) + +class DummyMeasurement(BaseTask): + def run(self): + f = open('foo.txt', 'w') + f.write('testing') + f.close() + + return defer.succeed(self) + +class DummyMeasurementFailOnce(BaseTask): + def run(self): + f = open('dummyTaskFailOnce.txt', 'w') + f.write('fail') + f.close() + if self.failure >= 1: + return defer.succeed() + else: + return defer.fail() + +class DummyNetTest(NetTest): + def __init__(self, num_measurements=1): + NetTest.__init__(self, StringIO(net_test_string), dummyInputs, dummyOptions) + self.num_measurements = num_measurements + def generateMeasurements(self): + for i in range(self.num_measurements): + yield DummyMeasurement() + +class DummyDirector(object): + def __init__(self): + pass + +class DummyReporter(object): + def __init__(self): + pass + def write(self, result): + pass + +class TestNetTest(unittest.TestCase): + def test_load_net_test_from_file(self): + """ + Given a file like object verify that the net test cases are properly + generated. + """ + __, net_test_file = mkstemp() + with open(net_test_file, 'w') as f: + f.write(net_test_string) + f.close() + + net_test_from_string = NetTest(StringIO(net_test_string), + dummyInputs, dummyOptions, DummyReporter()) + net_test_from_file = NetTest(net_test_file, dummyInputs, + dummyOptions, DummyReporter()) + + # XXX: the returned classes are not the same because the + # module path is not correct, so the test below fails. + # TODO: figure out how to verify that the instantiated + # classes are done so properly. + + #self.assertEqual(net_test_from_string.test_cases, + # net_test_from_file.test_cases) + unlink(net_test_file) +

1 0

[ooni-probe/master] Fix imports in nettest unittest
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 232a208152173cadcdd68f852c0a6254dec59481 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 12 18:52:21 2013 +0100 Fix imports in nettest unittest --- tests/test_nettest.py | 11 ++++++----- 1 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_nettest.py b/tests/test_nettest.py index e23e612..af1f8d0 100644 --- a/tests/test_nettest.py +++ b/tests/test_nettest.py @@ -1,12 +1,13 @@ -from ooni.measurements import Measurement, NetTest -from ooni.managers import Director, MeasurementsManager -from ooni.tasks import BaseTask, TaskWithTimeout -from os import unlink +import os from StringIO import StringIO from tempfile import TemporaryFile, mkstemp + from twisted.trial import unittest from twisted.internet import defer, reactor +from ooni.nettest import NetTest +from ooni.tasks import BaseTask + net_test_string = """ from twisted.python import usage from ooni.nettest import NetTestCase @@ -88,5 +89,5 @@ class TestNetTest(unittest.TestCase): #self.assertEqual(net_test_from_string.test_cases, # net_test_from_file.test_cases) - unlink(net_test_file) + os.unlink(net_test_file)

1 0

[ooni-probe/master] Minor cleaning of Measurement Task
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit fad04d7b705cfe2b70e571099f02d37b9acaed80 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 12 19:34:31 2013 +0100 Minor cleaning of Measurement Task --- ooni/tasks.py | 7 ++----- 1 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ooni/tasks.py b/ooni/tasks.py index 55f09bd..a53287e 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -106,7 +106,7 @@ class Measurement(TaskWithTimeout): self.netTest = net_test def succeeded(self): - self.net_test.succeeded() + self.net_test.succeeded(self) def failed(self): pass @@ -115,10 +115,7 @@ class Measurement(TaskWithTimeout): self.net_test.timedOut() def run(self): - d = defer.maybeDeferred(self.test) - d.addCallback(self.success) - d.addErrback(self.failure) - return d + return defer.maybeDeferred(self.test) class ReportEntry(TaskWithTimeout): def __init__(self, reporter, measurement):

1 0

[ooni-probe/master] Check to see if the test methods are callables
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 606df6ea2010a2d0a172149bbbfd263e58d67cb7 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 12 19:06:30 2013 +0100 Check to see if the test methods are callables --- tests/test_nettest.py | 40 +++++++++++++++++++++++++++++++--------- 1 files changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/test_nettest.py b/tests/test_nettest.py index af1f8d0..8546031 100644 --- a/tests/test_nettest.py +++ b/tests/test_nettest.py @@ -67,9 +67,12 @@ class DummyReporter(object): pass class TestNetTest(unittest.TestCase): + def assertCallable(self, thing): + self.assertIn('__call__', dir(thing)) + def test_load_net_test_from_file(self): """ - Given a file like object verify that the net test cases are properly + Given a file verify that the net test cases are properly generated. """ __, net_test_file = mkstemp() @@ -77,17 +80,36 @@ class TestNetTest(unittest.TestCase): f.write(net_test_string) f.close() - net_test_from_string = NetTest(StringIO(net_test_string), - dummyInputs, dummyOptions, DummyReporter()) net_test_from_file = NetTest(net_test_file, dummyInputs, dummyOptions, DummyReporter()) - # XXX: the returned classes are not the same because the - # module path is not correct, so the test below fails. - # TODO: figure out how to verify that the instantiated - # classes are done so properly. + test_methods = set() + for test_class, test_method in net_test_from_file.test_cases: + instance = test_class() + c = getattr(instance, test_method) + self.assertCallable(c) + + test_methods.add(test_method) + + self.assertEqual(set(['test_a', 'test_b']), test_methods) - #self.assertEqual(net_test_from_string.test_cases, - # net_test_from_file.test_cases) os.unlink(net_test_file) + def test_load_net_test_from_string(self): + """ + Given a file like object verify that the net test cases are properly + generated. + """ + net_test_from_string = NetTest(StringIO(net_test_string), + dummyInputs, dummyOptions, DummyReporter()) + + test_methods = set() + for test_class, test_method in net_test_from_string.test_cases: + instance = test_class() + c = getattr(instance, test_method) + self.assertCallable(c) + + test_methods.add(test_method) + + self.assertEqual(set(['test_a', 'test_b']), test_methods) +

1 0

[ooni-probe/master] add processTest to NetTest class
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit f7a6a5229c859aa2cc66f307c0b2363ca00346db Author: aagbsn <aagbsn(a)extc.org> Date: Sat Jan 12 20:35:19 2013 +0000 add processTest to NetTest class ripped out of runner.py and needs testing --- ooni/nettest.py | 98 ++++++++++++++++++++++++++++++++++++++++-- ooni/nettesttask.py | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+), 4 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index ecf7765..511c3da 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -30,6 +30,12 @@ class NetTest(object): self.report = report + def start(self): + """ + Start tests and generate measurements. + """ + raise NotImplementedError + def loadNetTest(self, net_test_object): """ Creates all the necessary test_cases (a list of tuples containing the @@ -52,11 +58,16 @@ class NetTest(object): """ try: if os.path.isfile(net_test_object): - return self._loadNetTestFile(net_test_object) + test_cases = self._loadNetTestFile(net_test_object) except TypeError: if isinstance(net_test_object, StringIO) or \ isinstance(net_test_object, str): - return self._loadNetTestString(net_test_object) + test_cases = self._loadNetTestString(net_test_object) + + if not test_cases: + raise NoTestCasesFound + + return test_cases def _loadNetTestString(self, net_test_string): """ @@ -110,8 +121,79 @@ class NetTest(object): measurement.netTest = self yield measurement -class NoPostProcessor(Exception): - pass + def processTestCasesOptions(self): + self.options #XXX is this cmd_line_options? + + # get set of unique classes + test_classes = set([]) + for test_class, test_method in self.test_cases: + test_classes.add(test_class) + + #XXX where should the options bound to a test_class get stashed? + for test_class in test_classes: + options = self._processOptions() + + #XXX: is options passed to init the same as cmd_line_options??? + def _processTest(self, nettest_test_case, cmd_line_options): + """ + Process the parameters and :class:`twisted.python.usage.Options` of a + :class:`ooni.nettest.Nettest`. + + :param obj: + An uninstantiated old test, which should be a subclass of + :class:`ooni.plugoo.tests.OONITest`. + + :param cmd_line_options: + A configured and instantiated :class:`twisted.python.usage.Options` + class. + + """ + obj = nettest_test_case + if not hasattr(obj.usageOptions, 'optParameters'): + obj.usageOptions.optParameters = [] + + if obj.inputFile: + obj.usageOptions.optParameters.append(obj.inputFile) + + if obj.baseParameters: + for parameter in obj.baseParameters: + obj.usageOptions.optParameters.append(parameter) + + if obj.baseFlags: + if not hasattr(obj.usageOptions, 'optFlags'): + obj.usageOptions.optFlags = [] + for flag in obj.baseFlags: + obj.usageOptions.optFlags.append(flag) + + options = obj.usageOptions() + + options.parseOptions(cmd_line_options['subargs']) + obj.localOptions = options + + if obj.inputFile: + obj.inputFilename = options[obj.inputFile[0]] + + try: + log.debug("processing options") + tmp_test_case_object = obj() + tmp_test_case_object._checkRequiredOptions() + + except usage.UsageError, e: + test_name = tmp_test_case_object.name + log.err("There was an error in running %s!" % test_name) + log.err("%s" % e) + options.opt_help() + raise usage.UsageError("Error in parsing command line args for %s" % test_name) + + # who checks for root? + if obj.requiresRoot: + try: + checkForRoot() + except NotRootError: + log.err("%s requires root to run" % obj.name) + sys.exit(1) + + return obj class NetTestCase(object): """ @@ -261,3 +343,11 @@ class NetTestCase(object): def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) +class FailureToLoadNetTest(Exception): + pass +class NoPostProcessor(Exception): + pass +class InvalidOption(Exception): + pass +class MissingRequiredOption(Exception): + pass diff --git a/ooni/nettesttask.py b/ooni/nettesttask.py new file mode 100644 index 0000000..48c5108 --- /dev/null +++ b/ooni/nettesttask.py @@ -0,0 +1,118 @@ +from twisted.internet.task import CooperativeTask +from twisted.internet import defer +from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError +from ooni.utils import log +import time +from twisted.internet.task import cooperate + +class NetTestTask(CooperativeTask): + """ + The object produced by a NetTestTaskFactory. + + A NetTestTask wraps a test_ callable with its options and input unit. + + """ + def __init__(self, test_case, test_input, oonib_reporter=None, yaml_reporter=None): + test_class, test_method = test_case + #log.debug("Running %s with %s..." % (test_method, test_input)) + self.oonib_reporter = oonib_reporter + self.oonib_reporter = yaml_reporter + self.test_instance = test_class() + self.test_instance.input = test_input + self.test_instance.report = {} + self.test_instance._start_time = time.time() + self.test_instance._setUp() + self.test_instance.setUp() + self.test = getattr(self.test_instance, test_method) + + # XXX: override CoordinatedTask methods + def start(self): #??? + d = defer.maybeDeferred(self.test) + d.addCallback(self.test_done) + d.addErrback(self.test_error) + return d + + def write_report(self): + if not self.oonib_reporter: + return self.yaml_reporter.testDone(self.test_instance, str(self.test)) + d1 = self.oonib_reporter.testDone(self.test_instance, str(self.test)) + d2 = self.yaml_reporter.testDone(self.test_instance, str(self.test)) + dl = defer.DeferredList([d1, d2]) + @dl.addErrback + def reportingFailed(failure): + log.err("Error in reporting %s" % self.test) + log.exception(failure) + return dl + + def test_done(self, result): + log.msg("Finished running %s" % self.test) + log.debug("Deferred callback result: %s" % result) + return self.write_report() + + def test_error(self, failure): + log.err("Error in running %s" % self.test) + log.exception(failure) + return self.write_report() + + #XXX: does not implement tests_done! + +class NetTestTaskFactory(object): + def __init__(self, test_cases, input_unit_list): + self.input_unit_list = input_unit_list + self.inputs = self.generate_inputs() + self.test_cases = test_cases + + def __iter__(self): + return self + + def next(self): + return self.inputs.next() + # XXX: raise exception or fire callback when inputs are exhausted + + def generate_inputs(self): + for input_unit in self.input_unit_list: + for test_case in self.test_cases: + yield NetTestTask(test_case, input_unit) + +(a)defer.inlineCallbacks +def runTestCases(test_cases, options, cmd_line_options): + + log.debug("Running %s" % test_cases) + log.debug("Options %s" % options) + log.debug("cmd_line_options %s" % dict(cmd_line_options)) + + test_inputs = options['inputs'] + + oonib_reporter = OONIBReporter(cmd_line_options) + yaml_reporter = YAMLReporter(cmd_line_options) + + if cmd_line_options['collector']: + log.msg("Using remote collector, please be patient while we create the report.") + try: + yield oonib_reporter.createReport(options) + except OONIBReportError: + log.err("Error in creating new report") + log.msg("We will only create reports to a file") + oonib_reporter = None + else: + oonib_reporter = None + + yield yaml_reporter.createReport(options) + log.msg("Reporting to file %s" % yaml_reporter._stream.name) + + nettest_task_factory = NetTestTaskFactory(test_cases, test_inputs) + + #XXX: resume is not supported! + try: + #XXX: override the default cooperator, set up own scheduler + #XXX: add callback when tasks are all exhausted + for nettest_task in nettest_task_factory.generate_inputs(): + nettest_task.yaml_reporter = yaml_reporter + nettest_task.oonib_reporter = oonib_reporter + log.debug("Running %s with input unit %s" % (nettest_task, + nettest_task.test_instance.input)) + # feed the cooperator + nettest_task.start() + + except Exception: + log.exception("Problem in running test")

1 0

[ooni-probe/master] Test and implement TasksWithTimeout
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 2b4455c659dc4faf0b0085baeb63cf455583ac35 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 12 21:29:35 2013 +0100 Test and implement TasksWithTimeout Do some refactoring of the unittesting logic --- ooni/director.py | 8 ++-- ooni/managers.py | 6 +- ooni/tasks.py | 12 ++-- tests/test_managers.py | 135 ++++++++++++++++++------------------------------ 4 files changed, 64 insertions(+), 97 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index cf23269..0707664 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -1,4 +1,4 @@ -from ooni.managers import ReportingEntryManager, MeasurementsManager +from ooni.managers import ReportingEntryManager, MeasurementManager from ooni.nettest import NetTest class Director(object): @@ -50,9 +50,9 @@ class Director(object): self.netTests = [] - self.measurementsManager = MeasurementsManager(manager=self, + self.measurementManager = MeasurementManager(manager=self, netTests=self.netTests) - self.measurementsManager.director = self + self.measurementManager.director = self self.reportEntryManager = ReportingEntryManager() self.reportEntryManager.director = self @@ -67,7 +67,7 @@ class Director(object): net_test = NetTest(net_test_file, inputs, options, report) net_test.director = self - self.measurementsManager.schedule(net_test.generateMeasurements()) + self.measurementManager.schedule(net_test.generateMeasurements()) def measurementTimedOut(self, measurement): """ diff --git a/ooni/managers.py b/ooni/managers.py index be11473..cee6086 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -117,7 +117,7 @@ class TaskManager(object): def succeeded(self, result, task): raise NotImplemented -class MeasurementsManager(TaskManager): +class MeasurementManager(TaskManager): """ This is the Measurement Tracker. In here we keep track of active measurements and issue new measurements once the active ones have been completed. @@ -137,10 +137,10 @@ class MeasurementsManager(TaskManager): director = None def succeeded(self, result, measurement): - pass + self.director.measurementSucceeded(measurement) def failed(self, failure, measurement): - pass + self.director.measurementFailed(failure, measurement) class Report(object): reportEntryManager = None diff --git a/ooni/tasks.py b/ooni/tasks.py index a53287e..703aec3 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -1,4 +1,4 @@ -from twisted.internet import defer +from twisted.internet import defer, reactor class BaseTask(object): _timer = None @@ -61,15 +61,15 @@ class TaskWithTimeout(BaseTask): def _succeeded(self, result): self._cancelTimer() - BaseTask._succeeded(self, result) + return BaseTask._succeeded(self, result) def _failed(self, failure): self._cancelTimer() - BaseTask._failed(self, failure) + return BaseTask._failed(self, failure) - def _run(self): - d = BaseTask._run(self) + def start(self): self._timer = reactor.callLater(self.timeout, self._timedOut) + d = BaseTask.start(self) return d def timedOut(self): @@ -121,7 +121,7 @@ class ReportEntry(TaskWithTimeout): def __init__(self, reporter, measurement): self.reporter = reporter self.measurement = measurement - TimedOutTask.__init__(self) + TaskWithTimeout.__init__(self) def run(self): return self.reporter.writeReportEntry(self.measurement) diff --git a/tests/test_managers.py b/tests/test_managers.py index 59eba5f..a10933e 100644 --- a/tests/test_managers.py +++ b/tests/test_managers.py @@ -2,9 +2,8 @@ from twisted.trial import unittest from twisted.python import failure from twisted.internet import defer -from ooni.tasks import BaseTask -from ooni.managers import TaskManager - +from ooni.tasks import BaseTask, TaskWithTimeout +from ooni.managers import TaskManager, MeasurementManager mockFailure = failure.Failure(Exception('mock')) @@ -23,15 +22,19 @@ class MockFailOnceTask(BaseTask): else: return defer.fail(mockFailure) +class MockSuccessTaskWithTimeout(TaskWithTimeout): + def run(self): + return defer.succeed(42) + +class MockFailTaskWithTimeout(TaskWithTimeout): + def run(self): + return defer.fail(mockFailure) + class MockTaskManager(TaskManager): def __init__(self): self.successes = [] def failed(self, failure, task): - # print "TASK" - # print task - # print "FAILURES (%s)" % task.failures - # print failure pass def succeeded(self, result, task): @@ -39,117 +42,81 @@ class MockTaskManager(TaskManager): class TestTaskManager(unittest.TestCase): def setUp(self): - self.taskManager = MockTaskManager() - self.taskManager.concurrency = 10 - self.taskManager.retries = 2 - - self.taskManager.start() + self.measurementManager = MockTaskManager() + self.measurementManager.concurrency = 10 + self.measurementManager.retries = 2 - def tearDown(self): - pass - - def test_schedule_successful_one_task(self): - mock_task = MockSuccessTask() - self.taskManager.schedule(mock_task) - - @mock_task.done.addCallback - def done(res): - self.assertEqual(self.taskManager.successes, - [(42, mock_task)]) - return mock_task.done + self.measurementManager.start() - def test_schedule_failing_one_task(self): - mock_task = MockFailTask() - self.taskManager.schedule(mock_task) - - @mock_task.done.addCallback - def done(failure): - self.assertEqual(len(self.taskManager.failures), 3) - - self.assertEqual(failure, (mockFailure, mock_task)) - - return mock_task.done - - def test_schedule_successful_ten_tasks(self): + def schedule_failing_tasks(self, task_type, number=1): all_done = [] - for x in range(10): - mock_task = MockSuccessTask() + for x in range(number): + mock_task = task_type() all_done.append(mock_task.done) - self.taskManager.schedule(mock_task) - - d = defer.DeferredList(all_done) - @d.addCallback - def done(res): - for task_result, task_instance in self.taskManager.successes: - self.assertEqual(task_result, 42) - self.assertIsInstance(task_instance, MockSuccessTask) - - return d - - def test_schedule_failing_ten_tasks(self): - all_done = [] - for x in range(10): - mock_task = MockFailTask() - all_done.append(mock_task.done) - self.taskManager.schedule(mock_task) + self.measurementManager.schedule(mock_task) d = defer.DeferredList(all_done) @d.addCallback def done(res): # 10*2 because 2 is the number of retries - self.assertEqual(len(self.taskManager.failures), 10*3) - for task_result, task_instance in self.taskManager.failures: + self.assertEqual(len(self.measurementManager.failures), number*3) + for task_result, task_instance in self.measurementManager.failures: self.assertEqual(task_result, mockFailure) - self.assertIsInstance(task_instance, MockFailTask) + self.assertIsInstance(task_instance, task_type) return d - def test_schedule_successful_27_tasks(self): + def schedule_successful_tasks(self, task_type, number=1): all_done = [] - for x in range(27): - mock_task = MockSuccessTask() + for x in range(number): + mock_task = task_type() all_done.append(mock_task.done) - self.taskManager.schedule(mock_task) + self.measurementManager.schedule(mock_task) d = defer.DeferredList(all_done) @d.addCallback def done(res): - for task_result, task_instance in self.taskManager.successes: + for task_result, task_instance in self.measurementManager.successes: self.assertEqual(task_result, 42) - self.assertIsInstance(task_instance, MockSuccessTask) + self.assertIsInstance(task_instance, task_type) return d - def test_schedule_failing_27_tasks(self): - all_done = [] - for x in range(27): - mock_task = MockFailTask() - all_done.append(mock_task.done) - self.taskManager.schedule(mock_task) + def test_schedule_successful_one_task(self): + return self.schedule_successful_tasks(MockSuccessTask) - d = defer.DeferredList(all_done) - @d.addCallback - def done(res): - # 10*2 because 2 is the number of retries - self.assertEqual(len(self.taskManager.failures), 27*3) - for task_result, task_instance in self.taskManager.failures: - self.assertEqual(task_result, mockFailure) - self.assertIsInstance(task_instance, MockFailTask) + def test_schedule_successful_one_task_with_timeout(self): + return self.schedule_successful_tasks(MockSuccessTaskWithTimeout) - return d + def test_schedule_failing_one_task(self): + return self.schedule_failing_tasks(MockFailTask) + + def test_schedule_failing_one_task_with_timeout(self): + return self.schedule_failing_tasks(MockFailTaskWithTimeout) + def test_schedule_successful_ten_tasks(self): + return self.schedule_successful_tasks(MockSuccessTask, number=10) + + def test_schedule_failing_ten_tasks(self): + return self.schedule_failing_tasks(MockFailTask, number=10) + + def test_schedule_successful_27_tasks(self): + return self.schedule_successful_tasks(MockSuccessTask, number=27) + + def test_schedule_failing_27_tasks(self): + return self.schedule_failing_tasks(MockFailTask, number=27) def test_task_retry_and_succeed(self): mock_task = MockFailOnceTask() - self.taskManager.schedule(mock_task) + self.measurementManager.schedule(mock_task) @mock_task.done.addCallback def done(res): - self.assertEqual(len(self.taskManager.failures), 1) + self.assertEqual(len(self.measurementManager.failures), 1) - self.assertEqual(self.taskManager.failures, + self.assertEqual(self.measurementManager.failures, [(mockFailure, mock_task)]) - self.assertEqual(self.taskManager.successes, + self.assertEqual(self.measurementManager.successes, [(42, mock_task)]) return mock_task.done

1 0

[ooni-probe/master] Import handleAllFailures and failureToString from ooni.errors
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 08f87c126d122ad770142f4cfb9990dd6efeccbe Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 12 19:41:07 2013 +0100 Import handleAllFailures and failureToString from ooni.errors --- nettests/blocking/tcpconnect.py | 2 +- ooni/templates/dnst.py | 4 +++- ooni/templates/httpt.py | 2 +- ooni/templates/tcpt.py | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nettests/blocking/tcpconnect.py b/nettests/blocking/tcpconnect.py index b444c3b..3b22427 100644 --- a/nettests/blocking/tcpconnect.py +++ b/nettests/blocking/tcpconnect.py @@ -6,7 +6,7 @@ from twisted.internet.error import ConnectionRefusedError from twisted.internet.error import TCPTimedOutError, TimeoutError from ooni import nettest -from ooni.nettest import handleAllFailures +from ooni.errors import handleAllFailures from ooni.utils import log class TCPFactory(Factory): diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index 08d660a..243e63f 100644 --- a/ooni/templates/dnst.py +++ b/ooni/templates/dnst.py @@ -11,7 +11,9 @@ from twisted.names.client import Resolver from twisted.names.error import DNSQueryRefusedError from ooni.utils import log -from ooni.nettest import NetTestCase, failureToString +from ooni.nettest import NetTestCase +from ooni.errors import failureToString + from socket import gaierror diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index a265f4c..548c766 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -18,7 +18,7 @@ from ooni import config from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni.utils.txagentwithsocks import Agent, TrueHeaders -from ooni.nettest import handleAllFailures +from ooni.errors import handleAllFailures class InvalidSocksProxyOption(Exception): diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py index 323ead8..df63103 100644 --- a/ooni/templates/tcpt.py +++ b/ooni/templates/tcpt.py @@ -2,7 +2,8 @@ from twisted.internet import protocol, defer, reactor from twisted.internet.error import ConnectionDone from twisted.internet.endpoints import TCP4ClientEndpoint -from ooni.nettest import NetTestCase, failureToString +from ooni.nettest import NetTestCase +from ooni.errors import failureToString from ooni.utils import log class TCPSender(protocol.Protocol):

1 0

[ooni-probe/master] Unittesting for MeasurementManager
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit ebe629c757de00639a55592e7693107bc3d92094 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Jan 13 11:07:51 2013 +0100 Unittesting for MeasurementManager --- tests/test_managers.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 files changed, 84 insertions(+), 3 deletions(-) diff --git a/tests/test_managers.py b/tests/test_managers.py index d4ee2b5..d24ee5e 100644 --- a/tests/test_managers.py +++ b/tests/test_managers.py @@ -186,16 +186,97 @@ class TestTaskManager(unittest.TestCase): for x in range(56): mock_task = MockFailOnceTask() all_done.append(mock_task.done) - self.taskManager.schedule(mock_task) + self.measurementManager.schedule(mock_task) d = defer.DeferredList(all_done) @d.addCallback def done(res): - self.assertEqual(len(self.taskManager.failures), 56) + self.assertEqual(len(self.measurementManager.failures), 56) - for task_result, task_instance in self.taskManager.successes: + for task_result, task_instance in self.measurementManager.successes: self.assertEqual(task_result, 42) self.assertIsInstance(task_instance, MockFailOnceTask) return d +class MockNetTest(object): + def __init__(self): + self.successes = [] + + def succeeded(self, measurement): + self.successes.append(measurement) + +class MockMeasurement(TaskWithTimeout): + def __init__(self, net_test): + TaskWithTimeout.__init__(self) + self.netTest = net_test + + def succeeded(self, result): + return self.netTest.succeeded(self) + +class MockSuccessMeasurement(MockMeasurement): + def run(self): + return defer.succeed(42) + +class MockFailMeasurement(MockMeasurement): + def run(self): + return defer.fail(mockFailure) + +class MockFailOnceMeasurement(MockMeasurement): + def run(self): + if self.failures >= 1: + return defer.succeed(42) + else: + return defer.fail(mockFailure) + +class MockDirector(object): + def __init__(self): + self.successes = [] + + def measurementFailed(self, failure, measurement): + pass + + def measurementSucceeded(self, measurement): + self.successes.append(measurement) + +class TestMeasurementManager(unittest.TestCase): + def setUp(self): + mock_director = MockDirector() + + self.measurementManager = MeasurementManager() + self.measurementManager.director = mock_director + + self.measurementManager.concurrency = 10 + self.measurementManager.retries = 2 + + self.measurementManager.start() + + self.mockNetTest = MockNetTest() + + def test_schedule_and_net_test_notified(self, number=1): + # XXX we should probably be inheriting from the base test class + mock_task = MockSuccessMeasurement(self.mockNetTest) + self.measurementManager.schedule(mock_task) + + @mock_task.done.addCallback + def done(res): + self.assertEqual(self.mockNetTest.successes, + [mock_task]) + + self.assertEqual(len(self.mockNetTest.successes), 1) + return mock_task.done + + def test_schedule_failing_one_measurement(self): + mock_task = MockFailMeasurement(self.mockNetTest) + self.measurementManager.schedule(mock_task) + + @mock_task.done.addCallback + def done(failure): + self.assertEqual(len(self.measurementManager.failures), 3) + + self.assertEqual(failure, (mockFailure, mock_task)) + self.assertEqual(len(self.mockNetTest.successes), 0) + + return mock_task.done + +

1 0