June 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Removed auxiliar files created by tests and improved PEP8
by art＠torproject.org 26 Jun '14

26 Jun '14

commit 5e289c8e2c9e75db0ea548bbdce165a06d1d8965 Author: kudrom <kudrom(a)riseup.net> Date: Fri Jun 20 20:11:40 2014 +0200 Removed auxiliar files created by tests and improved PEP8 --- .gitignore | 2 ++ ooni/tests/disable_test_dns.py | 1 + ooni/tests/mocks.py | 26 ++++++++++++++-- ooni/tests/test_deck.py | 58 ++++++++++++++++++++++------------- ooni/tests/test_director.py | 2 ++ ooni/tests/test_managers.py | 30 ++++++++++-------- ooni/tests/test_mutate.py | 8 +++-- ooni/tests/test_nettest.py | 24 +++++++++------ ooni/tests/test_onion.py | 3 +- ooni/tests/test_oonibclient.py | 61 +++++++++++++++++++------------------ ooni/tests/test_oonicli.py | 35 ++++++++++++--------- ooni/tests/test_otime.py | 3 +- ooni/tests/test_reporter.py | 25 ++++++++++----- ooni/tests/test_safe_represent.py | 3 +- ooni/tests/test_templates.py | 6 +++- ooni/tests/test_trueheaders.py | 16 +++++----- ooni/tests/test_txscapy.py | 20 +++++++----- ooni/tests/test_utils.py | 7 +++-- 18 files changed, 208 insertions(+), 122 deletions(-) diff --git a/.gitignore b/.gitignore index 0a6915d..5f6f71f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,5 @@ ooniprobe.resume docs/build/* .vagrant/* + +cover/* \ No newline at end of file diff --git a/ooni/tests/disable_test_dns.py b/ooni/tests/disable_test_dns.py index e9bb524..13b1ae8 100644 --- a/ooni/tests/disable_test_dns.py +++ b/ooni/tests/disable_test_dns.py @@ -10,6 +10,7 @@ from twisted.internet import reactor from twisted.names import dns from twisted.names.client import Resolver + class DNSTest(unittest.TestCase): def test_a_lookup_ooni_query(self): def done_query(message, *arg): diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py index 3fd75aa..f323f56 100644 --- a/ooni/tests/mocks.py +++ b/ooni/tests/mocks.py @@ -5,6 +5,7 @@ from ooni.tasks import BaseTask, TaskWithTimeout from ooni.nettest import NetTest from ooni.managers import TaskManager + class MockMeasurementFailOnce(BaseTask): def run(self): f = open('dummyTaskFailOnce.txt', 'w') @@ -15,6 +16,7 @@ class MockMeasurementFailOnce(BaseTask): else: return defer.fail(failure.Failure) + class MockMeasurementManager(TaskManager): def __init__(self): self.successes = [] @@ -26,6 +28,7 @@ class MockMeasurementManager(TaskManager): def succeeded(self, result, task): self.successes.append((result, task)) + class MockReporter(object): def __init__(self): self.created = defer.Deferred() @@ -39,20 +42,24 @@ class MockReporter(object): def finish(self): pass + class MockFailure(Exception): pass -## from test_managers +# # from test_managers mockFailure = failure.Failure(MockFailure('mock')) + class MockSuccessTask(BaseTask): def run(self): return defer.succeed(42) + class MockFailTask(BaseTask): def run(self): return defer.fail(mockFailure) + class MockFailOnceTask(BaseTask): def run(self): if self.failures >= 1: @@ -60,14 +67,17 @@ class MockFailOnceTask(BaseTask): else: return defer.fail(mockFailure) + class MockSuccessTaskWithTimeout(TaskWithTimeout): def run(self): return defer.succeed(42) + class MockFailTaskThatTimesOut(TaskWithTimeout): def run(self): return defer.Deferred() + class MockTimeoutOnceTask(TaskWithTimeout): def run(self): if self.failures >= 1: @@ -75,6 +85,7 @@ class MockTimeoutOnceTask(TaskWithTimeout): else: return defer.Deferred() + class MockFailTaskWithTimeout(TaskWithTimeout): def run(self): return defer.fail(mockFailure) @@ -87,6 +98,7 @@ class MockNetTest(object): def succeeded(self, measurement): self.successes.append(measurement) + class MockMeasurement(TaskWithTimeout): def __init__(self, net_test): TaskWithTimeout.__init__(self) @@ -95,14 +107,17 @@ class MockMeasurement(TaskWithTimeout): def succeeded(self, result): return self.netTest.succeeded(42) + class MockSuccessMeasurement(MockMeasurement): def run(self): return defer.succeed(42) + class MockFailMeasurement(MockMeasurement): def run(self): return defer.fail(mockFailure) + class MockFailOnceMeasurement(MockMeasurement): def run(self): if self.failures >= 1: @@ -110,6 +125,7 @@ class MockFailOnceMeasurement(MockMeasurement): else: return defer.fail(mockFailure) + class MockDirector(object): def __init__(self): self.successes = [] @@ -120,6 +136,7 @@ class MockDirector(object): def measurementSucceeded(self, measurement): self.successes.append(measurement) + ## from test_reporter.py class MockOReporter(object): def __init__(self): @@ -133,17 +150,21 @@ class MockOReporter(object): def createReport(self): from ooni.utils import log + log.debug("Creating report with %s" % self) self.created.callback(self) + class MockOReporterThatFailsWrite(MockOReporter): def writeReportEntry(self, entry): raise MockFailure + class MockOReporterThatFailsOpen(MockOReporter): def createReport(self): raise MockFailure + class MockOReporterThatFailsWriteOnce(MockOReporter): def __init__(self): self.failure = 0 @@ -154,7 +175,8 @@ class MockOReporterThatFailsWriteOnce(MockOReporter): return defer.succeed(42) else: self.failure += 1 - raise MockFailure + raise MockFailure + class MockTaskManager(TaskManager): def __init__(self): diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index fbfbef3..7e9fc7f 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -28,6 +28,7 @@ class DummyTestCase(NetTestCase): class BaseTestCase(unittest.TestCase): def setUp(self): + self.filename = "" self.cwd = os.getcwd() self.dummy_deck_content = """- options: collector: null @@ -43,38 +44,45 @@ class BaseTestCase(unittest.TestCase): testdeck: null """ + class TestInputFile(BaseTestCase): + def tearDown(self): + if self.filename != "": + os.remove(self.filename) + def test_file_cached(self): - file_hash = sha256(self.dummy_deck_content).hexdigest() + self.filename = file_hash = sha256(self.dummy_deck_content).hexdigest() input_file = InputFile(file_hash, base_path='.') with open(file_hash, 'w+') as f: f.write(self.dummy_deck_content) assert input_file.fileCached def test_file_invalid_hash(self): - invalid_hash = 'a'*64 + self.filename = invalid_hash = 'a' * 64 with open(invalid_hash, 'w+') as f: - f.write("b"*100) + f.write("b" * 100) input_file = InputFile(invalid_hash, base_path='.') self.assertRaises(AssertionError, input_file.verify) def test_save_descriptor(self): descriptor = { - 'name': 'spam', - 'id': 'spam', - 'version': 'spam', - 'author': 'spam', - 'date': 'spam', - 'description': 'spam' + 'name': 'spam', + 'id': 'spam', + 'version': 'spam', + 'author': 'spam', + 'date': 'spam', + 'description': 'spam' } - file_id = 'a'*64 + file_id = 'a' * 64 + self.filename = file_id + '.desc' input_file = InputFile(file_id, base_path='.') input_file.load(descriptor) input_file.save() - assert os.path.isfile(file_id) + assert os.path.isfile(self.filename) assert input_file.descriptorCached + class MockOONIBClient(object): def lookupTestHelpers(self, required_test_helpers): ret = { @@ -85,11 +93,12 @@ class MockOONIBClient(object): } for required_test_helper in required_test_helpers: ret[required_test_helper] = { - 'address': '127.0.0.1', - 'collector': 'httpo://thirteenchars1234.onion' - } + 'address': '127.0.0.1', + 'collector': 'httpo://thirteenchars1234.onion' + } return defer.succeed(ret) + class TestDeck(BaseTestCase): def setUp(self): super(TestDeck, self).setUp() @@ -100,6 +109,12 @@ class TestDeck(BaseTestCase): with open(os.path.join(self.cwd, 'dummy_test.py'), 'w+') as f: f.write(net_test_string) + def tearDown(self): + os.remove(os.path.join(self.cwd, 'dummy_test.py')) + os.remove(self.deck_file) + if self.filename != "": + os.remove(self.filename) + def test_open_deck(self): deck = Deck(decks_directory=".") deck.bouncer = "httpo://foo.onion" @@ -111,15 +126,16 @@ class TestDeck(BaseTestCase): deck.bouncer = "httpo://foo.onion" deck.loadDeck(self.deck_file) deck.load({'name': 'spam', - 'id': 'spam', - 'version': 'spam', - 'author': 'spam', - 'date': 'spam', - 'description': 'spam' - }) + 'id': 'spam', + 'version': 'spam', + 'author': 'spam', + 'date': 'spam', + 'description': 'spam' + }) deck.save() + self.filename = self.deck_file + ".desc" deck.verify() - + @defer.inlineCallbacks def test_lookuptest_helpers(self): deck = Deck(decks_directory=".") diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index 23afe30..1033f8e 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -7,6 +7,7 @@ from twisted.internet import defer from twisted.trial import unittest from txtorcon import TorControlProtocol + proto = MagicMock() proto.tor_protocol = TorControlProtocol() @@ -24,6 +25,7 @@ proto.tor_protocol.post_bootstrap = defer.succeed(state) mock_launch_tor = MagicMock() mock_launch_tor.return_value = defer.succeed(proto) + class TestDirector(unittest.TestCase): def tearDown(self): config.tor_state = None diff --git a/ooni/tests/test_managers.py b/ooni/tests/test_managers.py index b6abab7..ce921e3 100644 --- a/ooni/tests/test_managers.py +++ b/ooni/tests/test_managers.py @@ -18,6 +18,7 @@ from ooni.settings import config class TestTaskManager(unittest.TestCase): timeout = 1 + def setUp(self): self.measurementManager = MockTaskManager() self.measurementManager.concurrency = 20 @@ -39,6 +40,7 @@ class TestTaskManager(unittest.TestCase): self.measurementManager.schedule(mock_task) d = defer.DeferredList(all_done) + @d.addCallback def done(res): for task_result, task_instance in self.measurementManager.successes: @@ -56,16 +58,17 @@ class TestTaskManager(unittest.TestCase): self.measurementManager.schedule(mock_task) d = defer.DeferredList(all_done) + @d.addCallback def done(res): # 10*2 because 2 is the number of retries - self.assertEqual(self.measurementManager.failures, number*3) + self.assertEqual(self.measurementManager.failures, number * 3) # XXX @aagbsn is there a reason why you switched to using an int # over a using a list? # self.assertEqual(len(self.measurementManager.failures), number*3) # for task_result, task_instance in self.measurementManager.failures: - # self.assertEqual(task_result, mockFailure) - # self.assertIsInstance(task_instance, task_type) + # self.assertEqual(task_result, mockFailure) + # self.assertIsInstance(task_instance, task_type) return d @@ -100,7 +103,7 @@ class TestTaskManager(unittest.TestCase): self.assertEqual(self.measurementManager.failures, 1) # self.assertEqual(len(self.measurementManager.failures), 1) # for task_result, task_instance in self.measurementManager.failures: - # self.assertIsInstance(task_instance, task_type) + # self.assertIsInstance(task_instance, task_type) return mock_task.done @@ -119,9 +122,9 @@ class TestTaskManager(unittest.TestCase): @mock_task.done.addBoth def done(res): self.assertEqual(self.measurementManager.failures, 1) - #self.assertEqual(len(self.measurementManager.failures), 1) + # self.assertEqual(len(self.measurementManager.failures), 1) # for task_result, task_instance in self.measurementManager.failures: - # self.assertIsInstance(task_instance, task_type) + # self.assertIsInstance(task_instance, task_type) for task_result, task_instance in self.measurementManager.successes: self.assertEqual(task_result, 42) @@ -129,7 +132,6 @@ class TestTaskManager(unittest.TestCase): return mock_task.done - def test_schedule_failing_one_task(self): return self.schedule_failing_tasks(MockFailTask) @@ -155,11 +157,11 @@ class TestTaskManager(unittest.TestCase): @mock_task.done.addCallback def done(res): self.assertEqual(self.measurementManager.failures, 1) - #self.assertEqual(len(self.measurementManager.failures), 1) + # self.assertEqual(len(self.measurementManager.failures), 1) # self.assertEqual(self.measurementManager.failures, - # [(mockFailure, mock_task)]) + # [(mockFailure, mock_task)]) self.assertEqual(self.measurementManager.successes, - [(42, mock_task)]) + [(42, mock_task)]) return mock_task.done @@ -179,13 +181,14 @@ class TestTaskManager(unittest.TestCase): @d.addCallback def done(res): self.assertEqual(self.measurementManager.failures, number) - #self.assertEqual(len(self.measurementManager.failures), number) + # self.assertEqual(len(self.measurementManager.failures), number) for task_result, task_instance in self.measurementManager.successes: self.assertEqual(task_result, 42) self.assertIsInstance(task_instance, MockFailOnceTask) return d + class TestMeasurementManager(unittest.TestCase): def setUp(self): mock_director = MockDirector() @@ -208,9 +211,10 @@ class TestMeasurementManager(unittest.TestCase): @mock_task.done.addCallback def done(res): self.assertEqual(self.mockNetTest.successes, - [42]) + [42]) self.assertEqual(len(self.mockNetTest.successes), 1) + return mock_task.done def test_schedule_failing_one_measurement(self): @@ -220,7 +224,7 @@ class TestMeasurementManager(unittest.TestCase): @mock_task.done.addErrback def done(failure): self.assertEqual(self.measurementManager.failures, 3) - #self.assertEqual(len(self.measurementManager.failures), 3) + # self.assertEqual(len(self.measurementManager.failures), 3) self.assertEqual(failure, mockFailure) self.assertEqual(len(self.mockNetTest.successes), 0) diff --git a/ooni/tests/test_mutate.py b/ooni/tests/test_mutate.py index 7e30586..ff7676c 100644 --- a/ooni/tests/test_mutate.py +++ b/ooni/tests/test_mutate.py @@ -1,15 +1,17 @@ import unittest from ooni.kit import daphn3 + class TestDaphn3(unittest.TestCase): def test_mutate_string(self): original_string = '\x00\x00\x00' mutated = daphn3.daphn3MutateString(original_string, 1) self.assertEqual(mutated, '\x00\x01\x00') + def test_mutate_daphn3(self): original_dict = [{'client': '\x00\x00\x00'}, - {'server': '\x00\x00\x00'}] - mutated_dict = daphn3.daphn3Mutate(original_dict, 1, 1) + {'server': '\x00\x00\x00'}] + mutated_dict = daphn3.daphn3Mutate(original_dict, 1, 1) self.assertEqual(mutated_dict, [{'client': '\x00\x00\x00'}, - {'server': '\x00\x01\x00'}]) + {'server': '\x00\x01\x00'}]) diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 92278ce..5a871ee 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -1,6 +1,7 @@ import os from StringIO import StringIO from tempfile import TemporaryFile, mkstemp +import shutil from twisted.trial import unittest from twisted.internet import defer, reactor @@ -36,7 +37,7 @@ class DummyTestCase(NetTestCase): self.report['foo'] = 'foo' """ -net_test_root_required = net_test_string+""" +net_test_root_required = net_test_string + """ requiresRoot = True """ @@ -107,22 +108,26 @@ class HTTPBasedTest(httpt.HTTPTest): dummyInputs = range(1) dummyArgs = ('--spam', 'notham') -dummyOptions = {'spam':'notham'} +dummyOptions = {'spam': 'notham'} dummyInvalidArgs = ('--cram', 'jam') -dummyInvalidOptions= {'cram':'jam'} +dummyInvalidOptions = {'cram': 'jam'} dummyArgsWithRequiredOptions = ('--foo', 'moo', '--bar', 'baz') -dummyRequiredOptions = {'foo':'moo', 'bar':'baz'} +dummyRequiredOptions = {'foo': 'moo', 'bar': 'baz'} dummyArgsWithFile = ('--spam', 'notham', '--file', 'dummyInputFile.txt') + class TestNetTest(unittest.TestCase): timeout = 1 + HOME_DIR = 'ooni_home' + DUMMY_INPUT_FILE = 'dummyInputFile.txt' + def setUp(self): - with open('dummyInputFile.txt', 'w') as f: + with open(self.DUMMY_INPUT_FILE, 'w') as f: for i in range(10): f.write("%s\n" % i) - config.initialize_ooni_home('ooni_home') - config.read_config_file() + def tearDown(self): + os.remove(self.DUMMY_INPUT_FILE) def assertCallable(self, thing): self.assertIn('__call__', dir(thing)) @@ -211,7 +216,7 @@ class TestNetTest(unittest.TestCase): ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() - nt = NetTest(ntl,None) + nt = NetTest(ntl, None) nt.initializeInputProcessor() # XXX: if you use the same test_class twice you will have consumed all @@ -259,13 +264,14 @@ class TestNetTest(unittest.TestCase): return d def test_require_root_succeed(self): - #XXX: will require root to run + # XXX: will require root to run ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_root_required) for test_class, method in ntl.testCases: self.assertTrue(test_class.requiresRoot) + class TestNettestTimeout(unittest.TestCase): @defer.inlineCallbacks def setUp(self): diff --git a/ooni/tests/test_onion.py b/ooni/tests/test_onion.py index 9830235..94d9dd9 100644 --- a/ooni/tests/test_onion.py +++ b/ooni/tests/test_onion.py @@ -1,6 +1,7 @@ -from twisted.trial import unittest +from twisted.trial import unittest from ooni.utils import onion + class TestOnion(unittest.TestCase): def test_tor_details(self): assert isinstance(onion.tor_details, dict) diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index b0bb9ba..872000e 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -13,6 +13,7 @@ from ooni.oonibclient import OONIBClient input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1' deck_id = 'd4ae40ecfb3c1b943748cce503ab8233efce7823f3e391058fc0f87829c644ed' + class TestOONIBClient(unittest.TestCase): def setUp(self): host = '127.0.0.1' @@ -25,20 +26,22 @@ class TestOONIBClient(unittest.TestCase): data_dir = '/tmp/testooni' config.advanced.data_dir = data_dir - try: shutil.rmtree(data_dir) - except: pass + try: + shutil.rmtree(data_dir) + except: + pass os.mkdir(data_dir) os.mkdir(os.path.join(data_dir, 'inputs')) os.mkdir(os.path.join(data_dir, 'decks')) except Exception as ex: self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") self.oonibclient = OONIBClient('http://' + host + ':' + str(port)) - + @defer.inlineCallbacks def test_query(self): res = yield self.oonibclient.queryBackend('GET', '/policy/input') self.assertTrue(isinstance(res, list)) - + @defer.inlineCallbacks def test_get_input_list(self): input_list = yield self.oonibclient.getInputList() @@ -47,7 +50,7 @@ class TestOONIBClient(unittest.TestCase): @defer.inlineCallbacks def test_get_input_descriptor(self): input_descriptor = yield self.oonibclient.getInput(input_id) - for key in ['name', 'description', + for key in ['name', 'description', 'version', 'author', 'date', 'id']: self.assertTrue(hasattr(input_descriptor, key)) @@ -63,7 +66,7 @@ class TestOONIBClient(unittest.TestCase): @defer.inlineCallbacks def test_get_deck_descriptor(self): deck_descriptor = yield self.oonibclient.getDeck(deck_id) - for key in ['name', 'description', + for key in ['name', 'description', 'version', 'author', 'date', 'id']: self.assertTrue(hasattr(deck_descriptor, key)) @@ -74,9 +77,9 @@ class TestOONIBClient(unittest.TestCase): def test_lookup_invalid_helpers(self): self.oonibclient.address = 'http://127.0.0.1:8888' return self.failUnlessFailure( - self.oonibclient.lookupTestHelpers([ - 'sdadsadsa', 'dns' - ]), e.CouldNotFindTestHelper) + self.oonibclient.lookupTestHelpers([ + 'sdadsadsa', 'dns' + ]), e.CouldNotFindTestHelper) @defer.inlineCallbacks def test_lookup_no_test_helpers(self): @@ -105,8 +108,8 @@ class TestOONIBClient(unittest.TestCase): except: pass - for path in ['/policy/input', '/policy/nettest', - '/input', '/input/'+'a'*64, '/fooo']: + for path in ['/policy/input', '/policy/nettest', + '/input', '/input/' + 'a' * 64, '/fooo']: yield all_requests(path) for path in ['/bouncer']: @@ -116,35 +119,35 @@ class TestOONIBClient(unittest.TestCase): @defer.inlineCallbacks def test_create_report(self): res = yield self.oonibclient.queryBackend('POST', '/report', { - 'software_name': 'spam', - 'software_version': '2.0', - 'probe_asn': 'AS0', - 'probe_cc': 'ZZ', - 'test_name': 'foobar', - 'test_version': '1.0', - 'input_hashes': [] + 'software_name': 'spam', + 'software_version': '2.0', + 'probe_asn': 'AS0', + 'probe_cc': 'ZZ', + 'test_name': 'foobar', + 'test_version': '1.0', + 'input_hashes': [] }) assert isinstance(res['report_id'], unicode) @defer.inlineCallbacks def test_report_lifecycle(self): res = yield self.oonibclient.queryBackend('POST', '/report', { - 'software_name': 'spam', - 'software_version': '2.0', - 'probe_asn': 'AS0', - 'probe_cc': 'ZZ', - 'test_name': 'foobar', - 'test_version': '1.0', - 'input_hashes': [] + 'software_name': 'spam', + 'software_version': '2.0', + 'probe_asn': 'AS0', + 'probe_cc': 'ZZ', + 'test_name': 'foobar', + 'test_version': '1.0', + 'input_hashes': [] }) report_id = str(res['report_id']) - res = yield self.oonibclient.queryBackend('POST', '/report/'+report_id, { + res = yield self.oonibclient.queryBackend('POST', '/report/' + report_id, { 'content': '---\nspam: ham\n...\n' }) - res = yield self.oonibclient.queryBackend('POST', '/report/'+report_id, { + res = yield self.oonibclient.queryBackend('POST', '/report/' + report_id, { 'content': '---\nspam: ham\n...\n' }) - - res = yield self.oonibclient.queryBackend('POST', '/report/'+report_id+'/close') + + res = yield self.oonibclient.queryBackend('POST', '/report/' + report_id + '/close') diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index cf6a340..eebad28 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -9,6 +9,7 @@ from ooni.tests import is_internet_connected from ooni.settings import config from ooni.oonicli import runWithDirector + def verify_header(header): assert 'input_hashes' in header.keys() assert 'options' in header.keys() @@ -20,6 +21,7 @@ def verify_header(header): assert 'test_name' in header.keys() assert 'test_version' in header.keys() + def verify_entry(entry): assert 'input' in entry @@ -46,7 +48,7 @@ class TestRunDirector(unittest.TestCase): pass @defer.inlineCallbacks - def run_test(self, test_name, args, verify_function): + def run_helper(self, test_name, args, verify_function): output_file = 'test_report.yaml' sys.argv = ['', '-n', '-o', output_file, test_name] sys.argv.extend(args) @@ -72,9 +74,10 @@ class TestRunDirector(unittest.TestCase): assert 'factor' in entry assert 'headers_diff' in entry assert 'headers_match' in entry - yield self.run_test('blocking/http_requests', - ['-u', 'http://torproject.org/'], - verify_function) + + yield self.run_helper('blocking/http_requests', + ['-u', 'http://torproject.org/'], + verify_function) @defer.inlineCallbacks def test_http_requests_with_file(self): @@ -86,9 +89,10 @@ class TestRunDirector(unittest.TestCase): assert 'factor' in entry assert 'headers_diff' in entry assert 'headers_match' in entry - yield self.run_test('blocking/http_requests', - ['-f', 'example-input.txt'], - verify_function) + + yield self.run_helper('blocking/http_requests', + ['-f', 'example-input.txt'], + verify_function) @defer.inlineCallbacks def test_dnsconsistency(self): @@ -97,11 +101,12 @@ class TestRunDirector(unittest.TestCase): assert 'control_resolver' in entry assert 'tampering' in entry assert len(entry['tampering']) == 1 - yield self.run_test('blocking/dns_consistency', - ['-b', '8.8.8.8:53', - '-t', '8.8.8.8', - '-f', 'example-input.txt'], - verify_function) + + yield self.run_helper('blocking/dns_consistency', + ['-b', '8.8.8.8:53', + '-t', '8.8.8.8', + '-f', 'example-input.txt'], + verify_function) @defer.inlineCallbacks def test_http_header_field_manipulation(self): @@ -118,6 +123,6 @@ class TestRunDirector(unittest.TestCase): assert 'request_line_capitalization' in entry['tampering'] assert 'total' in entry['tampering'] - yield self.run_test('manipulation/http_header_field_manipulation', - ['-b', 'http://64.9.225.221'], - verify_function) + yield self.run_helper('manipulation/http_header_field_manipulation', + ['-b', 'http://64.9.225.221'], + verify_function) diff --git a/ooni/tests/test_otime.py b/ooni/tests/test_otime.py index 80979f2..f3997eb 100644 --- a/ooni/tests/test_otime.py +++ b/ooni/tests/test_otime.py @@ -4,6 +4,7 @@ from ooni import otime test_date = datetime(2002, 6, 26, 22, 45, 49) + class TestOtime(unittest.TestCase): def test_timestamp(self): self.assertEqual(otime.timestamp(test_date), "2002-06-26T224549Z") @@ -11,5 +12,3 @@ class TestOtime(unittest.TestCase): def test_fromTimestamp(self): time_stamp = otime.timestamp(test_date) self.assertEqual(test_date, otime.fromTimestamp(time_stamp)) - - diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 23c1190..e643f9a 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -1,6 +1,7 @@ import yaml import json import time +import os from mock import MagicMock from twisted.internet import defer @@ -10,11 +11,13 @@ from ooni.utils.net import StringProducer from ooni import errors as e from ooni.reporter import YAMLReporter, OONIBReporter + class MockTest(object): _start_time = time.time() report = {'report_content': 'ham'} input = 'spam' + test_details = { 'test_name': 'spam', 'test_version': '1.0', @@ -25,7 +28,7 @@ test_details = { } oonib_new_report_message = { - 'report_id': "2014-01-29T202038Z_AS0_"+"A"*50, + 'report_id': "2014-01-29T202038Z_AS0_" + "A" * 50, 'backend_version': "1.0" } @@ -33,9 +36,14 @@ oonib_generic_error_message = { 'error': 'generic-error' } + class TestYAMLReporter(unittest.TestCase): def setUp(self): - pass + self.filename = "" + + def tearDown(self): + if self.filename != "": + os.remove(self.filename) def test_write_report(self): test = MockTest() @@ -44,6 +52,7 @@ class TestYAMLReporter(unittest.TestCase): y_reporter.createReport() y_reporter.testDone(test, 'spam') with open(y_reporter.report_path) as f: + self.filename = y_reporter.report_path report_entries = yaml.safe_load_all(f) # Check for keys in header entry = report_entries.next() @@ -51,13 +60,13 @@ class TestYAMLReporter(unittest.TestCase): entry = report_entries.next() # Check for first entry of report - assert all(x in entry \ - for x in ['report_content', 'input', \ - 'test_name', 'test_started', \ + assert all(x in entry + for x in ['report_content', 'input', + 'test_name', 'test_started', 'test_runtime']) + class TestOONIBReporter(unittest.TestCase): - def setUp(self): self.mock_response = {} self.collector_address = 'http://example.com' @@ -65,12 +74,14 @@ class TestOONIBReporter(unittest.TestCase): self.oonib_reporter = OONIBReporter(test_details, self.collector_address) self.oonib_reporter.agent = MagicMock() self.mock_agent_response = MagicMock() + def deliverBody(body_receiver): body_receiver.dataReceived(json.dumps(self.mock_response)) body_receiver.connectionLost(None) + self.mock_agent_response.deliverBody = deliverBody self.oonib_reporter.agent.request.return_value = defer.succeed(self.mock_agent_response) - + @defer.inlineCallbacks def test_create_report(self): self.mock_response = oonib_new_report_message diff --git a/ooni/tests/test_safe_represent.py b/ooni/tests/test_safe_represent.py index 82a5196..459f4fe 100644 --- a/ooni/tests/test_safe_represent.py +++ b/ooni/tests/test_safe_represent.py @@ -6,9 +6,10 @@ from ooni.reporter import OSafeDumper from scapy.all import IP, UDP + class TestScapyRepresent(unittest.TestCase): def test_represent_scapy(self): - data = IP()/UDP() + data = IP() / UDP() yaml.dump_all([data], Dumper=OSafeDumper) diff --git a/ooni/tests/test_templates.py b/ooni/tests/test_templates.py index 66960c3..cbd378c 100644 --- a/ooni/tests/test_templates.py +++ b/ooni/tests/test_templates.py @@ -4,18 +4,22 @@ from twisted.internet.error import DNSLookupError from twisted.internet import reactor, defer from twisted.trial import unittest + class TestHTTPT(unittest.TestCase): def setUp(self): from twisted.web.resource import Resource from twisted.web.server import Site + class DummyResource(Resource): isLeaf = True + def render_GET(self, request): return "%s" % request.method + r = DummyResource() factory = Site(r) self.port = reactor.listenTCP(8880, factory) - + def tearDown(self): self.port.stopListening() diff --git a/ooni/tests/test_trueheaders.py b/ooni/tests/test_trueheaders.py index 70f8a5d..8e9f053 100644 --- a/ooni/tests/test_trueheaders.py +++ b/ooni/tests/test_trueheaders.py @@ -3,20 +3,20 @@ from twisted.trial import unittest from ooni.utils.trueheaders import TrueHeaders dummy_headers_dict = { - 'Header1': ['Value1', 'Value2'], - 'Header2': ['ValueA', 'ValueB'] + 'Header1': ['Value1', 'Value2'], + 'Header2': ['ValueA', 'ValueB'] } dummy_headers_dict2 = { - 'Header1': ['Value1', 'Value2'], - 'Header2': ['ValueA', 'ValueB'], - 'Header3': ['ValueA', 'ValueB'], + 'Header1': ['Value1', 'Value2'], + 'Header2': ['ValueA', 'ValueB'], + 'Header3': ['ValueA', 'ValueB'], } dummy_headers_dict3 = { - 'Header1': ['Value1', 'Value2'], - 'Header2': ['ValueA', 'ValueB'], - 'Header4': ['ValueA', 'ValueB'], + 'Header1': ['Value1', 'Value2'], + 'Header2': ['ValueA', 'ValueB'], + 'Header4': ['ValueA', 'ValueB'], } diff --git a/ooni/tests/test_txscapy.py b/ooni/tests/test_txscapy.py index 5222e0c..045eeeb 100644 --- a/ooni/tests/test_txscapy.py +++ b/ooni/tests/test_txscapy.py @@ -5,10 +5,12 @@ from twisted.trial import unittest from ooni.utils import txscapy defer.setDebugging(True) + + class TestTxScapy(unittest.TestCase): def setUp(self): # if not txscapy.hasRawSocketPermission(): - # self.skipTest("No raw socket permissions...") + # self.skipTest("No raw socket permissions...") mock_super_socket = MagicMock() mock_super_socket.ins.fileno.return_value = 1 self.scapy_factory = txscapy.ScapyFactory('foo', mock_super_socket) @@ -17,26 +19,28 @@ class TestTxScapy(unittest.TestCase): self.scapy_factory.connectionLost(None) def test_pcapdnet_installed(self): - assert txscapy.pcapdnet_installed() == True - + assert txscapy.pcapdnet_installed() is True + def test_send_packet_no_answer(self): from scapy.all import IP, TCP + sender = txscapy.ScapySender() self.scapy_factory.registerProtocol(sender) - packet = IP(dst='8.8.8.8')/TCP(dport=53) + packet = IP(dst='8.8.8.8') / TCP(dport=53) sender.startSending([packet]) self.scapy_factory.super_socket.send.assert_called_with(packet) assert len(sender.sent_packets) == 1 - + @defer.inlineCallbacks def test_send_packet_with_answer(self): from scapy.all import IP, TCP + sender = txscapy.ScapySender() self.scapy_factory.registerProtocol(sender) - packet_sent = IP(dst='8.8.8.8',src='127.0.0.1')/TCP(dport=53, sport=5300) - packet_received = IP(dst='127.0.0.1', src='8.8.8.8')/TCP(sport=53, dport=5300) - + packet_sent = IP(dst='8.8.8.8', src='127.0.0.1') / TCP(dport=53, sport=5300) + packet_received = IP(dst='127.0.0.1', src='8.8.8.8') / TCP(sport=53, dport=5300) + d = sender.startSending([packet_sent]) self.scapy_factory.super_socket.send.assert_called_with(packet_sent) diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index e21ab89..7c2ab2d 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -11,17 +11,20 @@ class TestUtils(unittest.TestCase): f.write("0\n") f.close() for i in xrange(1, 20): - f = open(basefilename+".%s" % i, "w+") + f = open("%s.%d" % (basefilename, i), "w+") f.write("%s\n" % i) f.close() pushFilenameStack(basefilename) for i in xrange(1, 20): - f = open(basefilename+".%s" % i) + f = open("%s.%d" % (basefilename, i)) c = f.readlines()[0].strip() self.assertEqual(str(i-1), str(c)) f.close() + for i in xrange(1, 21): + os.remove("%s.%d" % (basefilename, i)) + def test_log_encode(self): logmsgs = ( (r"spam\x07\x08", "spam\a\b"),

1 0

[ooni-probe/master] Safe config use between tests
by art＠torproject.org 26 Jun '14

26 Jun '14

commit bbd6a1ada2d8dc2e3959b385408295d4c9970189 Author: kudrom <kudrom(a)riseup.net> Date: Sat Jun 21 20:44:44 2014 +0200 Safe config use between tests --- ooni/tests/__init__.py | 1 + ooni/tests/test_director.py | 3 +-- ooni/tests/test_geoip.py | 4 ---- ooni/tests/test_managers.py | 10 ++++++++++ ooni/tests/test_nettest.py | 1 + ooni/tests/test_oonibclient.py | 3 +++ ooni/tests/test_oonicli.py | 1 + 7 files changed, 17 insertions(+), 6 deletions(-) diff --git a/ooni/tests/__init__.py b/ooni/tests/__init__.py index b0bbfbd..e7fd48b 100644 --- a/ooni/tests/__init__.py +++ b/ooni/tests/__init__.py @@ -2,6 +2,7 @@ import socket from ooni.settings import config config.initialize_ooni_home('ooni_home') +config.read_config_file() config.logging = False config.advanced.debug = False diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index 1033f8e..cd0483b 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -29,8 +29,7 @@ mock_launch_tor.return_value = defer.succeed(proto) class TestDirector(unittest.TestCase): def tearDown(self): config.tor_state = None - config.tor.socks_port = None - config.tor.control_port = None + config.read_config_file() def test_get_net_tests(self): director = Director() diff --git a/ooni/tests/test_geoip.py b/ooni/tests/test_geoip.py index affcea6..b801cff 100644 --- a/ooni/tests/test_geoip.py +++ b/ooni/tests/test_geoip.py @@ -9,10 +9,6 @@ from ooni import geoip class TestGeoIP(unittest.TestCase): - def setUp(self): - config.initialize_ooni_home('ooni_home') - config.read_config_file() - def test_ip_to_location(self): location = geoip.IPToLocation('8.8.8.8') assert 'countrycode' in location diff --git a/ooni/tests/test_managers.py b/ooni/tests/test_managers.py index ce921e3..d7f3177 100644 --- a/ooni/tests/test_managers.py +++ b/ooni/tests/test_managers.py @@ -29,9 +29,19 @@ class TestTaskManager(unittest.TestCase): self.clock = task.Clock() data_dir = os.path.dirname(os.path.abspath(__file__)) data_dir = os.path.join(data_dir, '..', '..', 'data') + self.old_datadir = "" + if hasattr(config.global_options, 'datadir'): + self.old_datadir = config.global_options['datadir'] config.global_options['datadir'] = data_dir config.set_paths() + def tearDown(self): + if self.old_datadir == "": + del config.global_options['datadir'] + else: + config.global_options['datadir'] = self.old_datadir + config.set_paths() + def schedule_successful_tasks(self, task_type, number=1): all_done = [] for x in range(number): diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 5a871ee..92d6cea 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -304,6 +304,7 @@ class TestNettestTimeout(unittest.TestCase): def tearDown(self): self.factory.stopFactory() self.port.stopListening() + config.read_config_file() def test_nettest_timeout(self): ntl = NetTestLoader(('-u', 'http://localhost:8007/')) diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 872000e..c6615cb 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -37,6 +37,9 @@ class TestOONIBClient(unittest.TestCase): self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") self.oonibclient = OONIBClient('http://' + host + ':' + str(port)) + def tearDown(self): + config.read_config_file() + @defer.inlineCallbacks def test_query(self): res = yield self.oonibclient.queryBackend('GET', '/policy/input') diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index eebad28..4da63f0 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -38,6 +38,7 @@ class TestRunDirector(unittest.TestCase): f.write('http://blog.torproject.org/\n') def tearDown(self): + config.read_config_file() try: os.remove('test_report.yaml') except:

1 0

[ooni-probe/master] Expand ReportLog class. Make the editing log more robust
by art＠torproject.org 26 Jun '14

26 Jun '14

commit 8b7e8fc161b1dacf0ec51150121431b2b1cea6e9 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Jun 25 13:17:25 2014 +0200 Expand ReportLog class. Make the editing log more robust --- ooni/reporter.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index b149b79..44e6b96 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -413,6 +413,24 @@ class OONIBReportLog(object): def __init__(self, file_name=config.report_log_file): self.file_name = file_name self.create_report_log() + self._reports_incomplete = [] + self._reports_in_progress = [] + self._reports_to_upload = [] + + def update_stored_reports(self): + pass + + @property + def reports_incomplete(self): + pass + + @property + def reports_in_progress(self): + pass + + @property + def reports_to_upload(self): + pass def run(self, f, *arg, **kw): lock = defer.DeferredFilesystemLock(self.file_name + '.lock') @@ -439,9 +457,14 @@ class OONIBReportLog(object): def edit_log(self): with open(self.file_name) as rfp: report = yaml.safe_load(rfp) + # This should never happen. + if report is None: + report = {} with open(self.file_name, 'w+') as wfp: - yield report - wfp.write(yaml.safe_dump(report)) + try: + yield report + finally: + wfp.write(yaml.safe_dump(report)) def _not_created(self, report_file): with self.edit_log() as report:

1 0

[ooni-probe/master] Another cleanup and config coherence consolidated in ooni.tests.bases
by art＠torproject.org 26 Jun '14

26 Jun '14

commit 09574e93ff6c24b5dc5f6b5ffe0fb5dbca9888e8 Author: kudrom <kudrom(a)riseup.net> Date: Sun Jun 22 19:44:09 2014 +0200 Another cleanup and config coherence consolidated in ooni.tests.bases --- ooni/tests/bases.py | 11 +++++++++++ ooni/tests/mocks.py | 15 +++++++++++++++ ooni/tests/test_deck.py | 17 +---------------- ooni/tests/test_director.py | 5 +++-- ooni/tests/test_nettest.py | 12 ++++++------ ooni/tests/test_oonibclient.py | 6 ++---- ooni/tests/test_oonicli.py | 18 ++++++++---------- 7 files changed, 46 insertions(+), 38 deletions(-) diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py new file mode 100644 index 0000000..a359ef8 --- /dev/null +++ b/ooni/tests/bases.py @@ -0,0 +1,11 @@ +from twisted.trial import unittest + +from ooni.settings import config + + +class ConfigTestCase(unittest.TestCase): + def setUp(self): + config.initialize_ooni_home("ooni_home") + + def tearDown(self): + config.read_config_file() \ No newline at end of file diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py index f323f56..72865b1 100644 --- a/ooni/tests/mocks.py +++ b/ooni/tests/mocks.py @@ -189,3 +189,18 @@ class MockTaskManager(TaskManager): def succeeded(self, result, task): self.successes.append((result, task)) + +class MockOONIBClient(object): + def lookupTestHelpers(self, required_test_helpers): + ret = { + 'default': { + 'address': '127.0.0.1', + 'collector': 'httpo://thirteenchars1234.onion' + } + } + for required_test_helper in required_test_helpers: + ret[required_test_helper] = { + 'address': '127.0.0.1', + 'collector': 'httpo://thirteenchars1234.onion' + } + return defer.succeed(ret) diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index 7e9fc7f..d82c0eb 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -5,6 +5,7 @@ from twisted.trial import unittest from hashlib import sha256 from ooni.deck import InputFile, Deck +from ooni.tests.mocks import MockOONIBClient net_test_string = """ from twisted.python import usage @@ -83,22 +84,6 @@ class TestInputFile(BaseTestCase): assert input_file.descriptorCached -class MockOONIBClient(object): - def lookupTestHelpers(self, required_test_helpers): - ret = { - 'default': { - 'address': '127.0.0.1', - 'collector': 'httpo://thirteenchars1234.onion' - } - } - for required_test_helper in required_test_helpers: - ret[required_test_helper] = { - 'address': '127.0.0.1', - 'collector': 'httpo://thirteenchars1234.onion' - } - return defer.succeed(ret) - - class TestDeck(BaseTestCase): def setUp(self): super(TestDeck, self).setUp() diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index cd0483b..da81731 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -2,6 +2,7 @@ from mock import patch, MagicMock from ooni.settings import config from ooni.director import Director +from ooni.tests.bases import ConfigTestCase from twisted.internet import defer from twisted.trial import unittest @@ -26,10 +27,10 @@ mock_launch_tor = MagicMock() mock_launch_tor.return_value = defer.succeed(proto) -class TestDirector(unittest.TestCase): +class TestDirector(ConfigTestCase): def tearDown(self): + super(TestDirector, self).tearDown() config.tor_state = None - config.read_config_file() def test_get_net_tests(self): director = Director() diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 92d6cea..4ef6a55 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -18,6 +18,7 @@ from ooni.managers import TaskManager from ooni.tests.mocks import MockMeasurement, MockMeasurementFailOnce from ooni.tests.mocks import MockNetTest, MockDirector, MockReporter from ooni.tests.mocks import MockMeasurementManager +from ooni.tests.bases import ConfigTestCase net_test_string = """ from twisted.python import usage @@ -114,20 +115,19 @@ dummyInvalidOptions = {'cram': 'jam'} dummyArgsWithRequiredOptions = ('--foo', 'moo', '--bar', 'baz') dummyRequiredOptions = {'foo': 'moo', 'bar': 'baz'} dummyArgsWithFile = ('--spam', 'notham', '--file', 'dummyInputFile.txt') +dummyInputFile = 'dummyInputFile.txt' class TestNetTest(unittest.TestCase): timeout = 1 - HOME_DIR = 'ooni_home' - DUMMY_INPUT_FILE = 'dummyInputFile.txt' def setUp(self): - with open(self.DUMMY_INPUT_FILE, 'w') as f: + with open(dummyInputFile, 'w') as f: for i in range(10): f.write("%s\n" % i) def tearDown(self): - os.remove(self.DUMMY_INPUT_FILE) + os.remove(dummyInputFile) def assertCallable(self, thing): self.assertIn('__call__', dir(thing)) @@ -272,7 +272,7 @@ class TestNetTest(unittest.TestCase): self.assertTrue(test_class.requiresRoot) -class TestNettestTimeout(unittest.TestCase): +class TestNettestTimeout(ConfigTestCase): @defer.inlineCallbacks def setUp(self): from twisted.internet.protocol import Protocol, Factory @@ -302,9 +302,9 @@ class TestNettestTimeout(unittest.TestCase): config.advanced.measurement_timeout = 2 def tearDown(self): + super(TestNettestTimeout, self).tearDown() self.factory.stopFactory() self.port.stopListening() - config.read_config_file() def test_nettest_timeout(self): ntl = NetTestLoader(('-u', 'http://localhost:8007/')) diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index c6615cb..1497105 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -9,12 +9,13 @@ from ooni import errors as e from ooni.utils import log from ooni.settings import config from ooni.oonibclient import OONIBClient +from ooni.tests.bases import ConfigTestCase input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1' deck_id = 'd4ae40ecfb3c1b943748cce503ab8233efce7823f3e391058fc0f87829c644ed' -class TestOONIBClient(unittest.TestCase): +class TestOONIBClient(ConfigTestCase): def setUp(self): host = '127.0.0.1' port = 8889 @@ -37,9 +38,6 @@ class TestOONIBClient(unittest.TestCase): self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") self.oonibclient = OONIBClient('http://' + host + ':' + str(port)) - def tearDown(self): - config.read_config_file() - @defer.inlineCallbacks def test_query(self): res = yield self.oonibclient.queryBackend('GET', '/policy/input') diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 4da63f0..dd41c1d 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -6,6 +6,7 @@ from twisted.internet import defer from twisted.trial import unittest from ooni.tests import is_internet_connected +from ooni.tests.bases import ConfigTestCase from ooni.settings import config from ooni.oonicli import runWithDirector @@ -26,31 +27,28 @@ def verify_entry(entry): assert 'input' in entry -class TestRunDirector(unittest.TestCase): +class TestRunDirector(ConfigTestCase): def setUp(self): if not is_internet_connected(): self.skipTest("You must be connected to the internet to run this test") config.tor.socks_port = 9050 config.tor.control_port = None + self.filenames = ['example-input.txt'] with open('example-input.txt', 'w+') as f: f.write('http://torproject.org/\n') f.write('http://bridges.torproject.org/\n') f.write('http://blog.torproject.org/\n') def tearDown(self): - config.read_config_file() - try: - os.remove('test_report.yaml') - except: - pass - try: - os.remove('example-input.txt') - except: - pass + super(TestRunDirector, self).tearDown() + if len(self.filenames) > 0: + for filename in self.filenames: + os.remove(filename) @defer.inlineCallbacks def run_helper(self, test_name, args, verify_function): output_file = 'test_report.yaml' + self.filenames.append(output_file) sys.argv = ['', '-n', '-o', output_file, test_name] sys.argv.extend(args) yield runWithDirector(False, False)

1 0

[ooni-probe/master] Add tool for viewing the status of report submission and allow users to upload reports.
by art＠torproject.org 26 Jun '14

26 Jun '14

commit 80dc4719da78527458a97ab33b97b1156e67f113 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Jun 25 13:18:05 2014 +0200 Add tool for viewing the status of report submission and allow users to upload reports. --- bin/oonireport | 26 +++++++++++++++++ ooni/report/__init__.py | 1 + ooni/report/cli.py | 57 ++++++++++++++++++++++++++++++++++++ ooni/report/parser.py | 35 +++++++++++++++++++++++ ooni/report/tool.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 192 insertions(+) diff --git a/bin/oonireport b/bin/oonireport new file mode 100755 index 0000000..4e6d48a --- /dev/null +++ b/bin/oonireport @@ -0,0 +1,26 @@ +#!/usr/bin/env python +import os +import sys + +sys.path[:] = map(os.path.abspath, sys.path) +sys.path.insert(0, os.path.abspath(os.getcwd())) + +from twisted.internet import defer, reactor + +from ooni.report import cli + +def failed(failure): + log.err("Failed to run ooni-report") + log.exception(failure) + reactor.stop() + +def done(result): + reactor.stop() + +def start(): + d = defer.maybeDeferred(cli.run) + d.addCallback(done) + d.addErrback(done) + +reactor.callWhenRunning(start) +reactor.run() diff --git a/ooni/report/__init__.py b/ooni/report/__init__.py new file mode 100644 index 0000000..a4e2017 --- /dev/null +++ b/ooni/report/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1" diff --git a/ooni/report/cli.py b/ooni/report/cli.py new file mode 100644 index 0000000..eee5c53 --- /dev/null +++ b/ooni/report/cli.py @@ -0,0 +1,57 @@ +import os +import sys + +from ooni.report import __version__ +from ooni.report import tool +from ooni.settings import config + +from twisted.python import usage + + +class Options(usage.Options): + + synopsis = """%s [options] upload | status +""" % (os.path.basename(sys.argv[0]),) + + optParameters = [ + ["collector", "c", None, + "Specify the collector to upload the result to."], + ["bouncer", "b", None, + "Specify the bouncer to query for a collector."] + ] + + def opt_version(self): + print "oonireport version:", __version__ + sys.exit(0) + + def parseArgs(self, *args): + self['command'] = args[0] + if self['command'] not in ("upload", "status"): + raise usage.UsageError( + "Must specify either command upload or status" + ) + if self['command'] == "upload": + try: + self['report_file'] = args[1] + except KeyError: + self['report_file'] = None + + +def parse_options(): + options = Options() + options.parseOptions() + return dict(options) + + +def run(): + config.read_config_file() + options = parse_options() + if options['command'] == "upload" and options['report_file']: + return tool.upload(options['report_file'], + options['collector'], + options['bouncer']) + elif options['command'] == "upload": + return tool.upload_all(options['collector'], + options['bouncer']) + elif options['command'] == "status": + return tool.status() diff --git a/ooni/report/parser.py b/ooni/report/parser.py new file mode 100644 index 0000000..a986522 --- /dev/null +++ b/ooni/report/parser.py @@ -0,0 +1,35 @@ +import yaml + + +class ReportLoader(object): + _header_keys = ( + 'probe_asn', + 'probe_cc', + 'probe_ip', + 'start_time', + 'test_name', + 'test_version', + 'options', + 'input_hashes', + 'software_name', + 'software_version' + ) + + def __init__(self, report_filename): + self._fp = open(report_filename) + self._yfp = yaml.safe_load_all(self._fp) + + self.header = self._yfp.next() + + def __iter__(self): + return self + + def next(self): + try: + self._yfp.next() + except StopIteration: + self.close() + raise StopIteration + + def close(self): + self._fp.close() diff --git a/ooni/report/tool.py b/ooni/report/tool.py new file mode 100644 index 0000000..f3dd430 --- /dev/null +++ b/ooni/report/tool.py @@ -0,0 +1,73 @@ +import yaml + +from twisted.internet import defer + +from ooni.reporter import OONIBReporter, OONIBReportLog + +from ooni.utils import log +from ooni.report import parser +from ooni.settings import config +from ooni.oonibclient import OONIBClient + + +oonib_report_log = OONIBReportLog() + + +(a)defer.inlineCallbacks +def upload(report_file, collector=None, bouncer=None): + with open(config.report_log_file) as f: + report_log = yaml.safe_load(f) + + report = parser.ReportLoader(report_file) + if bouncer: + oonib_client = OONIBClient(bouncer) + collector = yield oonib_client.lookupTestCollector( + report.header['test_name'] + ) + + if collector is None: + try: + collector = report_log[report_file]['collector'] + except KeyError: + raise Exception( + "No collector or bouncer specified and report file not in log." + ) + + oonib_reporter = OONIBReporter(report.header, collector) + log.msg("Creating report for %s with %s" % (report_file, collector)) + report_id = yield oonib_reporter.createReport() + yield oonib_report_log.created(report_file, collector, report_id) + for entry in report: + print "Writing entry" + yield oonib_reporter.writeReportEntry(entry) + log.msg("Closing report.") + yield oonib_reporter.finish() + yield oonib_report_log.closed(report_file) + + +(a)defer.inlineCallbacks +def upload_all(collector=None, bouncer=None): + for report_file, value in oonib_report_log.reports_to_upload: + yield upload(report_file, collector, bouncer) + + +def print_report(report_file, value): + print "* %s" % report_file + print " %s" % value['created_at'] + + +def status(): + print "Reports to be uploaded" + print "----------------------" + for report_file, value in oonib_report_log.reports_to_upload: + print_report(report_file, value) + + print "Reports in progress" + print "-------------------" + for report_file, value in oonib_report_log.reports_in_progress: + print_report(report_file, value) + + print "Incomplete reports" + print "------------------" + for report_file, value in oonib_report_log.reports_incomplete: + print_report(report_file, value)

1 0

[ooni-probe/master] Bump version of ooniprobe man page
by art＠torproject.org 26 Jun '14

26 Jun '14

commit 1628c56f7a4458b1f64ae25efd8d4e2e192b5213 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 26 15:57:40 2014 +0200 Bump version of ooniprobe man page --- data/ooniprobe.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/ooniprobe.1 b/data/ooniprobe.1 index 076b31b..cb5309b 100644 --- a/data/ooniprobe.1 +++ b/data/ooniprobe.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "OONI" "1" "April 29, 2014" "0.1" "OONI" +.TH "ooniprobe" "1" "April 29, 2014" "1.0.2" "ooniprobe" .SH NAME ooniprobe - a network censorship measurement tool. .

1 0

[ooni-probe/master] Add manpage for oonireport.
by art＠torproject.org 26 Jun '14

26 Jun '14

commit e95ab9930f12b2d7a9e25763df544a6f215327ee Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 26 15:56:28 2014 +0200 Add manpage for oonireport. --- data/oonireport.1 | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/data/oonireport.1 b/data/oonireport.1 new file mode 100644 index 0000000..7910e67 --- /dev/null +++ b/data/oonireport.1 @@ -0,0 +1,100 @@ +.\" Man page generated from reStructuredText. +. +.TH "oonireport" "1" "June 26, 2014" "1.0.2" "oonireport" +.SH NAME +oonireport - tool for managing ooniprobe reports. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. + +.SH SYNOPSIS +.B oonireport +.RB [ --version ] +.RB [ --help ] +.RB [ \-c +.IR collector ] +.I "status | upload" +.I [ report file ] + +.SH DESCRIPTION +.sp +oonireport is a tool for viewing which ooniprobe reports have not been +submitted to a collector and upload them. + +.SH OPTIONS + +.TP +.BR \-\^h " or " \-\-help +Display this help and exit. +.TP +.BR \-\^c " or " \-\-collector +Specify the address of the collector to use for uploading reports. +.TP +.BR upload +If no argument is specified all the reports that have not been submitted will +be submitted to either the collector specified as an option (\-c) or to the +collector that was previously used. +If a report file is specified then only that report file will be uploaded +.TP +.BR status +Outputs all of the reports that are either "not uploaded", "incomplete" or +"pending". +The possible statuses are: + +.BR not-created +.sp +If it has not been created, because the user specified to not use a +collector. + +.BR creation-failed +.sp +If we attempted to create the report, but failed to do so either because +the collector does not accept our report or because it was unreachable at the +time. + +.BR created +.sp +If the report has been successfully created, but has not yet been closed. + +.BR incomplete +.sp +If the report has been created, but we have failed to update the report +with some data. + +.SH OONIREPORT +.sp +This tool is used to upload reports that are stored on the users filesystem and +have not been submitted to a collector. This can be either because the +collector was unreachable when we attempted to submit the report or because the policy +of the collector did not support the specified test. +.sp + +.SH AUTHOR +The Tor Project +.SH COPYRIGHT +2014, The Tor Project +.

1 0

[ooni-probe/master] Merge pull request #328 from kudrom/tests_cleaned
by art＠torproject.org 26 Jun '14

26 Jun '14

commit e958dbb016f15a070aa9c9d17e609f91471e0202 Merge: 27f8f55 09574e9 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Wed Jun 25 16:17:59 2014 +0200 Merge pull request #328 from kudrom/tests_cleaned Removed auxiliar files created by tests and improved PEP8 .gitignore | 2 ++ ooni/tests/__init__.py | 1 + ooni/tests/bases.py | 11 +++++++ ooni/tests/disable_test_dns.py | 1 + ooni/tests/mocks.py | 41 +++++++++++++++++++++-- ooni/tests/test_deck.py | 65 +++++++++++++++++++------------------ ooni/tests/test_director.py | 8 +++-- ooni/tests/test_geoip.py | 4 --- ooni/tests/test_managers.py | 40 +++++++++++++++-------- ooni/tests/test_mutate.py | 8 +++-- ooni/tests/test_nettest.py | 27 +++++++++------ ooni/tests/test_onion.py | 3 +- ooni/tests/test_oonibclient.py | 64 +++++++++++++++++++----------------- ooni/tests/test_oonicli.py | 52 +++++++++++++++-------------- ooni/tests/test_otime.py | 3 +- ooni/tests/test_reporter.py | 25 ++++++++++---- ooni/tests/test_safe_represent.py | 3 +- ooni/tests/test_templates.py | 6 +++- ooni/tests/test_trueheaders.py | 16 ++++----- ooni/tests/test_txscapy.py | 20 +++++++----- ooni/tests/test_utils.py | 7 ++-- 21 files changed, 256 insertions(+), 151 deletions(-)

1 0

[ooni-probe/master] Merge branch 'feature/report_completion'
by art＠torproject.org 26 Jun '14

26 Jun '14

commit 7ad1374fc6d18f8797d230626a327ddd8a37c9df Merge: e958dbb 80dc471 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 26 12:32:43 2014 +0200 Merge branch 'feature/report_completion' * feature/report_completion: Add tool for viewing the status of report submission and allow users to upload reports. Expand ReportLog class. Make the editing log more robust Better handling of OONIB report closing. Fix bug that lead to some reports not being submitted. Also record the PID of the process that created or failed to create the report. Improve readablity of output by removing some log messages. Implement system for keeping track of reports that have not been submitted. Start implementing report log for keeping track of which reports have been created and which have failed. Pep8 related fixes. Conflicts: ooni/tests/test_nettest.py ooni/tests/test_reporter.py bin/oonireport | 26 ++ ooni/deck.py | 14 +- ooni/director.py | 47 +-- ooni/errors.py | 123 +++++--- ooni/managers.py | 26 +- ooni/nettest.py | 130 +++++---- ooni/nettests/blocking/dns_consistency.py | 7 +- ooni/oonicli.py | 98 +++---- ooni/report/__init__.py | 1 + ooni/report/cli.py | 57 ++++ ooni/report/parser.py | 35 +++ ooni/report/tool.py | 73 +++++ ooni/reporter.py | 446 +++++++++++++++++++---------- ooni/settings.py | 2 + ooni/tasks.py | 26 +- ooni/tests/test_nettest.py | 20 +- ooni/tests/test_reporter.py | 72 ++++- 17 files changed, 833 insertions(+), 370 deletions(-) diff --cc ooni/tests/test_nettest.py index 4ef6a55,6a550d6..94db4b3 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@@ -8,18 -6,11 +6,13 @@@ from twisted.internet import defer, rea from twisted.python.usage import UsageError from ooni.settings import config - from ooni.errors import MissingRequiredOption, InvalidOption, FailureToLoadNetTest + from ooni.errors import MissingRequiredOption from ooni.nettest import NetTest, NetTestLoader - from ooni.tasks import BaseTask from ooni.director import Director - from ooni.managers import TaskManager - from ooni.tests.mocks import MockMeasurement, MockMeasurementFailOnce - from ooni.tests.mocks import MockNetTest, MockDirector, MockReporter - from ooni.tests.mocks import MockMeasurementManager +from ooni.tests.bases import ConfigTestCase + net_test_string = """ from twisted.python import usage from ooni.nettest import NetTestCase @@@ -115,9 -105,8 +107,10 @@@ dummyInvalidOptions = {'cram': 'jam' dummyArgsWithRequiredOptions = ('--foo', 'moo', '--bar', 'baz') dummyRequiredOptions = {'foo': 'moo', 'bar': 'baz'} dummyArgsWithFile = ('--spam', 'notham', '--file', 'dummyInputFile.txt') +dummyInputFile = 'dummyInputFile.txt' + + class TestNetTest(unittest.TestCase): timeout = 1 @@@ -272,7 -261,8 +265,8 @@@ self.assertTrue(test_class.requiresRoot) -class TestNettestTimeout(unittest.TestCase): +class TestNettestTimeout(ConfigTestCase): + @defer.inlineCallbacks def setUp(self): from twisted.internet.protocol import Protocol, Factory diff --cc ooni/tests/test_reporter.py index e643f9a,69d9775..a499eb3 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@@ -7,11 -7,10 +8,11 @@@ from mock import MagicMoc from twisted.internet import defer from twisted.trial import unittest - from ooni.utils.net import StringProducer from ooni import errors as e - from ooni.reporter import YAMLReporter, OONIBReporter + from ooni.reporter import YAMLReporter, OONIBReporter, OONIBReportLog + + class MockTest(object): _start_time = time.time() report = {'report_content': 'ham'} @@@ -38,12 -36,9 +39,13 @@@ oonib_generic_error_message = class TestYAMLReporter(unittest.TestCase): + def setUp(self): - pass + self.filename = "" + + def tearDown(self): + if self.filename != "": + os.remove(self.filename) def test_write_report(self): test = MockTest() @@@ -78,9 -75,9 +83,10 @@@ class TestOONIBReporter(unittest.TestCa def deliverBody(body_receiver): body_receiver.dataReceived(json.dumps(self.mock_response)) body_receiver.connectionLost(None) + self.mock_agent_response.deliverBody = deliverBody - self.oonib_reporter.agent.request.return_value = defer.succeed(self.mock_agent_response) + self.oonib_reporter.agent.request.return_value = defer.succeed( + self.mock_agent_response) @defer.inlineCallbacks def test_create_report(self):

1 0

[ooni-probe/master] Add oonireport to setup.py
by art＠torproject.org 26 Jun '14

26 Jun '14

commit 53dbf1b961fd4432618122d5f2ce1f0fb1bb1f1d Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 26 12:36:17 2014 +0200 Add oonireport to setup.py --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 9fff96c..8836220 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def download_geoip_files(dst): chunk = response.read(CHUNK) if not chunk: break f.write(chunk) - + with open(target_file, 'w+') as f: gf = gzip.open(target_gz_file, 'rb') while True: @@ -86,13 +86,13 @@ setup( url="https://ooni.torproject.org/", package_dir={'ooni': 'ooni'}, data_files=data_files, - packages=['ooni', 'ooni.api', 'ooni.kit', - 'ooni.nettests', 'ooni.nettests.manipulation', + packages=['ooni', 'ooni.api', 'ooni.kit', + 'ooni.nettests', 'ooni.nettests.manipulation', 'ooni.nettests.experimental', 'ooni.nettests.scanning', 'ooni.nettests.blocking', 'ooni.nettests.third_party', 'ooni.templates', 'ooni.tests', 'ooni.utils'], - scripts=["bin/ooniprobe"], + scripts=["bin/ooniprobe", "bin/oonireport"], dependency_links=dependency_links, install_requires=install_requires )

1 0