June 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Fix director unittest
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 1ac4e4fd8f13b8e358034d3b69174fdc19b02543 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 17:11:22 2013 +0100 Fix director unittest --- tests/test_director.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_director.py b/tests/test_director.py index 484cb21..a9dfbe8 100644 --- a/tests/test_director.py +++ b/tests/test_director.py @@ -42,9 +42,11 @@ class TestDirector(unittest.TestCase): pass def test_start_net_test(self): - net_test_loader = NetTestLoader(net_test_string, dummyArgs) - net_test_loader.checkOptions() - d = self.director.startNetTest('', net_test_loader, self.reporters) + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(net_test_string) + + ntl.checkOptions() + d = self.director.startNetTest('', ntl, self.reporters) @d.addCallback def done(result):

1 0

[ooni-probe/master] Rename private option to test_file to separate it from test string.
by art＠torproject.org 19 Jun '13

19 Jun '13

commit d1f41883c8d0187e6360b2a2b9075f0cb7eb7a37 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 17:08:43 2013 +0100 Rename private option to test_file to separate it from test string. --- ooni/oonicli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 2fd4688..243b90c 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -68,7 +68,7 @@ class Options(usage.Options): if self['testdeck']: return try: - self['test'] = args[0] + self['test_file'] = args[0] self['subargs'] = args[1:] except: raise usage.UsageError("No test filename specified!")

1 0

[ooni-probe/master] Merge branch 'master' into task_manager
by art＠torproject.org 19 Jun '13

19 Jun '13

commit d3ab36b744b6e8e9862fd14ffae586d1b1a8e32f Merge: 1710ca5 6f16d8c Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 16:36:44 2013 +0100 Merge branch 'master' into task_manager * master: Order matters for little ol pip requirements.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)

1 0

[ooni-probe/master] Fix unittests for nettest
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 04d5fc5d157b7e5ac47230ee018fb382756f85dc Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 17:05:27 2013 +0100 Fix unittests for nettest --- tests/test_nettest.py | 62 +++++++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/tests/test_nettest.py b/tests/test_nettest.py index 5645dbf..78240d5 100644 --- a/tests/test_nettest.py +++ b/tests/test_nettest.py @@ -61,7 +61,7 @@ class DummyTestCase(NetTestCase): self.report['foo'] = 'foo' """ -net_test_with_required_option = """ +net_test_string_with_required_option = """ from twisted.python import usage from ooni.nettest import NetTestCase @@ -124,8 +124,9 @@ class TestNetTest(unittest.TestCase): f.write(net_test_string) f.close() - ntl = NetTestLoader(net_test_file, - dummyArgs) + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestFile(net_test_file) + self.verifyMethods(ntl.testCases) os.unlink(net_test_file) @@ -134,8 +135,9 @@ class TestNetTest(unittest.TestCase): Given a file like object verify that the net test cases are properly generated. """ - ntl = NetTestLoader(net_test_string, - dummyArgs) + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(net_test_string) + self.verifyMethods(ntl.testCases) def test_load_net_test_from_StringIO(self): @@ -143,13 +145,15 @@ class TestNetTest(unittest.TestCase): Given a file like object verify that the net test cases are properly generated. """ - ntl = NetTestLoader(StringIO(net_test_string), - dummyArgs) + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(net_test_string) + self.verifyMethods(ntl.testCases) def test_load_with_option(self): - ntl = NetTestLoader(StringIO(net_test_string), - dummyArgs) + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(net_test_string) + self.assertIsInstance(ntl, NetTestLoader) for test_klass, test_meth in ntl.testCases: for option in dummyOptions.keys(): @@ -157,27 +161,32 @@ class TestNetTest(unittest.TestCase): def test_load_with_invalid_option(self): try: - ntl = NetTestLoader(StringIO(net_test_string), dummyInvalidArgs) + ntl = NetTestLoader(dummyInvalidArgs) + ntl.loadNetTestString(net_test_string) + ntl.checkOptions() raise Exception except UsageError: pass def test_load_with_required_option(self): - net_test = NetTestLoader(StringIO(net_test_with_required_option), - dummyArgsWithRequiredOptions) - self.assertIsInstance(net_test, NetTestLoader) + ntl = NetTestLoader(dummyArgsWithRequiredOptions) + ntl.loadNetTestString(net_test_string_with_required_option) + + self.assertIsInstance(ntl, NetTestLoader) def test_load_with_missing_required_option(self): try: - net_test = NetTestLoader(StringIO(net_test_with_required_option), - dummyArgs) + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(net_test_string_with_required_option) + except MissingRequiredOption: pass def test_net_test_inputs(self): - ntl = NetTestLoader(StringIO(net_test_string_with_file), - dummyArgsWithFile) + ntl = NetTestLoader(dummyArgsWithFile) + ntl.loadNetTestString(net_test_string_with_file) + ntl.checkOptions() # XXX: if you use the same test_class twice you will have consumed all @@ -189,17 +198,18 @@ class TestNetTest(unittest.TestCase): self.assertEqual(len(list(test_class.inputs)), 10) def test_setup_local_options_in_test_cases(self): - ntl = NetTestLoader(StringIO(net_test_string), - dummyArgs) + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(net_test_string) + ntl.checkOptions() for test_class, test_method in ntl.testCases: self.assertEqual(test_class.localOptions, dummyOptions) def test_generate_measurements_size(self): + ntl = NetTestLoader(dummyArgsWithFile) + ntl.loadNetTestString(net_test_string_with_file) - ntl = NetTestLoader(StringIO(net_test_string_with_file), - dummyArgsWithFile) ntl.checkOptions() net_test = NetTest(ntl, None) @@ -207,8 +217,9 @@ class TestNetTest(unittest.TestCase): self.assertEqual(len(measurements), 20) def test_net_test_completed_callback(self): - ntl = NetTestLoader(StringIO(net_test_string_with_file), - dummyArgsWithFile) + ntl = NetTestLoader(dummyArgsWithFile) + ntl.loadNetTestString(net_test_string_with_file) + ntl.checkOptions() director = Director() @@ -224,8 +235,9 @@ class TestNetTest(unittest.TestCase): def test_require_root_succeed(self): #XXX: will require root to run - ntl = NetTestLoader(StringIO(net_test_root_required), - dummyArgs) + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(net_test_root_required) + for test_class, method in ntl.testCases: self.assertTrue(test_class.requiresRoot)

1 0

[ooni-probe/master] Refactoring of NetTestLoader
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 8bead060681d86cdfa43f35b1b57567e51816aa6 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 17:05:39 2013 +0100 Refactoring of NetTestLoader * Make it clear that calling one of those methods can be extremely dangerous * Kill a bug spotted thanks to unittesting --- ooni/nettest.py | 86 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 6273d34..6323989 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -23,7 +23,10 @@ class NetTestLoader(object): def __init__(self, options): self.options = options - self.testCases = self.loadNetTest(options['test']) + if 'test_file' in options: + self.loadNetTestFile(options['test_file']) + elif 'test_string' in options: + self.loadNetTestString(options['test_string']) @property def testDetails(self): @@ -109,7 +112,45 @@ class NetTestLoader(object): assert usage_options == test_class.usageOptions return usage_options - def loadNetTest(self, net_test_file): + def loadNetTestString(self, net_test_string): + """ + Load NetTest from a string. + WARNING input to this function *MUST* be sanitized and *NEVER* be + untrusted. + Failure to do so will result in code exec. + + net_test_string: + + a string that contains the net test to be run. + """ + net_test_file_object = StringIO(net_test_string) + + ns = {} + test_cases = [] + exec net_test_file_object.read() in ns + for item in ns.itervalues(): + test_cases.extend(self._get_test_methods(item)) + + if not test_cases: + raise NoTestCasesFound + + self.setupTestCases(test_cases) + + def loadNetTestFile(self, net_test_file): + """ + Load NetTest from a file. + """ + test_cases = [] + module = filenameToModule(net_test_file) + for __, item in getmembers(module): + test_cases.extend(self._get_test_methods(item)) + + if not test_cases: + raise NoTestCasesFound + + self.setupTestCases(test_cases) + + def setupTestCases(self, test_cases): """ Creates all the necessary test_cases (a list of tuples containing the NetTestCase (test_class, test_method)) @@ -130,25 +171,10 @@ class NetTestLoader(object): is either a file path or a file like object that will be used to generate the test_cases. """ - test_cases = None - try: - if os.path.isfile(net_test_file): - test_cases = self._loadNetTestFile(net_test_file) - else: - net_test_file = StringIO(net_test_file) - raise TypeError("not a file path") - - except TypeError: - if hasattr(net_test_file, 'read'): - test_cases = self._loadNetTestFromFileObject(net_test_file) - - if not test_cases: - raise NoTestCasesFound - test_class, _ = test_cases[0] self.testVersion = test_class.version self.testName = test_class.name.lower().replace(' ','_') - return test_cases + self.testCases = test_cases def checkOptions(self): """ @@ -160,7 +186,8 @@ class NetTestLoader(object): for klass in test_classes: options = self.usageOptions() - options.parseOptions(self.options['subargs']) + options.parseOptions(self.options) + if options: klass.localOptions = options @@ -175,27 +202,6 @@ class NetTestLoader(object): inputs = [None] klass.inputs = inputs - def _loadNetTestFromFileObject(self, net_test_string): - """ - Load NetTest from a string - """ - ns = {} - test_cases = [] - exec net_test_string.read() in ns - for item in ns.itervalues(): - test_cases.extend(self._get_test_methods(item)) - return test_cases - - def _loadNetTestFile(self, net_test_file): - """ - Load NetTest from a file - """ - test_cases = [] - module = filenameToModule(net_test_file) - for __, item in getmembers(module): - test_cases.extend(self._get_test_methods(item)) - return test_cases - def _get_test_methods(self, item): """ Look for test_ methods in subclasses of NetTestCase

1 0

[ooni-probe/master] Remove now superfluos runner unittest
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 54d7a959cfd218beae394f940408f430701cb5e3 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 17:11:31 2013 +0100 Remove now superfluos runner unittest --- tests/test_runner.py | 80 -------------------------------------------------- 1 file changed, 80 deletions(-) diff --git a/tests/test_runner.py b/tests/test_runner.py deleted file mode 100644 index 0b148e1..0000000 --- a/tests/test_runner.py +++ /dev/null @@ -1,80 +0,0 @@ -from twisted.trial import unittest - - -from ooni.inputunit import InputUnit -from ooni.nettest import NetTestCase -from ooni.reporter import OReporter - -from ooni.runner import loadTestsAndOptions, runTestCasesWithInputUnit - -class DummyTestCase(NetTestCase): - def test_a(self): - self.report['bar'] = 'bar' - def test_b(self): - self.report['foo'] = 'foo' - -class DummyTestCasePP(DummyTestCase): - def postProcessor(self, report): - self.report['antani'] = 'sblinda' - -class DummyReporter(OReporter): - dummy_report = [] - def createReport(self, options): - pass - - def writeReportEntry(self, entry): - self.dummy_report.append(entry) - -class TestRunner(unittest.TestCase): - def test_load_test_and_options(self): - input_unit = InputUnit([0,1,2,3,4]) - cmd_line_options = {} - test_cases, options = loadTestsAndOptions([DummyTestCase], - cmd_line_options) - self.assertEqual(test_cases[0][1], 'test_b') - self.assertEqual(test_cases[1][1], 'test_a') - - def test_run_testcase_with_input_unit(self): - def done(result): - report = oreporter.dummy_report - self.assertEqual(len(report), 10*2) - # XXX debug why this is failing - # for idx, entry in enumerate(oreporter.dummy_report): - # if idx % 2 == 0: - # self.assertEqual(entry['report']['foo'], 'foo') - # else: - # self.assertEqual(entry['report']['bar'], 'bar') - - input_unit = InputUnit([0,1,2,3,4]) - cmd_line_options = {'collector': None} - - oreporter = DummyReporter(cmd_line_options) - oreporter.dummy_report = [] - - test_cases, options = loadTestsAndOptions([DummyTestCase], - cmd_line_options) - - d = runTestCasesWithInputUnit(test_cases, input_unit, oreporter, oreporter) - d.addBoth(done) - return d - - def test_with_post_processing(self): - def done(result): - report = oreporter.dummy_report - self.assertEqual(len(report), 6) - for entry in report: - if entry['test_name'] == 'summary': - self.assertEqual(entry['report'], {'antani': 'sblinda'}) - - input_unit = InputUnit([None]) - cmd_line_options = {'collector': None} - - oreporter = DummyReporter(cmd_line_options) - oreporter.dummy_report = [] - - test_cases, options = loadTestsAndOptions([DummyTestCasePP], - cmd_line_options) - - d = runTestCasesWithInputUnit(test_cases, input_unit, oreporter, oreporter) - d.addBoth(done) - return d

1 0

[ooni-probe/master] Make the starting of tests more robust.
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 5abfe2a902b60062f08feb22ef5a47268e51c8b1 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 18:12:13 2013 +0100 Make the starting of tests more robust. --- ooni/nettest.py | 11 +++++++---- ooni/oonicli.py | 18 +++++++++++++----- ooni/utils/geodata.py | 7 +++---- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 0b3e69c..1fe19f1 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -38,7 +38,11 @@ class NetTestLoader(object): config.privacy.includecountry or \ config.privacy.includecity): log.msg("We will include some geo data in the report") - client_geodata = geodata.IPToLocation(config.probe_ip) + try: + client_geodata = geodata.IPToLocation(config.probe_ip) + except e.GeoIPDataFilesNotFound: + log.err("Unable to find the geoip data files") + client_geodata = {'city': None, 'countrycode': None, 'asn': None} if config.privacy.includeip: client_geodata['ip'] = config.probe_ip @@ -51,8 +55,7 @@ class NetTestLoader(object): client_geodata['asn'] = 'AS0' elif 'asn' in client_geodata: # XXX this regexp should probably go inside of geodata - client_geodata['asn'] = \ - re.search('AS\d+', client_geodata['asn']).group(0) + client_geodata['asn'] = client_geodata['asn'] log.msg("Your AS number is: %s" % client_geodata['asn']) else: client_geodata['asn'] = None @@ -306,7 +309,7 @@ class NetTest(object): self.state.taskDone() if len(self.report.reporters) == 0: - raise AllReportersFailed + raise e.AllReportersFailed return report_results diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 73d7709..93bf8bf 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -35,8 +35,8 @@ class Options(usage.Options): optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], - ["collector", "c", None, - "Address of the collector of test results. (example: http://127.0.0.1:8888)"], + ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion', + "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"], ["logfile", "l", None, "log file name"], ["pcapfile", "O", None, "pcap file name"], ["parallelism", "p", "10", "input parallelism"], @@ -130,15 +130,22 @@ def runWithDirector(): director = Director() d = director.start() + def director_startup_failed(failure): + log.err("Failed to start the director") + log.exception(failure) + reactor.stop() + # Wait until director has started up (including bootstrapping Tor) before adding tess def post_director_start(_): for net_test_loader in test_list: - yaml_reporter = YAMLReporter(net_test_loader.testDetails) + test_details = net_test_loader.testDetails + + yaml_reporter = YAMLReporter(test_details) reporters = [yaml_reporter] if global_options['collector']: try: - oonib_reporter = OONIBReporter(net_test_loader.testDetails, + oonib_reporter = OONIBReporter(test_details, global_options['collector']) reporters.append(oonib_reporter) except InvalidOONIBCollectorAddress: @@ -152,7 +159,7 @@ def runWithDirector(): with open('collector') as f: reporter_url = random.choice(f.readlines()) reporter_url = reporter_url.split('#')[0].strip() - oonib_reporter = OONIBReporter(net_test_loader.testDetails, reporter_url) + oonib_reporter = OONIBReporter(test_details, reporter_url) reporters.append(oonib_reporter) log.debug("adding callback for startNetTest") @@ -160,4 +167,5 @@ def runWithDirector(): d.addCallback(shutdown) d.addCallback(post_director_start) + d.addErrback(director_startup_failed) reactor.run() diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index d9883ba..56ce05c 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -6,15 +6,13 @@ from twisted.internet import reactor, defer, protocol from ooni.utils import log, net from ooni import config +from ooni.errors import GeoIPDataFilesNotFound try: import pygeoip except ImportError: log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") -class GeoIPDataFilesNotFound(Exception): - pass - def IPToLocation(ipaddr): city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') @@ -29,7 +27,8 @@ def IPToLocation(ipaddr): location['countrycode'] = country_dat.country_code_by_addr(ipaddr) asn_dat = pygeoip.GeoIP(asn_file) - location['asn'] = asn_dat.org_by_addr(ipaddr) + asn = asn_dat.org_by_addr(ipaddr) + location['asn'] = re.search('AS\d+', asn).group(0) except IOError: log.err("Could not find GeoIP data files. Go into data/ "

1 0

[ooni-probe/master] Fix so that parsing of test decks works properly
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 49c442b48ac72420338ede93b2601a74c0f028f5 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 28 21:05:26 2013 +0100 Fix so that parsing of test decks works properly --- ooni/oonicli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index af06ad2..06aa20c 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -106,8 +106,8 @@ def runWithDirector(): if global_options['testdeck']: test_deck = yaml.safe_load(open(global_options['testdeck'])) for test in test_deck: - test_list.append(NetTestLoader(test['options'], - test_file=global_options['test_file'])) + test_list.append(NetTestLoader(test['options']['subargs'], + test_file=test['options']['test_file'])) else: log.debug("No test deck detected") test_list.append(NetTestLoader(global_options['subargs'],

1 0

[ooni-probe/master] Fix typo in test deck before_i_commit
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 18266c634c0538585986c0d31dd929ba83a22203 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 1 11:31:42 2013 +0100 Fix typo in test deck before_i_commit --- decks/before_i_commit.testdeck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decks/before_i_commit.testdeck b/decks/before_i_commit.testdeck index a9e08d8..d1b4062 100644 --- a/decks/before_i_commit.testdeck +++ b/decks/before_i_commit.testdeck @@ -20,7 +20,7 @@ logfile: reports/before_i_commit.log pcapfile: null reportfile: reports/http_host.yamloo - subargs: [-b, 'http://93.95.227.200', -c, 'http://nkvphnp3p6agi5qq.onion', -f, example_inputs/http_host_file.txt] + subargs: [-b, 'http://93.95.227.200', -f, example_inputs/http_host_file.txt] test_file: nettests/manipulation/http_host.py - options: collector: null

1 0

[ooni-probe/master] Add more debugging to managers
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 708a3a8a858ef3d3a6ae15d4700b3362f02d9722 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 28 20:46:14 2013 +0100 Add more debugging to managers --- ooni/managers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ooni/managers.py b/ooni/managers.py index 011e3b0..17b430d 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -1,5 +1,6 @@ import itertools from twisted.internet import defer +from ooni.utils import log def makeIterable(item): """ @@ -28,6 +29,9 @@ class TaskManager(object): The has failed to complete, we append it to the end of the task chain to be re-run once all the currently scheduled tasks have run. """ + log.err("Task %s has failed" % task) + log.exception(failure) + self._active_tasks.remove(task) self.failures.append((failure, task)) @@ -93,6 +97,8 @@ class TaskManager(object): Takes as argument a single task or a task iterable and appends it to the task generator queue. """ + log.debug("Starting this task %s" % repr(task_or_task_iterator)) + iterable = makeIterable(task_or_task_iterator) self._tasks = itertools.chain(self._tasks, iterable) @@ -138,7 +144,8 @@ class MeasurementManager(TaskManager): concurrency = 10 def succeeded(self, result, measurement): - pass + log.debug("Successfully performed measurement %s" % measurement) + log.debug(result) def failed(self, failure, measurement): pass @@ -149,7 +156,8 @@ class ReportEntryManager(TaskManager): concurrency = 20 def succeeded(self, result, task): - pass + log.debug("Successfully performed report %s" % task) + log.debug(result) def failed(self, failure, task): pass

1 0