tor-commits June 2013

tor-commits@lists.torproject.org

19 participants
1571 discussions

[ooni-probe/develop] Change how we pass arguments to the nettest loader
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 64715a714170b9175d07ea9420ef8b3dcf177c83 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 18:11:30 2013 +0100 Change how we pass arguments to the nettest loader * Explicit the passing of the nettest file argument --- ooni/errors.py | 4 ++++ ooni/nettest.py | 12 ++++++------ ooni/oonicli.py | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 2023dbf..a194b96 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -126,3 +126,7 @@ class InvalidOONIBCollectorAddress(Exception): class AllReportersFailed(Exception): pass +class GeoIPDataFilesNotFound(Exception): + pass + + diff --git a/ooni/nettest.py b/ooni/nettest.py index 6323989..0b3e69c 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -10,7 +10,7 @@ from ooni.utils import log, checkForRoot, NotRootError, geodata from ooni import config from ooni import otime -from ooni.errors import AllReportersFailed +from ooni import errors as e from inspect import getmembers from StringIO import StringIO @@ -21,12 +21,12 @@ class NoTestCasesFound(Exception): class NetTestLoader(object): method_prefix = 'test' - def __init__(self, options): + def __init__(self, options, test_file=None, test_string=None): self.options = options - if 'test_file' in options: - self.loadNetTestFile(options['test_file']) - elif 'test_string' in options: - self.loadNetTestString(options['test_string']) + if test_file: + self.loadNetTestFile(test_file) + elif test_string: + self.loadNetTestString(test_string) @property def testDetails(self): diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 243b90c..73d7709 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -107,10 +107,12 @@ def runWithDirector(): if global_options['testdeck']: test_deck = yaml.safe_load(open(global_options['testdeck'])) for test in test_deck: - test_list.append(NetTestLoader(test['options'])) + test_list.append(NetTestLoader(test['options'], + test_file=global_options['test_file'])) else: log.debug("No test deck detected") - test_list.append(NetTestLoader(global_options)) + test_list.append(NetTestLoader(global_options['subargs'], + test_file=global_options['test_file'])) # check each test's usageOptions for net_test_loader in test_list:

1 0

[ooni-probe/develop] Remove now superfluos runner unittest
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 54d7a959cfd218beae394f940408f430701cb5e3 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 17:11:31 2013 +0100 Remove now superfluos runner unittest --- tests/test_runner.py | 80 -------------------------------------------------- 1 file changed, 80 deletions(-) diff --git a/tests/test_runner.py b/tests/test_runner.py deleted file mode 100644 index 0b148e1..0000000 --- a/tests/test_runner.py +++ /dev/null @@ -1,80 +0,0 @@ -from twisted.trial import unittest - - -from ooni.inputunit import InputUnit -from ooni.nettest import NetTestCase -from ooni.reporter import OReporter - -from ooni.runner import loadTestsAndOptions, runTestCasesWithInputUnit - -class DummyTestCase(NetTestCase): - def test_a(self): - self.report['bar'] = 'bar' - def test_b(self): - self.report['foo'] = 'foo' - -class DummyTestCasePP(DummyTestCase): - def postProcessor(self, report): - self.report['antani'] = 'sblinda' - -class DummyReporter(OReporter): - dummy_report = [] - def createReport(self, options): - pass - - def writeReportEntry(self, entry): - self.dummy_report.append(entry) - -class TestRunner(unittest.TestCase): - def test_load_test_and_options(self): - input_unit = InputUnit([0,1,2,3,4]) - cmd_line_options = {} - test_cases, options = loadTestsAndOptions([DummyTestCase], - cmd_line_options) - self.assertEqual(test_cases[0][1], 'test_b') - self.assertEqual(test_cases[1][1], 'test_a') - - def test_run_testcase_with_input_unit(self): - def done(result): - report = oreporter.dummy_report - self.assertEqual(len(report), 10*2) - # XXX debug why this is failing - # for idx, entry in enumerate(oreporter.dummy_report): - # if idx % 2 == 0: - # self.assertEqual(entry['report']['foo'], 'foo') - # else: - # self.assertEqual(entry['report']['bar'], 'bar') - - input_unit = InputUnit([0,1,2,3,4]) - cmd_line_options = {'collector': None} - - oreporter = DummyReporter(cmd_line_options) - oreporter.dummy_report = [] - - test_cases, options = loadTestsAndOptions([DummyTestCase], - cmd_line_options) - - d = runTestCasesWithInputUnit(test_cases, input_unit, oreporter, oreporter) - d.addBoth(done) - return d - - def test_with_post_processing(self): - def done(result): - report = oreporter.dummy_report - self.assertEqual(len(report), 6) - for entry in report: - if entry['test_name'] == 'summary': - self.assertEqual(entry['report'], {'antani': 'sblinda'}) - - input_unit = InputUnit([None]) - cmd_line_options = {'collector': None} - - oreporter = DummyReporter(cmd_line_options) - oreporter.dummy_report = [] - - test_cases, options = loadTestsAndOptions([DummyTestCasePP], - cmd_line_options) - - d = runTestCasesWithInputUnit(test_cases, input_unit, oreporter, oreporter) - d.addBoth(done) - return d

1 0

[ooni-probe/develop] Add more debugging to managers
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 708a3a8a858ef3d3a6ae15d4700b3362f02d9722 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 28 20:46:14 2013 +0100 Add more debugging to managers --- ooni/managers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ooni/managers.py b/ooni/managers.py index 011e3b0..17b430d 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -1,5 +1,6 @@ import itertools from twisted.internet import defer +from ooni.utils import log def makeIterable(item): """ @@ -28,6 +29,9 @@ class TaskManager(object): The has failed to complete, we append it to the end of the task chain to be re-run once all the currently scheduled tasks have run. """ + log.err("Task %s has failed" % task) + log.exception(failure) + self._active_tasks.remove(task) self.failures.append((failure, task)) @@ -93,6 +97,8 @@ class TaskManager(object): Takes as argument a single task or a task iterable and appends it to the task generator queue. """ + log.debug("Starting this task %s" % repr(task_or_task_iterator)) + iterable = makeIterable(task_or_task_iterator) self._tasks = itertools.chain(self._tasks, iterable) @@ -138,7 +144,8 @@ class MeasurementManager(TaskManager): concurrency = 10 def succeeded(self, result, measurement): - pass + log.debug("Successfully performed measurement %s" % measurement) + log.debug(result) def failed(self, failure, measurement): pass @@ -149,7 +156,8 @@ class ReportEntryManager(TaskManager): concurrency = 20 def succeeded(self, result, task): - pass + log.debug("Successfully performed report %s" % task) + log.debug(result) def failed(self, failure, task): pass

1 0

[ooni-probe/develop] Make the starting of tests more robust.
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 5abfe2a902b60062f08feb22ef5a47268e51c8b1 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 18:12:13 2013 +0100 Make the starting of tests more robust. --- ooni/nettest.py | 11 +++++++---- ooni/oonicli.py | 18 +++++++++++++----- ooni/utils/geodata.py | 7 +++---- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 0b3e69c..1fe19f1 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -38,7 +38,11 @@ class NetTestLoader(object): config.privacy.includecountry or \ config.privacy.includecity): log.msg("We will include some geo data in the report") - client_geodata = geodata.IPToLocation(config.probe_ip) + try: + client_geodata = geodata.IPToLocation(config.probe_ip) + except e.GeoIPDataFilesNotFound: + log.err("Unable to find the geoip data files") + client_geodata = {'city': None, 'countrycode': None, 'asn': None} if config.privacy.includeip: client_geodata['ip'] = config.probe_ip @@ -51,8 +55,7 @@ class NetTestLoader(object): client_geodata['asn'] = 'AS0' elif 'asn' in client_geodata: # XXX this regexp should probably go inside of geodata - client_geodata['asn'] = \ - re.search('AS\d+', client_geodata['asn']).group(0) + client_geodata['asn'] = client_geodata['asn'] log.msg("Your AS number is: %s" % client_geodata['asn']) else: client_geodata['asn'] = None @@ -306,7 +309,7 @@ class NetTest(object): self.state.taskDone() if len(self.report.reporters) == 0: - raise AllReportersFailed + raise e.AllReportersFailed return report_results diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 73d7709..93bf8bf 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -35,8 +35,8 @@ class Options(usage.Options): optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], - ["collector", "c", None, - "Address of the collector of test results. (example: http://127.0.0.1:8888)"], + ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion', + "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"], ["logfile", "l", None, "log file name"], ["pcapfile", "O", None, "pcap file name"], ["parallelism", "p", "10", "input parallelism"], @@ -130,15 +130,22 @@ def runWithDirector(): director = Director() d = director.start() + def director_startup_failed(failure): + log.err("Failed to start the director") + log.exception(failure) + reactor.stop() + # Wait until director has started up (including bootstrapping Tor) before adding tess def post_director_start(_): for net_test_loader in test_list: - yaml_reporter = YAMLReporter(net_test_loader.testDetails) + test_details = net_test_loader.testDetails + + yaml_reporter = YAMLReporter(test_details) reporters = [yaml_reporter] if global_options['collector']: try: - oonib_reporter = OONIBReporter(net_test_loader.testDetails, + oonib_reporter = OONIBReporter(test_details, global_options['collector']) reporters.append(oonib_reporter) except InvalidOONIBCollectorAddress: @@ -152,7 +159,7 @@ def runWithDirector(): with open('collector') as f: reporter_url = random.choice(f.readlines()) reporter_url = reporter_url.split('#')[0].strip() - oonib_reporter = OONIBReporter(net_test_loader.testDetails, reporter_url) + oonib_reporter = OONIBReporter(test_details, reporter_url) reporters.append(oonib_reporter) log.debug("adding callback for startNetTest") @@ -160,4 +167,5 @@ def runWithDirector(): d.addCallback(shutdown) d.addCallback(post_director_start) + d.addErrback(director_startup_failed) reactor.run() diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index d9883ba..56ce05c 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -6,15 +6,13 @@ from twisted.internet import reactor, defer, protocol from ooni.utils import log, net from ooni import config +from ooni.errors import GeoIPDataFilesNotFound try: import pygeoip except ImportError: log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") -class GeoIPDataFilesNotFound(Exception): - pass - def IPToLocation(ipaddr): city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') @@ -29,7 +27,8 @@ def IPToLocation(ipaddr): location['countrycode'] = country_dat.country_code_by_addr(ipaddr) asn_dat = pygeoip.GeoIP(asn_file) - location['asn'] = asn_dat.org_by_addr(ipaddr) + asn = asn_dat.org_by_addr(ipaddr) + location['asn'] = re.search('AS\d+', asn).group(0) except IOError: log.err("Could not find GeoIP data files. Go into data/ "

1 0

[ooni-probe/develop] Fix typo in test deck before_i_commit
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 18266c634c0538585986c0d31dd929ba83a22203 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 1 11:31:42 2013 +0100 Fix typo in test deck before_i_commit --- decks/before_i_commit.testdeck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decks/before_i_commit.testdeck b/decks/before_i_commit.testdeck index a9e08d8..d1b4062 100644 --- a/decks/before_i_commit.testdeck +++ b/decks/before_i_commit.testdeck @@ -20,7 +20,7 @@ logfile: reports/before_i_commit.log pcapfile: null reportfile: reports/http_host.yamloo - subargs: [-b, 'http://93.95.227.200', -c, 'http://nkvphnp3p6agi5qq.onion', -f, example_inputs/http_host_file.txt] + subargs: [-b, 'http://93.95.227.200', -f, example_inputs/http_host_file.txt] test_file: nettests/manipulation/http_host.py - options: collector: null

1 0

[ooni-probe/develop] Set the default collector via the command line option
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit c714841fc7a818ee7fcbc2f46ab7ccba3f0ee40c Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 28 20:47:05 2013 +0100 Set the default collector via the command line option --- collector | 1 - ooni/oonicli.py | 11 +---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/collector b/collector deleted file mode 100644 index 0b260a1..0000000 --- a/collector +++ /dev/null @@ -1 +0,0 @@ -httpo://nkvphnp3p6agi5qq.onion diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 93bf8bf..af06ad2 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -29,8 +29,7 @@ class Options(usage.Options): " files listed on the command line") optFlags = [["help", "h"], - ["resume", "r"], - ["no-default-reporter", "n"]] + ["resume", "r"]] optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, @@ -154,14 +153,6 @@ def runWithDirector(): log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") sys.exit(1) - # Select one of the baked-in reporters unless the user has requested otherwise - if not global_options['no-default-reporter']: - with open('collector') as f: - reporter_url = random.choice(f.readlines()) - reporter_url = reporter_url.split('#')[0].strip() - oonib_reporter = OONIBReporter(test_details, reporter_url) - reporters.append(oonib_reporter) - log.debug("adding callback for startNetTest") d.addCallback(director.startNetTest, net_test_loader, reporters) d.addCallback(shutdown)

1 0

[ooni-probe/develop] Update test decks with the proper key
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 74d61678afc2804796c844f099c6d7c84766a857 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 28 20:48:47 2013 +0100 Update test decks with the proper key --- decks/before_i_commit.testdeck | 8 ++++---- decks/short_no_root.deck | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/decks/before_i_commit.testdeck b/decks/before_i_commit.testdeck index f177a64..a9e08d8 100644 --- a/decks/before_i_commit.testdeck +++ b/decks/before_i_commit.testdeck @@ -5,7 +5,7 @@ pcapfile: null reportfile: reports/captive_portal_test.yamloo subargs: [] - test: nettests/manipulation/captiveportal.py + test_file: nettests/manipulation/captiveportal.py - options: collector: null help: 0 @@ -13,7 +13,7 @@ pcapfile: null reportfile: reports/dns_tamper_test.yamloo subargs: [-T, example_inputs/dns_tamper_test_resolvers.txt, -f, example_inputs/dns_tamper_file.txt] - test: nettests/blocking/dnsconsistency.py + test_file: nettests/blocking/dnsconsistency.py - options: collector: null help: 0 @@ -21,7 +21,7 @@ pcapfile: null reportfile: reports/http_host.yamloo subargs: [-b, 'http://93.95.227.200', -c, 'http://nkvphnp3p6agi5qq.onion', -f, example_inputs/http_host_file.txt] - test: nettests/manipulation/http_host.py + test_file: nettests/manipulation/http_host.py - options: collector: null help: 0 @@ -29,4 +29,4 @@ pcapfile: null reportfile: reports/header_field_manipulation.yamloo subargs: [-b, 'http://93.95.227.200'] - test: nettests/manipulation/http_header_field_manipulation.py + test_file: nettests/manipulation/http_header_field_manipulation.py diff --git a/decks/short_no_root.deck b/decks/short_no_root.deck index afb7b75..18117f5 100644 --- a/decks/short_no_root.deck +++ b/decks/short_no_root.deck @@ -8,7 +8,7 @@ reportfile: null resume: 0 subargs: [] - test: nettests/manipulation/captiveportal.py + test_file: nettests/manipulation/captiveportal.py testdeck: null - options: collector: null @@ -20,7 +20,7 @@ reportfile: null resume: 0 subargs: [-f, example_inputs/url_lists_file.txt] - test: nettests/blocking/http_requests.py + test_file: nettests/blocking/http_requests.py testdeck: null - options: collector: null @@ -32,7 +32,7 @@ reportfile: null resume: 0 subargs: [-f, example_inputs/tcpconnect_host_file.txt] - test: nettests/blocking/tcpconnect.py + test_file: nettests/blocking/tcpconnect.py testdeck: null - options: collector: null @@ -44,7 +44,7 @@ reportfile: null resume: 0 subargs: [-f, example_inputs/http_host_file.txt, -T, example_inputs/dns_tamper_test_resolvers.txt] - test: nettests/blocking/dnsconsistency.py + test_file: nettests/blocking/dnsconsistency.py testdeck: null - options: collector: null @@ -56,5 +56,5 @@ reportfile: null resume: 0 subargs: [-b, 'http://93.95.227.200'] - test: nettests/manipulation/http_header_field_manipulation.py + test_file: nettests/manipulation/http_header_field_manipulation.py testdeck: null

1 0

[ooni-probe/develop] Merge branch 'master' into feature/task_manager
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 9b00fa993757df5081c042f1deda32f907d235f9 Merge: 46c4db1 a62e36d Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Mar 5 16:33:01 2013 +0100 Merge branch 'master' into feature/task_manager * master: Add changelog for ooni Update ooniprobe.conf.sample Add tests for YAML reporter Open YAML report in createReport Update tests to reflect change of syntax in NetTestLoader Update NetTest tests to reflect change of syntax Conflicts: ooni/errors.py tests/test_director.py tests/test_nettest.py ChangeLog.md | 253 ++++++++++++++++++++++++++++++++++++++++++++++++ ooni/errors.py | 6 ++ ooni/reporter.py | 12 ++- ooniprobe.conf.sample | 4 +- tests/test_managers.py | 2 +- tests/test_reporter.py | 115 +++++++++++++++++----- 6 files changed, 362 insertions(+), 30 deletions(-) diff --cc ooni/errors.py index a194b96,36a042f..85f39ab --- a/ooni/errors.py +++ b/ooni/errors.py @@@ -126,7 -126,9 +126,13 @@@ class InvalidOONIBCollectorAddress(Exce class AllReportersFailed(Exception): pass +class GeoIPDataFilesNotFound(Exception): + pass + + + class ReportNotCreated(Exception): + pass + + class ReportAlreadyClosed(Exception): + pass +

1 0

[ooni-probe/develop] Fix so that parsing of test decks works properly
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 49c442b48ac72420338ede93b2601a74c0f028f5 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 28 21:05:26 2013 +0100 Fix so that parsing of test decks works properly --- ooni/oonicli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index af06ad2..06aa20c 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -106,8 +106,8 @@ def runWithDirector(): if global_options['testdeck']: test_deck = yaml.safe_load(open(global_options['testdeck'])) for test in test_deck: - test_list.append(NetTestLoader(test['options'], - test_file=global_options['test_file'])) + test_list.append(NetTestLoader(test['options']['subargs'], + test_file=test['options']['test_file'])) else: log.debug("No test deck detected") test_list.append(NetTestLoader(global_options['subargs'],

1 0

[ooni-probe/develop] Rename ooniprobe to ooni.
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 033764eeef9b4f27df4512e447638447ff8fc293 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Mar 18 15:38:17 2013 -0400 Rename ooniprobe to ooni. --- bin/ooniprobe | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ooniprobe b/bin/ooniprobe index ba537ab..695b137 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -8,10 +8,10 @@ sys.path[:] = map(os.path.abspath, sys.path) sys.path.insert(0, os.path.abspath(os.getcwd())) # This is a hack to overcome a bug in python -from ooniprobe.utils.hacks import patched_reduce_ex +from ooni.utils.hacks import patched_reduce_ex copy_reg._reduce_ex = patched_reduce_ex # from ooni.oonicli import run # run() -from ooniprobe.oonicli import runWithDirector +from ooni.oonicli import runWithDirector runWithDirector()

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits June 2013