June 2013 - tor-commits - lists.torproject.org

[ooni-probe/develop] Raise NoMoreReporters exception on failure
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 71d8151930c287c4b0e7f17940cb2954ac171f83 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Mar 6 15:24:56 2013 +0100 Raise NoMoreReporters exception on failure --- ooni/errors.py | 4 +++- ooni/reporter.py | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 85f39ab..ca2211a 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -129,10 +129,12 @@ class AllReportersFailed(Exception): class GeoIPDataFilesNotFound(Exception): pass - class ReportNotCreated(Exception): pass class ReportAlreadyClosed(Exception): pass +class NoMoreReporters(Exception): + pass + diff --git a/ooni/reporter.py b/ooni/reporter.py index 594c6b8..84dad2f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -413,8 +413,7 @@ class Report(object): # Don't forward the exception unless there are no more reporters if len(self.reporters) == 0: log.err("Removed last reporter %s" % reporter) - failure.reporter = reporter - return failure + raise NoMoreReporters def write(self, measurement): """

1 0

[ooni-probe/develop] Merge branch 'master' into feature/task_manager
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 46c4db194d2e0cda648a500291f0d605dd445e96 Merge: 18266c6 997a747 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Mar 5 16:21:24 2013 +0100 Merge branch 'master' into feature/task_manager * master: Remove unneeded commented out line Flatten the reporter keys Return something in txscapy Update before_i_commit script to make it work (cherry picked from commit 9cdf5596c8f16cac819afdaf1b9b537b36f796a6) Refactor logic for importing scapy related functions Add sent packets to report and use startSending instead of sendPackets Conflicts: decks/before_i_commit.testdeck ooni/reporter.py | 15 +++++------- ooni/templates/scapyt.py | 6 ++--- ooni/utils/net.py | 2 +- ooni/utils/txscapy.py | 56 +++++++++++++++++++++++++++++++++------------ scripts/example_parser.py | 5 ++-- 5 files changed, 54 insertions(+), 30 deletions(-) diff --cc ooni/reporter.py index 69d4892,1f6c3e6..a0c9466 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -147,19 -183,13 +147,16 @@@ class OReporter(object) else: test_input = test.input - test_started = test._start_time - test_runtime = time.time() - test_started - - report = {'input': test_input, - 'test_name': test_name, - 'test_started': test_started, - 'test_runtime': test_runtime, - 'report': test_report} - return defer.maybeDeferred(self.writeReportEntry, report) + test_report['input'] = test_input + test_report['test_name'] = test_name + test_report['test_started'] = test._start_time + test_report['test_runtime'] = time.time() - test._start_time + + return defer.maybeDeferred(self.writeReportEntry, test_report) +class InvalidDestination(ReporterException): + pass + class YAMLReporter(OReporter): """ These are useful functions for reporting to YAML format.

1 0

[ooni-probe/develop] Rename ooniprobe to ooni.
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 033764eeef9b4f27df4512e447638447ff8fc293 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Mar 18 15:38:17 2013 -0400 Rename ooniprobe to ooni. --- bin/ooniprobe | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ooniprobe b/bin/ooniprobe index ba537ab..695b137 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -8,10 +8,10 @@ sys.path[:] = map(os.path.abspath, sys.path) sys.path.insert(0, os.path.abspath(os.getcwd())) # This is a hack to overcome a bug in python -from ooniprobe.utils.hacks import patched_reduce_ex +from ooni.utils.hacks import patched_reduce_ex copy_reg._reduce_ex = patched_reduce_ex # from ooni.oonicli import run # run() -from ooniprobe.oonicli import runWithDirector +from ooni.oonicli import runWithDirector runWithDirector()

1 0

[ooni-probe/develop] Fix so that parsing of test decks works properly
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 49c442b48ac72420338ede93b2601a74c0f028f5 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 28 21:05:26 2013 +0100 Fix so that parsing of test decks works properly --- ooni/oonicli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index af06ad2..06aa20c 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -106,8 +106,8 @@ def runWithDirector(): if global_options['testdeck']: test_deck = yaml.safe_load(open(global_options['testdeck'])) for test in test_deck: - test_list.append(NetTestLoader(test['options'], - test_file=global_options['test_file'])) + test_list.append(NetTestLoader(test['options']['subargs'], + test_file=test['options']['test_file'])) else: log.debug("No test deck detected") test_list.append(NetTestLoader(global_options['subargs'],

1 0

[ooni-probe/develop] Merge branch 'master' into feature/task_manager
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 9b00fa993757df5081c042f1deda32f907d235f9 Merge: 46c4db1 a62e36d Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Mar 5 16:33:01 2013 +0100 Merge branch 'master' into feature/task_manager * master: Add changelog for ooni Update ooniprobe.conf.sample Add tests for YAML reporter Open YAML report in createReport Update tests to reflect change of syntax in NetTestLoader Update NetTest tests to reflect change of syntax Conflicts: ooni/errors.py tests/test_director.py tests/test_nettest.py ChangeLog.md | 253 ++++++++++++++++++++++++++++++++++++++++++++++++ ooni/errors.py | 6 ++ ooni/reporter.py | 12 ++- ooniprobe.conf.sample | 4 +- tests/test_managers.py | 2 +- tests/test_reporter.py | 115 +++++++++++++++++----- 6 files changed, 362 insertions(+), 30 deletions(-) diff --cc ooni/errors.py index a194b96,36a042f..85f39ab --- a/ooni/errors.py +++ b/ooni/errors.py @@@ -126,7 -126,9 +126,13 @@@ class InvalidOONIBCollectorAddress(Exce class AllReportersFailed(Exception): pass +class GeoIPDataFilesNotFound(Exception): + pass + + + class ReportNotCreated(Exception): + pass + + class ReportAlreadyClosed(Exception): + pass +

1 0

[ooni-probe/develop] Update test decks with the proper key
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 74d61678afc2804796c844f099c6d7c84766a857 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 28 20:48:47 2013 +0100 Update test decks with the proper key --- decks/before_i_commit.testdeck | 8 ++++---- decks/short_no_root.deck | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/decks/before_i_commit.testdeck b/decks/before_i_commit.testdeck index f177a64..a9e08d8 100644 --- a/decks/before_i_commit.testdeck +++ b/decks/before_i_commit.testdeck @@ -5,7 +5,7 @@ pcapfile: null reportfile: reports/captive_portal_test.yamloo subargs: [] - test: nettests/manipulation/captiveportal.py + test_file: nettests/manipulation/captiveportal.py - options: collector: null help: 0 @@ -13,7 +13,7 @@ pcapfile: null reportfile: reports/dns_tamper_test.yamloo subargs: [-T, example_inputs/dns_tamper_test_resolvers.txt, -f, example_inputs/dns_tamper_file.txt] - test: nettests/blocking/dnsconsistency.py + test_file: nettests/blocking/dnsconsistency.py - options: collector: null help: 0 @@ -21,7 +21,7 @@ pcapfile: null reportfile: reports/http_host.yamloo subargs: [-b, 'http://93.95.227.200', -c, 'http://nkvphnp3p6agi5qq.onion', -f, example_inputs/http_host_file.txt] - test: nettests/manipulation/http_host.py + test_file: nettests/manipulation/http_host.py - options: collector: null help: 0 @@ -29,4 +29,4 @@ pcapfile: null reportfile: reports/header_field_manipulation.yamloo subargs: [-b, 'http://93.95.227.200'] - test: nettests/manipulation/http_header_field_manipulation.py + test_file: nettests/manipulation/http_header_field_manipulation.py diff --git a/decks/short_no_root.deck b/decks/short_no_root.deck index afb7b75..18117f5 100644 --- a/decks/short_no_root.deck +++ b/decks/short_no_root.deck @@ -8,7 +8,7 @@ reportfile: null resume: 0 subargs: [] - test: nettests/manipulation/captiveportal.py + test_file: nettests/manipulation/captiveportal.py testdeck: null - options: collector: null @@ -20,7 +20,7 @@ reportfile: null resume: 0 subargs: [-f, example_inputs/url_lists_file.txt] - test: nettests/blocking/http_requests.py + test_file: nettests/blocking/http_requests.py testdeck: null - options: collector: null @@ -32,7 +32,7 @@ reportfile: null resume: 0 subargs: [-f, example_inputs/tcpconnect_host_file.txt] - test: nettests/blocking/tcpconnect.py + test_file: nettests/blocking/tcpconnect.py testdeck: null - options: collector: null @@ -44,7 +44,7 @@ reportfile: null resume: 0 subargs: [-f, example_inputs/http_host_file.txt, -T, example_inputs/dns_tamper_test_resolvers.txt] - test: nettests/blocking/dnsconsistency.py + test_file: nettests/blocking/dnsconsistency.py testdeck: null - options: collector: null @@ -56,5 +56,5 @@ reportfile: null resume: 0 subargs: [-b, 'http://93.95.227.200'] - test: nettests/manipulation/http_header_field_manipulation.py + test_file: nettests/manipulation/http_header_field_manipulation.py testdeck: null

1 0

[ooni-probe/develop] Set the default collector via the command line option
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit c714841fc7a818ee7fcbc2f46ab7ccba3f0ee40c Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 28 20:47:05 2013 +0100 Set the default collector via the command line option --- collector | 1 - ooni/oonicli.py | 11 +---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/collector b/collector deleted file mode 100644 index 0b260a1..0000000 --- a/collector +++ /dev/null @@ -1 +0,0 @@ -httpo://nkvphnp3p6agi5qq.onion diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 93bf8bf..af06ad2 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -29,8 +29,7 @@ class Options(usage.Options): " files listed on the command line") optFlags = [["help", "h"], - ["resume", "r"], - ["no-default-reporter", "n"]] + ["resume", "r"]] optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, @@ -154,14 +153,6 @@ def runWithDirector(): log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") sys.exit(1) - # Select one of the baked-in reporters unless the user has requested otherwise - if not global_options['no-default-reporter']: - with open('collector') as f: - reporter_url = random.choice(f.readlines()) - reporter_url = reporter_url.split('#')[0].strip() - oonib_reporter = OONIBReporter(test_details, reporter_url) - reporters.append(oonib_reporter) - log.debug("adding callback for startNetTest") d.addCallback(director.startNetTest, net_test_loader, reporters) d.addCallback(shutdown)

1 0

[ooni-probe/develop] Fix typo in test deck before_i_commit
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 18266c634c0538585986c0d31dd929ba83a22203 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 1 11:31:42 2013 +0100 Fix typo in test deck before_i_commit --- decks/before_i_commit.testdeck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decks/before_i_commit.testdeck b/decks/before_i_commit.testdeck index a9e08d8..d1b4062 100644 --- a/decks/before_i_commit.testdeck +++ b/decks/before_i_commit.testdeck @@ -20,7 +20,7 @@ logfile: reports/before_i_commit.log pcapfile: null reportfile: reports/http_host.yamloo - subargs: [-b, 'http://93.95.227.200', -c, 'http://nkvphnp3p6agi5qq.onion', -f, example_inputs/http_host_file.txt] + subargs: [-b, 'http://93.95.227.200', -f, example_inputs/http_host_file.txt] test_file: nettests/manipulation/http_host.py - options: collector: null

1 0

[ooni-probe/develop] Make the starting of tests more robust.
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 5abfe2a902b60062f08feb22ef5a47268e51c8b1 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 27 18:12:13 2013 +0100 Make the starting of tests more robust. --- ooni/nettest.py | 11 +++++++---- ooni/oonicli.py | 18 +++++++++++++----- ooni/utils/geodata.py | 7 +++---- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 0b3e69c..1fe19f1 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -38,7 +38,11 @@ class NetTestLoader(object): config.privacy.includecountry or \ config.privacy.includecity): log.msg("We will include some geo data in the report") - client_geodata = geodata.IPToLocation(config.probe_ip) + try: + client_geodata = geodata.IPToLocation(config.probe_ip) + except e.GeoIPDataFilesNotFound: + log.err("Unable to find the geoip data files") + client_geodata = {'city': None, 'countrycode': None, 'asn': None} if config.privacy.includeip: client_geodata['ip'] = config.probe_ip @@ -51,8 +55,7 @@ class NetTestLoader(object): client_geodata['asn'] = 'AS0' elif 'asn' in client_geodata: # XXX this regexp should probably go inside of geodata - client_geodata['asn'] = \ - re.search('AS\d+', client_geodata['asn']).group(0) + client_geodata['asn'] = client_geodata['asn'] log.msg("Your AS number is: %s" % client_geodata['asn']) else: client_geodata['asn'] = None @@ -306,7 +309,7 @@ class NetTest(object): self.state.taskDone() if len(self.report.reporters) == 0: - raise AllReportersFailed + raise e.AllReportersFailed return report_results diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 73d7709..93bf8bf 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -35,8 +35,8 @@ class Options(usage.Options): optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], - ["collector", "c", None, - "Address of the collector of test results. (example: http://127.0.0.1:8888)"], + ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion', + "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"], ["logfile", "l", None, "log file name"], ["pcapfile", "O", None, "pcap file name"], ["parallelism", "p", "10", "input parallelism"], @@ -130,15 +130,22 @@ def runWithDirector(): director = Director() d = director.start() + def director_startup_failed(failure): + log.err("Failed to start the director") + log.exception(failure) + reactor.stop() + # Wait until director has started up (including bootstrapping Tor) before adding tess def post_director_start(_): for net_test_loader in test_list: - yaml_reporter = YAMLReporter(net_test_loader.testDetails) + test_details = net_test_loader.testDetails + + yaml_reporter = YAMLReporter(test_details) reporters = [yaml_reporter] if global_options['collector']: try: - oonib_reporter = OONIBReporter(net_test_loader.testDetails, + oonib_reporter = OONIBReporter(test_details, global_options['collector']) reporters.append(oonib_reporter) except InvalidOONIBCollectorAddress: @@ -152,7 +159,7 @@ def runWithDirector(): with open('collector') as f: reporter_url = random.choice(f.readlines()) reporter_url = reporter_url.split('#')[0].strip() - oonib_reporter = OONIBReporter(net_test_loader.testDetails, reporter_url) + oonib_reporter = OONIBReporter(test_details, reporter_url) reporters.append(oonib_reporter) log.debug("adding callback for startNetTest") @@ -160,4 +167,5 @@ def runWithDirector(): d.addCallback(shutdown) d.addCallback(post_director_start) + d.addErrback(director_startup_failed) reactor.run() diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index d9883ba..56ce05c 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -6,15 +6,13 @@ from twisted.internet import reactor, defer, protocol from ooni.utils import log, net from ooni import config +from ooni.errors import GeoIPDataFilesNotFound try: import pygeoip except ImportError: log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") -class GeoIPDataFilesNotFound(Exception): - pass - def IPToLocation(ipaddr): city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') @@ -29,7 +27,8 @@ def IPToLocation(ipaddr): location['countrycode'] = country_dat.country_code_by_addr(ipaddr) asn_dat = pygeoip.GeoIP(asn_file) - location['asn'] = asn_dat.org_by_addr(ipaddr) + asn = asn_dat.org_by_addr(ipaddr) + location['asn'] = re.search('AS\d+', asn).group(0) except IOError: log.err("Could not find GeoIP data files. Go into data/ "

1 0

[ooni-probe/develop] Add more debugging to managers
by isis＠torproject.org 26 Jun '13

26 Jun '13

commit 708a3a8a858ef3d3a6ae15d4700b3362f02d9722 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 28 20:46:14 2013 +0100 Add more debugging to managers --- ooni/managers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ooni/managers.py b/ooni/managers.py index 011e3b0..17b430d 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -1,5 +1,6 @@ import itertools from twisted.internet import defer +from ooni.utils import log def makeIterable(item): """ @@ -28,6 +29,9 @@ class TaskManager(object): The has failed to complete, we append it to the end of the task chain to be re-run once all the currently scheduled tasks have run. """ + log.err("Task %s has failed" % task) + log.exception(failure) + self._active_tasks.remove(task) self.failures.append((failure, task)) @@ -93,6 +97,8 @@ class TaskManager(object): Takes as argument a single task or a task iterable and appends it to the task generator queue. """ + log.debug("Starting this task %s" % repr(task_or_task_iterator)) + iterable = makeIterable(task_or_task_iterator) self._tasks = itertools.chain(self._tasks, iterable) @@ -138,7 +144,8 @@ class MeasurementManager(TaskManager): concurrency = 10 def succeeded(self, result, measurement): - pass + log.debug("Successfully performed measurement %s" % measurement) + log.debug(result) def failed(self, failure, measurement): pass @@ -149,7 +156,8 @@ class ReportEntryManager(TaskManager): concurrency = 20 def succeeded(self, result, task): - pass + log.debug("Successfully performed report %s" % task) + log.debug(result) def failed(self, failure, task): pass

1 0