- tor-commits - lists.torproject.org

[ooni-probe/master] hotfix: fix serialisation of reports
by art＠torproject.org 29 Apr '16

29 Apr '16

commit a6e291e913148ee1764655af932573fe4f77fe2c Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Feb 21 15:37:41 2016 +0100 hotfix: fix serialisation of reports --- ooni/report/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/report/parser.py b/ooni/report/parser.py index a986522..bef948c 100644 --- a/ooni/report/parser.py +++ b/ooni/report/parser.py @@ -26,7 +26,7 @@ class ReportLoader(object): def next(self): try: - self._yfp.next() + return self._yfp.next() except StopIteration: self.close() raise StopIteration

1 0

[ooni-probe/master] Start adding support for not reporting to file on disk
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 687a6ffdf9e2fabfeb8711cbbd085d0378fb7a9e Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 4 17:44:04 2015 +0100 Start adding support for not reporting to file on disk --- ooni/director.py | 5 +++-- ooni/nettest.py | 1 + ooni/oonicli.py | 11 ++++++++--- ooni/reporter.py | 38 ++++++++++++++++++++++++-------------- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index f633048..4aa5fba 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -233,7 +233,7 @@ class Director(object): @defer.inlineCallbacks def startNetTest(self, net_test_loader, report_filename, - collector_address=None): + collector_address=None, no_yamloo=False): """ Create the Report for the NetTest and start the report NetTest. @@ -248,7 +248,8 @@ class Director(object): self.startSniffing(net_test_loader.testDetails) report = Report(net_test_loader.testDetails, report_filename, - self.reportEntryManager, collector_address) + self.reportEntryManager, collector_address, + no_yamloo) net_test = NetTest(net_test_loader, report) net_test.director = self diff --git a/ooni/nettest.py b/ooni/nettest.py index 848d958..5207b8a 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -131,6 +131,7 @@ def getNetTestInformation(net_test_file): class NetTestLoader(object): method_prefix = 'test' collector = None + yamloo = True requiresTor = False reportID = None diff --git a/ooni/oonicli.py b/ooni/oonicli.py index d75aa41..b9133c9 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -34,7 +34,8 @@ class Options(usage.Options): optFlags = [["help", "h"], ["resume", "r"], - ["no-collector", "n"], + ["no-collector", "n", "disable writing to collector"], + ["no-yamloo", "N", "disable writing to YAML file"], ["no-geoip", "g"], ["list", "s"], ["printdeck", "p"], @@ -228,6 +229,9 @@ def setupCollector(global_options, net_test_loader): def createDeck(global_options, url=None): log.msg("Creating deck for: %s" % (url)) + if global_options['no-yamloo']: + log.msg("Will not write to a yamloo report file") + deck = Deck(no_collector=global_options['no-collector']) deck.bouncer = global_options['bouncer'] @@ -316,8 +320,9 @@ def runTestWithDirector(director, global_options, url=None, net_test_loader.annotations = global_options['annotations'] director.startNetTest(net_test_loader, - global_options['reportfile'], - collector) + global_options['reportfile'], + collector, + global_options['no-yamloo']) return director.allTestsDone d.addCallback(setup_nettest) diff --git a/ooni/reporter.py b/ooni/reporter.py index a8272f5..db6f7c1 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -554,7 +554,8 @@ class OONIBReportLog(object): class Report(object): def __init__(self, test_details, report_filename, - reportEntryManager, collector_address=None): + reportEntryManager, collector_address=None, + no_yamloo=False): """ This is an abstraction layer on top of all the configured reporters. @@ -574,14 +575,19 @@ class Report(object): collector: The address of the oonib collector for this report. + no_yamloo: + If we should disable reporting to disk. """ self.test_details = test_details self.collector_address = collector_address self.report_log = OONIBReportLog() - self.yaml_reporter = YAMLReporter(test_details, report_filename=report_filename) - self.report_filename = self.yaml_reporter.report_path + self.yaml_reporter = None + self.report_filename = None + if not no_yamloo: + self.yaml_reporter = YAMLReporter(test_details, report_filename=report_filename) + self.report_filename = self.yaml_reporter.report_path self.oonib_reporter = None if collector_address: @@ -627,11 +633,13 @@ class Report(object): if self.oonib_reporter: deferreds.append(self.open_oonib_reporter()) else: - deferreds.append(self.report_log.not_created(self.report_filename)) + if self.yaml_reporter: + deferreds.append(self.report_log.not_created(self.report_filename)) - yaml_report_created = \ - defer.maybeDeferred(self.yaml_reporter.createReport) - yaml_report_created.addErrback(yaml_report_failed) + if self.yaml_reporter: + yaml_report_created = \ + defer.maybeDeferred(self.yaml_reporter.createReport) + yaml_report_created.addErrback(yaml_report_failed) dl = defer.DeferredList(deferreds) dl.addCallback(all_reports_openned) @@ -666,10 +674,11 @@ class Report(object): if not d.called: d.callback(None) - write_yaml_report = ReportEntry(self.yaml_reporter, measurement) - self.reportEntryManager.schedule(write_yaml_report) - write_yaml_report.done.addErrback(yaml_report_failed) - deferreds.append(write_yaml_report.done) + if self.yaml_reporter: + write_yaml_report = ReportEntry(self.yaml_reporter, measurement) + self.reportEntryManager.schedule(write_yaml_report) + write_yaml_report.done.addErrback(yaml_report_failed) + deferreds.append(write_yaml_report.done) if self.oonib_reporter: write_oonib_report = ReportEntry(self.oonib_reporter, measurement) @@ -707,9 +716,10 @@ class Report(object): if not d.called: d.callback(None) - close_yaml = defer.maybeDeferred(self.yaml_reporter.finish) - close_yaml.addErrback(yaml_report_failed) - deferreds.append(close_yaml) + if self.yaml_reporter: + close_yaml = defer.maybeDeferred(self.yaml_reporter.finish) + close_yaml.addErrback(yaml_report_failed) + deferreds.append(close_yaml) if self.oonib_reporter: close_oonib = self.oonib_reporter.finish()

1 0

[ooni-probe/master] Run one test at a time
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 2e88a046d9750b7187a04b9acf6981fb380b6802 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Feb 3 22:54:45 2016 +0100 Run one test at a time --- ooni/oonicli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index b9133c9..9509a18 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -300,6 +300,7 @@ def runTestWithDirector(director, global_options, url=None, # Wait until director has started up (including bootstrapping Tor) # before adding tests + @defer.inlineCallbacks def post_director_start(_): for net_test_loader in deck.netTestLoaders: # Decks can specify different collectors @@ -319,11 +320,10 @@ def runTestWithDirector(director, global_options, url=None, net_test_loader.annotations = global_options['annotations'] - director.startNetTest(net_test_loader, - global_options['reportfile'], - collector, - global_options['no-yamloo']) - return director.allTestsDone + yield director.startNetTest(net_test_loader, + global_options['reportfile'], + collector, + global_options['no-yamloo']) d.addCallback(setup_nettest) d.addCallback(post_director_start)

1 0

[ooni-probe/master] hotfix: fix bug that lead to body not being included
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 3a4d9ef5482f6af884a23a05b9548e816f3bdc18 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Feb 22 18:27:35 2016 +0100 hotfix: fix bug that lead to body not being included --- ooni/templates/httpt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index a753955..3702e86 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -136,9 +136,9 @@ class HTTPTest(NetTestCase): if response: request_response['response'] = { 'headers': list(response.headers.getAllRawHeaders()), - 'body': response_body if self.localOptions.get('withoutbody',1) == 0 else '', + 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '', 'code': response.code - } + } if failure_string: request_response['failure'] = failure_string

1 0

[ooni-probe/master] Improvements to the output of the http_invalid_request_line test
by art＠torproject.org 29 Apr '16

29 Apr '16

commit da4ad2f16e3bd7d87f8d66714452a2746518147c Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Feb 4 16:26:44 2016 +0100 Improvements to the output of the http_invalid_request_line test --- ooni/nettests/manipulation/http_invalid_request_line.py | 3 +++ ooni/templates/httpt.py | 4 ++-- ooni/templates/tcpt.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/manipulation/http_invalid_request_line.py b/ooni/nettests/manipulation/http_invalid_request_line.py index b6ce2da..f81d953 100644 --- a/ooni/nettests/manipulation/http_invalid_request_line.py +++ b/ooni/nettests/manipulation/http_invalid_request_line.py @@ -45,8 +45,11 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): def check_for_manipulation(self, response, payload): log.debug("Checking if %s == %s" % (response, payload)) if response != payload: + log.msg("Detected manipulation!") + log.msg(response) self.report['tampering'] = True else: + log.msg("No manipulation detected.") self.report['tampering'] = self.report['tampering'] | False def test_random_invalid_method(self): diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 15630d5..a49c39b 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -347,9 +347,9 @@ class HTTPTest(NetTestCase): def errback(failure, request): if request['tor']['is_tor']: - log.err("Error performing torified request: %s" % request['url']) + log.err("Error performing torified HTTP request: %s" % request['url']) else: - log.err("Error performing request: %s" % request['url']) + log.err("Error performing HTTP request: %s" % request['url']) failure_string = handleAllFailures(failure) self.addToReport(request, failure_string=failure_string) return failure diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py index bdedebc..5e1d21a 100644 --- a/ooni/templates/tcpt.py +++ b/ooni/templates/tcpt.py @@ -44,6 +44,7 @@ class TCPSender(protocol.Protocol): self.transport.write(payload) class TCPSenderFactory(protocol.Factory): + noisy = False def buildProtocol(self, addr): return TCPSender()

1 0

[ooni-probe/master] Bump up the required minimum version of Twisted to 13.2.0
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 187ca855cadce110807d9d1f999274a236ac292c Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Feb 5 20:53:11 2016 +0100 Bump up the required minimum version of Twisted to 13.2.0 * This allows us to use some features included in 13.2.0 (such as readBody) and should not be an issues since wheezy-backports has 13.2.0. --- .travis.yml | 4 ---- ooni/tests/test_oonicli.py | 8 +++++--- requirements.txt | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72d7663..acda409 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,5 @@ language: python env: - - TWISTED=Twisted==12.2 - - TWISTED=Twisted==12.3 - - TWISTED=Twisted==13.0 - - TWISTED=Twisted==13.1 - TWISTED=Twisted==13.2 - TWISTED=Twisted==14.0 - TWISTED=Twisted==14.0.1 diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 834fb55..564f9e6 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -8,7 +8,6 @@ from ooni.tests import is_internet_connected from ooni.tests.bases import ConfigTestCase from ooni.settings import config from ooni.oonicli import runWithDirector -from ooni.errors import InsufficientPrivileges from ooni.utils.net import hasRawSocketPermission @@ -140,8 +139,11 @@ class TestRunDirector(ConfigTestCase): def verify_function(entry): assert 'queries' in entry assert 'control_resolver' in entry - assert 'tampering' in entry - assert len(entry['tampering']) == 1 + assert 'errors' in entry + assert 'inconsistent' in entry + assert 'failures' in entry + assert 'successful' in entry + assert len(entry['inconsistent']) == 0 yield self.run_helper('blocking/dns_consistency', ['-b', '8.8.8.8:53', diff --git a/requirements.txt b/requirements.txt index 8b8d5ca..0143b58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML>=3.10 -Twisted>=12.2.0 +Twisted>=13.2.0 ipaddr>=2.1.10 pyOpenSSL>=0.13 geoip

1 0

[ooni-probe/master] Fixes spotted to issues highlighted by @bassosimone in review of PR
by art＠torproject.org 29 Apr '16

29 Apr '16

commit fdd18a673d38161a585cefc91aa70ac2afd5d57a Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 10 18:34:41 2016 +0100 Fixes spotted to issues highlighted by @bassosimone in review of PR Initialise tampering to None and set it False when we detect it not happening Actually properly strip null characters form response body, but only after having tried to decode it Remove un-used YAML format --- data/ooniprobe.conf.sample | 1 - ooni/nettests/manipulation/http_invalid_request_line.py | 4 ++-- ooni/templates/httpt.py | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index a7d8d29..7aaacd7 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -22,7 +22,6 @@ reports: # This is a prefix for each packet capture file (.pcap) per test: pcap: null collector: null - format: yaml advanced: geoip_data_dir: /usr/share/GeoIP debug: false diff --git a/ooni/nettests/manipulation/http_invalid_request_line.py b/ooni/nettests/manipulation/http_invalid_request_line.py index f81d953..2109677 100644 --- a/ooni/nettests/manipulation/http_invalid_request_line.py +++ b/ooni/nettests/manipulation/http_invalid_request_line.py @@ -40,7 +40,7 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): def setUp(self): self.port = int(self.localOptions['backendport']) self.address = self.localOptions['backend'] - self.report['tampering'] = False + self.report['tampering'] = None def check_for_manipulation(self, response, payload): log.debug("Checking if %s == %s" % (response, payload)) @@ -50,7 +50,7 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): self.report['tampering'] = True else: log.msg("No manipulation detected.") - self.report['tampering'] = self.report['tampering'] | False + self.report['tampering'] = False def test_random_invalid_method(self): """ diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 3a36b18..d2cc6e1 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -131,11 +131,12 @@ class HTTPTest(NetTestCase): def _representBody(body): # XXX perhaps add support for decoding gzip in the future. try: - body.replace('\0', '') body = unicode(body, 'ascii') + body = body.replace('\0', '') except UnicodeDecodeError: try: body = unicode(body, 'utf-8') + body = body.replace('\0', '') except UnicodeDecodeError: body = base64Dict(body)

1 0

[ooni-probe/master] Handle more DNS exceptions
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 2b5c77f44c6c25aec9979d931aa72375377c120f Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Feb 4 16:25:54 2016 +0100 Handle more DNS exceptions --- ooni/errors.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ooni/errors.py b/ooni/errors.py index 3b327f7..e523de0 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -5,6 +5,7 @@ from twisted.web.error import Error from twisted.internet.error import ConnectionRefusedError, TCPTimedOutError from twisted.internet.error import DNSLookupError, ConnectError, ConnectionLost +from twisted.names.error import DNSNameError, DNSServerError from twisted.internet.error import TimeoutError as GenericTimeoutError from twisted.internet.error import ProcessDone, ConnectionDone @@ -26,6 +27,8 @@ known_failures = [ (CancelledError, 'task_timed_out'), (gaierror, 'address_family_not_supported_error'), (DNSLookupError, 'dns_lookup_error'), + (DNSNameError, 'dns_name_error'), + (DNSServerError, 'dns_server_failure'), (TCPTimedOutError, 'tcp_timed_out_error'), (ResponseNeverReceived, 'response_never_received'), (DeferTimeoutError, 'deferred_timeout_error'),

1 0

[ooni-probe/master] Fixes to unittests
by art＠torproject.org 29 Apr '16

29 Apr '16

commit ef5e3cbcd1c3a64288f404c3182d15311cb78ddf Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Feb 5 20:03:52 2016 +0100 Fixes to unittests --- ooni/tests/test_oonicli.py | 9 +-------- ooni/tests/test_reporter.py | 3 ++- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 564f9e6..c57205c 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -158,14 +158,7 @@ class TestRunDirector(ConfigTestCase): assert 'agent' in entry assert 'requests' in entry assert 'socksproxy' in entry - assert 'tampering' in entry - assert 'header_field_name' in entry['tampering'] - assert 'header_field_number' in entry['tampering'] - assert 'header_field_value' in entry['tampering'] - assert 'header_name_capitalization' in entry['tampering'] - assert 'header_name_diff' in entry['tampering'] - assert 'request_line_capitalization' in entry['tampering'] - assert 'total' in entry['tampering'] + assert 'errors' in entry yield self.run_helper('manipulation/http_header_field_manipulation', ['-b', 'http://4.15.35.157:80'], diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index f1f7c3a..4b99584 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -26,7 +26,8 @@ test_details = { 'input_hashes': [], 'probe_asn': 'AS0', 'probe_cc': 'ZZ', - 'start_time': time.time() + 'start_time': time.time(), + 'data_format_version': '0.2.0' } oonib_new_report_message = {

1 0

[ooni-probe/master] Integrate withoutbody optflag lost in rebase
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 31c3a81dba626d3defed46213c7e6d6bfde632d6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Feb 5 17:47:29 2016 +0100 Integrate withoutbody optflag lost in rebase --- ooni/nettests/blocking/http_requests.py | 2 +- ooni/templates/httpt.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index f21b041..e00b4c6 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -18,7 +18,7 @@ class UsageOptions(usage.Options): ['factor', 'f', 0.8, 'What factor should be used for triggering censorship (0.8 == 80%)']] optFlags = [ - ["withoutbody","B", "don't include HTTP response body inside of the report"], + ["withoutbody", "B", "don't include HTTP response body inside of the report"], ] diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index a49c39b..3a36b18 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -151,9 +151,13 @@ class HTTPTest(NetTestCase): } } if response: + if self.localOptions.get('withoutbody', 0) is 0: + response_body = _representBody(response_body) + else: + response_body = '' session['response'] = { 'headers': _representHeaders(response.headers), - 'body': _representBody(response_body), + 'body': response_body, 'code': response.code } session['failure'] = None

1 0