- tor-commits - lists.torproject.org

[ooni-probe/master] Properly represent also the packets that are sent
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 8215d8807b3f6c43c766baf4a04f454d83038eea Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Apr 29 10:51:08 2016 +0200 Properly represent also the packets that are sent --- ooni/templates/scapyt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index f4b6280..2bb9139 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -148,7 +148,7 @@ class BaseScapyTest(NetTestCase): scapySender.stopSending() for sent_packet in packets: - self.report['sent_packets'].append(sent_packet) + self.report['sent_packets'].append(_representPacket(sent_packet)) ScapyTest = BaseScapyTest

1 0

[ooni-probe/master] Check to ensure response_body is either a str or unicode before sanitising it (#485)
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 4598237c15331cf97fb99b53593b99e8fb182ee7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 28 12:10:15 2016 +0200 Check to ensure response_body is either a str or unicode before sanitising it (#485) --- ooni/templates/httpt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index fd6c8ab..e608f8f 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -172,7 +172,8 @@ class HTTPTest(NetTestCase): else: response_body = '' # Attempt to redact the IP address of the probe from the responses - if config.privacy.includeip is False and config.probe_ip.address is not None: + if (config.privacy.includeip is False and config.probe_ip.address is not None and + (isinstance(response_body, str) or isinstance(response_body, unicode))): response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") session['response'] = { 'headers': _representHeaders(response.headers),

1 0

[ooni-probe/master] Update Changelog and put a release date on it.
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 3ca8cb86c732fef3d3c27221b0025ad953465e70 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Apr 26 23:22:39 2016 +0200 Update Changelog and put a release date on it. --- ChangeLog.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 47851e1..82d04cd 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,7 +1,7 @@ Changelog ========= -v1.4.0 (XXX, XX Apr 2016) +v1.4.0 (Wed, 27 Apr 2016) ------------------------- * Support for reporting using JSON @@ -12,11 +12,16 @@ v1.4.0 (XXX, XX Apr 2016) * OpenVPN censorship circumvention test +* Add test for vanilla Tor + * Support for disabling reporting to disk * Improvements to HTTP response body decoding (includes fix that lead to empty bodies being misrepresented) +* Attempt to scrub the probe IP address from the body of HTTP responses + + v1.3.2 (Fri, 20 Nov 2015) -------------------------

1 0

[ooni-probe/master] The upload of the release failed and hence need to re-issue upload
by art＠torproject.org 29 Apr '16

29 Apr '16

commit bff00ebbf32730f0d2a44a41e40a7c5e3b0027c0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Apr 27 20:27:24 2016 +0200 The upload of the release failed and hence need to re-issue upload --- ChangeLog.rst | 6 ++++++ ooni/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 471e282..c24d46d 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,12 @@ Changelog ========= +v1.4.1 (Wed, 27 Apr 2016) +------------------------- + +* Fix problem with uploading of release + + v1.4.0 (Wed, 27 Apr 2016) ------------------------- diff --git a/ooni/__init__.py b/ooni/__init__.py index 5ccd2da..cbf84b7 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.4.0" +__version__ = "1.4.1" __all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'report', 'reporter',

1 0

[ooni-probe/master] Use the official scapy repository instead of the dev fork
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 8af20fbfad44ef87564dbb4c421d539f4aa76016 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Apr 29 11:26:28 2016 +0200 Use the official scapy repository instead of the dev fork --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7aeaabc..7107d9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ pyOpenSSL>=0.13 geoip txtorcon>=0.7 txsocksx>=0.0.2 -scapy-real>=2.2.0-dev +scapy>=2.2.0 pypcap>=1.1 service-identity pydumbnet

1 0

[ooni-probe/master] Set the input to be the specified URL in the http_requests test
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 76298b89ba341380ec4d1ab8e640eb7a4a9ca531 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 30 14:07:43 2016 +0200 Set the input to be the specified URL in the http_requests test --- ooni/nettests/blocking/http_requests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index e2c9671..abc7efc 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -68,6 +68,7 @@ class HTTPRequestsTest(httpt.HTTPTest): self.url = self.localOptions['url'] else: raise Exception("No input specified") + self.report['input'] = self.url self.factor = self.localOptions['factor'] self.report['control_failure'] = None

1 0

[ooni-probe/master] Fix bug that lead to the probe_asn and probe_cc not being set
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 29e63ed80eb9298e3ce86f7d4582b2c6ee909afe Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 30 14:07:22 2016 +0200 Fix bug that lead to the probe_asn and probe_cc not being set --- ooni/director.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ooni/director.py b/ooni/director.py index 4aa5fba..ab8da65 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -241,12 +241,16 @@ class Director(object): net_test_loader: an instance of :class:ooni.nettest.NetTestLoader """ + # Here we set the test details again since the geoip lookups may + # not have already been done and probe_asn and probe_ip + # are not set. + net_test_loader.setTestDetails() + if self.allTestsDone.called: self.allTestsDone = defer.Deferred() if config.privacy.includepcap: self.startSniffing(net_test_loader.testDetails) - report = Report(net_test_loader.testDetails, report_filename, self.reportEntryManager, collector_address, no_yamloo)

1 0

[ooni-probe/master] Add test for YAML based reporter
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 1e9f2316859825343ae2ae2f28b6fd5ff2b4266b Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Mar 26 19:48:00 2016 +0100 Add test for YAML based reporter --- ooni/tests/test_reporter.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 94192ee..9774449 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -36,6 +36,13 @@ oonib_new_report_message = { 'supported_formats': ["yaml", "json"] } +# This is used for testing legacy collectors +oonib_new_report_yaml_message = { + 'report_id': "20140129T202038Z_AS0_" + "A" * 50, + 'backend_version': "1.0" +} + + oonib_generic_error_message = { 'error': 'generic-error' } @@ -103,6 +110,13 @@ class TestOONIBReporter(unittest.TestCase): yield self.oonib_reporter.writeReportEntry(req) assert self.oonib_reporter.agent.request.called + @defer.inlineCallbacks + def test_write_report_entry_in_yaml(self): + self.mock_response = oonib_new_report_yaml_message + yield self.oonib_reporter.createReport() + req = {'content': 'something'} + yield self.oonib_reporter.writeReportEntry(req) + assert self.oonib_reporter.agent.request.called class TestOONIBReportLog(unittest.TestCase):

1 0

[ooni-probe/master] [Refactoring] oonicli logic
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 18385338507bf4c3f5959e0de68a3ad33f291353 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 30 14:08:01 2016 +0200 [Refactoring] oonicli logic --- ooni/oonicli.py | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 9509a18..63312b5 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -209,25 +209,22 @@ def setupAnnotations(global_options): global_options["annotations"] = annotations return annotations -def setupCollector(global_options, net_test_loader): - collector = None - if not global_options['no-collector']: - if global_options['collector']: - collector = global_options['collector'] - elif 'collector' in config.reports \ - and config.reports['collector']: - collector = config.reports['collector'] - elif net_test_loader.collector: - collector = net_test_loader.collector - - if collector and collector.startswith('httpo:') \ +def setupCollector(global_options, collector_address): + if global_options['collector']: + collector_address = global_options['collector'] + elif 'collector' in config.reports \ + and config.reports['collector']: + collector_address = config.reports['collector'] + + if collector_address.startswith('httpo:') \ and (not (config.tor_state or config.tor.socks_port)): raise errors.TorNotRunning - return collector + return collector_address def createDeck(global_options, url=None): - log.msg("Creating deck for: %s" % (url)) + if url: + log.msg("Creating deck for: %s" % (url)) if global_options['no-yamloo']: log.msg("Will not write to a yamloo report file") @@ -311,18 +308,16 @@ def runTestWithDirector(director, global_options, url=None, # If a collector is not specified in the deck, or the # deck is a singleton, the default collector set in # ooniprobe.conf will be used - - collector = setupCollector(global_options, net_test_loader) - - if collector and collector.startswith('httpo:') \ - and (not (config.tor_state or config.tor.socks_port)): - raise errors.TorNotRunning + collector_address = None + if not global_options['no-collector']: + collector_address = setupCollector(global_options, + net_test_loader.collector) net_test_loader.annotations = global_options['annotations'] yield director.startNetTest(net_test_loader, global_options['reportfile'], - collector, + collector_address, global_options['no-yamloo']) d.addCallback(setup_nettest)

1 0

[ooni-probe/master] Merge branch 'master' into feature/json_reports
by art＠torproject.org 29 Apr '16

29 Apr '16

commit dd58c40cb351b4d9d1b33fbef08919a8c225fad7 Merge: 1838533 a0fc5e6 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon Apr 4 17:58:05 2016 +0200 Merge branch 'master' into feature/json_reports README.rst | 41 ++++++--------------------------- docs/source/tests/psiphon.rst | 4 ++-- ooni/nettest.py | 3 +++ ooni/nettests/manipulation/http_host.py | 2 -- ooni/tasks.py | 1 - ooni/templates/httpt.py | 4 +--- 6 files changed, 13 insertions(+), 42 deletions(-) diff --cc ooni/templates/httpt.py index d2cc6e1,2f1136c..b9ca2e4 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@@ -152,20 -134,14 +152,19 @@@ class HTTPTest(NetTestCase) } } if response: - request_response['response'] = { - 'headers': list(response.headers.getAllRawHeaders()), - 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '', + if self.localOptions.get('withoutbody', 0) is 0: + response_body = _representBody(response_body) + else: + response_body = '' + session['response'] = { + 'headers': _representHeaders(response.headers), + 'body': response_body, 'code': response.code - } + } + session['failure'] = None if failure_string: - request_response['failure'] = failure_string - self.report['requests'].append(request_response) + session['failure'] = failure_string - + self.report['requests'].append(session) def _processResponseBody(self, response_body, request, response, body_processor): log.debug("Processing response body")

1 0