April 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge pull request #461 from TheTorProject/fix/hotfixes
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 16032999b51ed8e2575c64f94c08a0a45787b28c Merge: 52b44ac 9cc891d Author: Joe Landers <joe(a)joelanders.net> Date: Tue Feb 16 16:24:39 2016 +0100 Merge pull request #461 from TheTorProject/fix/hotfixes Pass missing start_tor argument ooni/oonicli.py | 2 ++ 1 file changed, 2 insertions(+)

1 0

[ooni-probe/master] Start implementing support for reporting using JSON
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 086d95180179b2d5e1433fec718a2f8d192ee90a Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 4 17:42:57 2015 +0100 Start implementing support for reporting using JSON Working on #444 --- data/ooniprobe.conf.sample | 1 + ooni/reporter.py | 36 +++++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index 7aaacd7..a7d8d29 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -22,6 +22,7 @@ reports: # This is a prefix for each packet capture file (.pcap) per test: pcap: null collector: null + format: yaml advanced: geoip_data_dir: /usr/share/GeoIP debug: false diff --git a/ooni/reporter.py b/ooni/reporter.py index 42787fb..a8272f5 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -257,22 +257,36 @@ class OONIBReporter(OReporter): if not re.match(regexp, self.collectorAddress): raise errors.InvalidOONIBCollectorAddress + def serializeEntry(self, entry): + if config.report.format == "json": + if isinstance(entry, Measurement): + report_entry = entry.testInstance.report + elif isinstance(entry, Failure): + report_entry = entry.value + elif isinstance(entry, dict): + report_entry = entry + report_entry["record_type"] = "entry" + report_entry["report_id"] = self.reportID + content = json.dumps(report_entry, ensure_ascii=True) + "\n" + else: + content = '---\n' + if isinstance(entry, Measurement): + content += safe_dump(entry.testInstance.report) + elif isinstance(entry, Failure): + content += entry.value + elif isinstance(entry, dict): + content += safe_dump(entry) + content += '...\n' + return content + @defer.inlineCallbacks def writeReportEntry(self, entry): log.debug("Writing report with OONIB reporter") - content = '---\n' - if isinstance(entry, Measurement): - content += safe_dump(entry.testInstance.report) - elif isinstance(entry, Failure): - content += entry.value - elif isinstance(entry, dict): - content += safe_dump(entry) - content += '...\n' - url = self.collectorAddress + '/report' + url = self.collectorAddress + '/report/' + self.reportID - request = {'report_id': self.reportID, - 'content': content} + request = {'format': config.report.format, + 'content': self.serializeEntry(entry)} log.debug("Updating report with id %s (%s)" % (self.reportID, url)) request_json = json.dumps(request)

1 0

[ooni-probe/master] hotfix: Move logic for detecting supported collectors into deck
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 52b44ac93f71a8bddb3b3a154f5517a6abda18e8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Feb 16 13:21:16 2016 +0100 hotfix: Move logic for detecting supported collectors into deck --- ooni/deck.py | 11 +++++++++++ ooni/errors.py | 6 ++++++ ooni/oonicli.py | 24 +++++++++--------------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 83a9a70..4049103 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -6,6 +6,9 @@ from ooni.settings import config from ooni.utils import log from ooni import errors as e +from twisted import version as _twisted_version +from twisted.python.versions import Version + from twisted.python.filepath import FilePath from twisted.internet import defer @@ -158,6 +161,14 @@ class Deck(InputFile): if not has_test_helper(missing_option): raise self.requiresTor = True + + if net_test_loader.collector and net_test_loader.collector.startswith('https://'): + _twisted_14_0_2_version = Version('twisted', 14, 0, 2) + if _twisted_version < _twisted_14_0_2_version: + raise e.HTTPCollectorUnsupported + elif net_test_loader.collector and net_test_loader.collector.startswith('http://'): + if config.advanced.insecure_collector is not True: + raise e.InsecureCollector self.netTestLoaders.append(net_test_loader) @defer.inlineCallbacks diff --git a/ooni/errors.py b/ooni/errors.py index 55b97e1..3b327f7 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -277,3 +277,9 @@ class ProtocolAlreadyRegistered(Exception): class LibraryNotInstalledError(Exception): pass + +class InsecureCollector(Exception): + pass + +class HTTPSCollectorUnsupported(Exception): + pass diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 864f700..1ba70d5 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -6,8 +6,6 @@ import yaml import random import urlparse -from twisted import version as _twisted_version -from twisted.python.versions import Version from twisted.python import usage from twisted.python.util import spewer from twisted.internet import defer, reactor, protocol @@ -262,25 +260,21 @@ def createDeck(global_options, url=None): log.err(e) print e.net_test_loader.usageOptions().getUsage() sys.exit(4) + except errors.HTTPSCollectorUnsupported: + log.err("HTTPS collectors require a twisted version of at least 14.0.2.") + sys.exit(6) + except errors.InsecureCollector: + log.err("Attempting to report to an insecure collector.") + log.err("To enable reporting to insecure collector set the " + "advanced->insecure_collector option to true in " + "your ooniprobe.conf file.") + sys.exit(7) except Exception as e: if config.advanced.debug: log.exception(e) log.err(e) sys.exit(5) - if net_test_loader.collector and net_test_loader.collector.startswith('https://'): - _twisted_14_0_2_version = Version('twisted', 14, 0, 2) - if _twisted_version < _twisted_14_0_2_version: - log.err("HTTPS collectors require a twisted version of at least 14.0.2.") - sys.exit(6) - elif net_test_loader.collector and net_test_loader.collector.startswith('http://'): - if config.advanced.insecure_collector is not True: - log.err("Attempting to report to an insecure collector.") - log.err("To enable reporting to insecure collector set the " - "advanced->insecure_collector option to true in " - "your ooniprobe.conf file.") - sys.exit(7) - return deck

1 0

[ooni-probe/master] Merge branch 'master' of github.com:TheTorProject/ooni-probe
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 2adc991557e39797821e5dd0f66a01ab7ccc2835 Merge: b4f14c0 a9d48f3 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Feb 5 17:09:27 2016 +0100 Merge branch 'master' of github.com:TheTorProject/ooni-probe * 'master' of github.com:TheTorProject/ooni-probe: Fix: Typo correction Consistency fix: lowercase ooni backend/probe words Fix typos in Readme.md README.rst | 6 +++--- docs/source/writing_test_helpers.rst | 26 +++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-)

1 0

[ooni-probe/master] Maintain backward compatibility
by art＠torproject.org 29 Apr '16

29 Apr '16

commit b86eaff915439fea01c15049b2b7252a39dc1afa Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Jan 30 12:36:49 2016 +0100 Maintain backward compatibility --- ooni/nettest.py | 2 +- ooni/oonibclient.py | 3 --- ooni/reporter.py | 42 +++++++++++++++++++++++++++--------------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 5207b8a..e347d55 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -330,7 +330,7 @@ class NetTestLoader(object): self.testClasses = set([]) self.testHelpers = {} - if config.reports.unique_id is True: + if config.reports.unique_id is True and not self.reportID: self.reportID = randomStr(64) for test_class, test_method in self.testCases: diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 126eeb7..388bb68 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -81,7 +81,6 @@ class OONIBClient(object): except ValueError: raise e.get_error(None) if 'error' in response: - print "Got this backend error message %s" % response log.err("Got this backend error message %s" % response) raise e.get_error(response['error']) return response @@ -171,7 +170,6 @@ class OONIBClient(object): @d.addErrback def err(err): log.err("Failed to get descriptor for deck %s" % deck_hash) - print err log.exception(err) return d @@ -193,7 +191,6 @@ class OONIBClient(object): @d.addErrback def err(err): log.err("Failed to download the deck %s" % deck_hash) - print err log.exception(err) return d diff --git a/ooni/reporter.py b/ooni/reporter.py index 3ea1b10..b725540 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -15,7 +15,6 @@ from twisted.python.util import untilConcludes from twisted.internet import defer, reactor from twisted.web.client import Agent from twisted.internet.error import ConnectionRefusedError -from twisted.python.failure import Failure from twisted.internet.endpoints import TCP4ClientEndpoint from txsocksx.http import SOCKS5Agent @@ -190,14 +189,17 @@ class YAMLReporter(OReporter): def writeReportEntry(self, entry): log.debug("Writing report with YAML reporter") - self._write('---\n') + content = '---\n' if isinstance(entry, Measurement): - self._write(safe_dump(entry.testInstance.report)) - elif isinstance(entry, Failure): - self._write(entry.value) + report_entry = entry.testInstance.report elif isinstance(entry, dict): - self._write(safe_dump(entry)) - self._write('...\n') + report_entry = entry + else: + raise Exception("Failed to serialise entry") + content += safe_dump(report_entry) + content += '...\n' + report_entry.update(self.testDetails) + self._write(content) def createReport(self): """ @@ -261,20 +263,28 @@ class OONIBReporter(OReporter): def serializeEntry(self, entry, serialisation_format="yaml"): if serialisation_format == "json": if isinstance(entry, Measurement): - report_entry = entry.testInstance.report - elif isinstance(entry, Failure): - report_entry = {'failure': entry.value} + report_entry = { + 'input': entry.testInstance.report.pop('input', None), + 'test_keys': entry.testInstance.report + } elif isinstance(entry, dict): - report_entry = entry + report_entry = { + 'input': entry.pop('input', None), + 'test_keys': entry + } + else: + raise Exception("Failed to serialise entry") + report_entry.update(self.testDetails) return report_entry else: content = '---\n' if isinstance(entry, Measurement): report_entry = entry.testInstance.report - elif isinstance(entry, Failure): - report_entry = {'failure': entry.value} elif isinstance(entry, dict): report_entry = entry + else: + raise Exception("Failed to serialise entry") + report_entry.update(self.testDetails) content += safe_dump(report_entry) content += '...\n' return content @@ -290,8 +300,10 @@ class OONIBReporter(OReporter): else: serialisation_format = 'yaml' - request = {'format': serialisation_format, - 'content': self.serializeEntry(entry, serialisation_format)} + request = { + 'format': serialisation_format, + 'content': self.serializeEntry(entry, serialisation_format) + } log.debug("Updating report with id %s (%s)" % (self.reportID, url)) request_json = json.dumps(request)

1 0

[ooni-probe/master] hotfix: fix serialisation of reports
by art＠torproject.org 29 Apr '16

29 Apr '16

commit a6e291e913148ee1764655af932573fe4f77fe2c Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Feb 21 15:37:41 2016 +0100 hotfix: fix serialisation of reports --- ooni/report/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/report/parser.py b/ooni/report/parser.py index a986522..bef948c 100644 --- a/ooni/report/parser.py +++ b/ooni/report/parser.py @@ -26,7 +26,7 @@ class ReportLoader(object): def next(self): try: - self._yfp.next() + return self._yfp.next() except StopIteration: self.close() raise StopIteration

1 0

[ooni-probe/master] Start adding support for not reporting to file on disk
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 687a6ffdf9e2fabfeb8711cbbd085d0378fb7a9e Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 4 17:44:04 2015 +0100 Start adding support for not reporting to file on disk --- ooni/director.py | 5 +++-- ooni/nettest.py | 1 + ooni/oonicli.py | 11 ++++++++--- ooni/reporter.py | 38 ++++++++++++++++++++++++-------------- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index f633048..4aa5fba 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -233,7 +233,7 @@ class Director(object): @defer.inlineCallbacks def startNetTest(self, net_test_loader, report_filename, - collector_address=None): + collector_address=None, no_yamloo=False): """ Create the Report for the NetTest and start the report NetTest. @@ -248,7 +248,8 @@ class Director(object): self.startSniffing(net_test_loader.testDetails) report = Report(net_test_loader.testDetails, report_filename, - self.reportEntryManager, collector_address) + self.reportEntryManager, collector_address, + no_yamloo) net_test = NetTest(net_test_loader, report) net_test.director = self diff --git a/ooni/nettest.py b/ooni/nettest.py index 848d958..5207b8a 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -131,6 +131,7 @@ def getNetTestInformation(net_test_file): class NetTestLoader(object): method_prefix = 'test' collector = None + yamloo = True requiresTor = False reportID = None diff --git a/ooni/oonicli.py b/ooni/oonicli.py index d75aa41..b9133c9 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -34,7 +34,8 @@ class Options(usage.Options): optFlags = [["help", "h"], ["resume", "r"], - ["no-collector", "n"], + ["no-collector", "n", "disable writing to collector"], + ["no-yamloo", "N", "disable writing to YAML file"], ["no-geoip", "g"], ["list", "s"], ["printdeck", "p"], @@ -228,6 +229,9 @@ def setupCollector(global_options, net_test_loader): def createDeck(global_options, url=None): log.msg("Creating deck for: %s" % (url)) + if global_options['no-yamloo']: + log.msg("Will not write to a yamloo report file") + deck = Deck(no_collector=global_options['no-collector']) deck.bouncer = global_options['bouncer'] @@ -316,8 +320,9 @@ def runTestWithDirector(director, global_options, url=None, net_test_loader.annotations = global_options['annotations'] director.startNetTest(net_test_loader, - global_options['reportfile'], - collector) + global_options['reportfile'], + collector, + global_options['no-yamloo']) return director.allTestsDone d.addCallback(setup_nettest) diff --git a/ooni/reporter.py b/ooni/reporter.py index a8272f5..db6f7c1 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -554,7 +554,8 @@ class OONIBReportLog(object): class Report(object): def __init__(self, test_details, report_filename, - reportEntryManager, collector_address=None): + reportEntryManager, collector_address=None, + no_yamloo=False): """ This is an abstraction layer on top of all the configured reporters. @@ -574,14 +575,19 @@ class Report(object): collector: The address of the oonib collector for this report. + no_yamloo: + If we should disable reporting to disk. """ self.test_details = test_details self.collector_address = collector_address self.report_log = OONIBReportLog() - self.yaml_reporter = YAMLReporter(test_details, report_filename=report_filename) - self.report_filename = self.yaml_reporter.report_path + self.yaml_reporter = None + self.report_filename = None + if not no_yamloo: + self.yaml_reporter = YAMLReporter(test_details, report_filename=report_filename) + self.report_filename = self.yaml_reporter.report_path self.oonib_reporter = None if collector_address: @@ -627,11 +633,13 @@ class Report(object): if self.oonib_reporter: deferreds.append(self.open_oonib_reporter()) else: - deferreds.append(self.report_log.not_created(self.report_filename)) + if self.yaml_reporter: + deferreds.append(self.report_log.not_created(self.report_filename)) - yaml_report_created = \ - defer.maybeDeferred(self.yaml_reporter.createReport) - yaml_report_created.addErrback(yaml_report_failed) + if self.yaml_reporter: + yaml_report_created = \ + defer.maybeDeferred(self.yaml_reporter.createReport) + yaml_report_created.addErrback(yaml_report_failed) dl = defer.DeferredList(deferreds) dl.addCallback(all_reports_openned) @@ -666,10 +674,11 @@ class Report(object): if not d.called: d.callback(None) - write_yaml_report = ReportEntry(self.yaml_reporter, measurement) - self.reportEntryManager.schedule(write_yaml_report) - write_yaml_report.done.addErrback(yaml_report_failed) - deferreds.append(write_yaml_report.done) + if self.yaml_reporter: + write_yaml_report = ReportEntry(self.yaml_reporter, measurement) + self.reportEntryManager.schedule(write_yaml_report) + write_yaml_report.done.addErrback(yaml_report_failed) + deferreds.append(write_yaml_report.done) if self.oonib_reporter: write_oonib_report = ReportEntry(self.oonib_reporter, measurement) @@ -707,9 +716,10 @@ class Report(object): if not d.called: d.callback(None) - close_yaml = defer.maybeDeferred(self.yaml_reporter.finish) - close_yaml.addErrback(yaml_report_failed) - deferreds.append(close_yaml) + if self.yaml_reporter: + close_yaml = defer.maybeDeferred(self.yaml_reporter.finish) + close_yaml.addErrback(yaml_report_failed) + deferreds.append(close_yaml) if self.oonib_reporter: close_oonib = self.oonib_reporter.finish()

1 0

[ooni-probe/master] Run one test at a time
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 2e88a046d9750b7187a04b9acf6981fb380b6802 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Feb 3 22:54:45 2016 +0100 Run one test at a time --- ooni/oonicli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index b9133c9..9509a18 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -300,6 +300,7 @@ def runTestWithDirector(director, global_options, url=None, # Wait until director has started up (including bootstrapping Tor) # before adding tests + @defer.inlineCallbacks def post_director_start(_): for net_test_loader in deck.netTestLoaders: # Decks can specify different collectors @@ -319,11 +320,10 @@ def runTestWithDirector(director, global_options, url=None, net_test_loader.annotations = global_options['annotations'] - director.startNetTest(net_test_loader, - global_options['reportfile'], - collector, - global_options['no-yamloo']) - return director.allTestsDone + yield director.startNetTest(net_test_loader, + global_options['reportfile'], + collector, + global_options['no-yamloo']) d.addCallback(setup_nettest) d.addCallback(post_director_start)

1 0

[ooni-probe/master] hotfix: fix bug that lead to body not being included
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 3a4d9ef5482f6af884a23a05b9548e816f3bdc18 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Feb 22 18:27:35 2016 +0100 hotfix: fix bug that lead to body not being included --- ooni/templates/httpt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index a753955..3702e86 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -136,9 +136,9 @@ class HTTPTest(NetTestCase): if response: request_response['response'] = { 'headers': list(response.headers.getAllRawHeaders()), - 'body': response_body if self.localOptions.get('withoutbody',1) == 0 else '', + 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '', 'code': response.code - } + } if failure_string: request_response['failure'] = failure_string

1 0

[ooni-probe/master] Improvements to the output of the http_invalid_request_line test
by art＠torproject.org 29 Apr '16

29 Apr '16

commit da4ad2f16e3bd7d87f8d66714452a2746518147c Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Feb 4 16:26:44 2016 +0100 Improvements to the output of the http_invalid_request_line test --- ooni/nettests/manipulation/http_invalid_request_line.py | 3 +++ ooni/templates/httpt.py | 4 ++-- ooni/templates/tcpt.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/manipulation/http_invalid_request_line.py b/ooni/nettests/manipulation/http_invalid_request_line.py index b6ce2da..f81d953 100644 --- a/ooni/nettests/manipulation/http_invalid_request_line.py +++ b/ooni/nettests/manipulation/http_invalid_request_line.py @@ -45,8 +45,11 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): def check_for_manipulation(self, response, payload): log.debug("Checking if %s == %s" % (response, payload)) if response != payload: + log.msg("Detected manipulation!") + log.msg(response) self.report['tampering'] = True else: + log.msg("No manipulation detected.") self.report['tampering'] = self.report['tampering'] | False def test_random_invalid_method(self): diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 15630d5..a49c39b 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -347,9 +347,9 @@ class HTTPTest(NetTestCase): def errback(failure, request): if request['tor']['is_tor']: - log.err("Error performing torified request: %s" % request['url']) + log.err("Error performing torified HTTP request: %s" % request['url']) else: - log.err("Error performing request: %s" % request['url']) + log.err("Error performing HTTP request: %s" % request['url']) failure_string = handleAllFailures(failure) self.addToReport(request, failure_string=failure_string) return failure diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py index bdedebc..5e1d21a 100644 --- a/ooni/templates/tcpt.py +++ b/ooni/templates/tcpt.py @@ -44,6 +44,7 @@ class TCPSender(protocol.Protocol): self.transport.write(payload) class TCPSenderFactory(protocol.Factory): + noisy = False def buildProtocol(self, addr): return TCPSender()

1 0