April 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Bump up the required minimum version of Twisted to 13.2.0
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 187ca855cadce110807d9d1f999274a236ac292c Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Feb 5 20:53:11 2016 +0100 Bump up the required minimum version of Twisted to 13.2.0 * This allows us to use some features included in 13.2.0 (such as readBody) and should not be an issues since wheezy-backports has 13.2.0. --- .travis.yml | 4 ---- ooni/tests/test_oonicli.py | 8 +++++--- requirements.txt | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72d7663..acda409 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,5 @@ language: python env: - - TWISTED=Twisted==12.2 - - TWISTED=Twisted==12.3 - - TWISTED=Twisted==13.0 - - TWISTED=Twisted==13.1 - TWISTED=Twisted==13.2 - TWISTED=Twisted==14.0 - TWISTED=Twisted==14.0.1 diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 834fb55..564f9e6 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -8,7 +8,6 @@ from ooni.tests import is_internet_connected from ooni.tests.bases import ConfigTestCase from ooni.settings import config from ooni.oonicli import runWithDirector -from ooni.errors import InsufficientPrivileges from ooni.utils.net import hasRawSocketPermission @@ -140,8 +139,11 @@ class TestRunDirector(ConfigTestCase): def verify_function(entry): assert 'queries' in entry assert 'control_resolver' in entry - assert 'tampering' in entry - assert len(entry['tampering']) == 1 + assert 'errors' in entry + assert 'inconsistent' in entry + assert 'failures' in entry + assert 'successful' in entry + assert len(entry['inconsistent']) == 0 yield self.run_helper('blocking/dns_consistency', ['-b', '8.8.8.8:53', diff --git a/requirements.txt b/requirements.txt index 8b8d5ca..0143b58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML>=3.10 -Twisted>=12.2.0 +Twisted>=13.2.0 ipaddr>=2.1.10 pyOpenSSL>=0.13 geoip

1 0

[ooni-probe/master] Fixes spotted to issues highlighted by @bassosimone in review of PR
by art＠torproject.org 29 Apr '16

29 Apr '16

commit fdd18a673d38161a585cefc91aa70ac2afd5d57a Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 10 18:34:41 2016 +0100 Fixes spotted to issues highlighted by @bassosimone in review of PR Initialise tampering to None and set it False when we detect it not happening Actually properly strip null characters form response body, but only after having tried to decode it Remove un-used YAML format --- data/ooniprobe.conf.sample | 1 - ooni/nettests/manipulation/http_invalid_request_line.py | 4 ++-- ooni/templates/httpt.py | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index a7d8d29..7aaacd7 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -22,7 +22,6 @@ reports: # This is a prefix for each packet capture file (.pcap) per test: pcap: null collector: null - format: yaml advanced: geoip_data_dir: /usr/share/GeoIP debug: false diff --git a/ooni/nettests/manipulation/http_invalid_request_line.py b/ooni/nettests/manipulation/http_invalid_request_line.py index f81d953..2109677 100644 --- a/ooni/nettests/manipulation/http_invalid_request_line.py +++ b/ooni/nettests/manipulation/http_invalid_request_line.py @@ -40,7 +40,7 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): def setUp(self): self.port = int(self.localOptions['backendport']) self.address = self.localOptions['backend'] - self.report['tampering'] = False + self.report['tampering'] = None def check_for_manipulation(self, response, payload): log.debug("Checking if %s == %s" % (response, payload)) @@ -50,7 +50,7 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): self.report['tampering'] = True else: log.msg("No manipulation detected.") - self.report['tampering'] = self.report['tampering'] | False + self.report['tampering'] = False def test_random_invalid_method(self): """ diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 3a36b18..d2cc6e1 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -131,11 +131,12 @@ class HTTPTest(NetTestCase): def _representBody(body): # XXX perhaps add support for decoding gzip in the future. try: - body.replace('\0', '') body = unicode(body, 'ascii') + body = body.replace('\0', '') except UnicodeDecodeError: try: body = unicode(body, 'utf-8') + body = body.replace('\0', '') except UnicodeDecodeError: body = base64Dict(body)

1 0

[ooni-probe/master] Handle more DNS exceptions
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 2b5c77f44c6c25aec9979d931aa72375377c120f Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Feb 4 16:25:54 2016 +0100 Handle more DNS exceptions --- ooni/errors.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ooni/errors.py b/ooni/errors.py index 3b327f7..e523de0 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -5,6 +5,7 @@ from twisted.web.error import Error from twisted.internet.error import ConnectionRefusedError, TCPTimedOutError from twisted.internet.error import DNSLookupError, ConnectError, ConnectionLost +from twisted.names.error import DNSNameError, DNSServerError from twisted.internet.error import TimeoutError as GenericTimeoutError from twisted.internet.error import ProcessDone, ConnectionDone @@ -26,6 +27,8 @@ known_failures = [ (CancelledError, 'task_timed_out'), (gaierror, 'address_family_not_supported_error'), (DNSLookupError, 'dns_lookup_error'), + (DNSNameError, 'dns_name_error'), + (DNSServerError, 'dns_server_failure'), (TCPTimedOutError, 'tcp_timed_out_error'), (ResponseNeverReceived, 'response_never_received'), (DeferTimeoutError, 'deferred_timeout_error'),

1 0

[ooni-probe/master] Fixes to unittests
by art＠torproject.org 29 Apr '16

29 Apr '16

commit ef5e3cbcd1c3a64288f404c3182d15311cb78ddf Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Feb 5 20:03:52 2016 +0100 Fixes to unittests --- ooni/tests/test_oonicli.py | 9 +-------- ooni/tests/test_reporter.py | 3 ++- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 564f9e6..c57205c 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -158,14 +158,7 @@ class TestRunDirector(ConfigTestCase): assert 'agent' in entry assert 'requests' in entry assert 'socksproxy' in entry - assert 'tampering' in entry - assert 'header_field_name' in entry['tampering'] - assert 'header_field_number' in entry['tampering'] - assert 'header_field_value' in entry['tampering'] - assert 'header_name_capitalization' in entry['tampering'] - assert 'header_name_diff' in entry['tampering'] - assert 'request_line_capitalization' in entry['tampering'] - assert 'total' in entry['tampering'] + assert 'errors' in entry yield self.run_helper('manipulation/http_header_field_manipulation', ['-b', 'http://4.15.35.157:80'], diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index f1f7c3a..4b99584 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -26,7 +26,8 @@ test_details = { 'input_hashes': [], 'probe_asn': 'AS0', 'probe_cc': 'ZZ', - 'start_time': time.time() + 'start_time': time.time(), + 'data_format_version': '0.2.0' } oonib_new_report_message = {

1 0

[ooni-probe/master] Integrate withoutbody optflag lost in rebase
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 31c3a81dba626d3defed46213c7e6d6bfde632d6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Feb 5 17:47:29 2016 +0100 Integrate withoutbody optflag lost in rebase --- ooni/nettests/blocking/http_requests.py | 2 +- ooni/templates/httpt.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index f21b041..e00b4c6 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -18,7 +18,7 @@ class UsageOptions(usage.Options): ['factor', 'f', 0.8, 'What factor should be used for triggering censorship (0.8 == 80%)']] optFlags = [ - ["withoutbody","B", "don't include HTTP response body inside of the report"], + ["withoutbody", "B", "don't include HTTP response body inside of the report"], ] diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index a49c39b..3a36b18 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -151,9 +151,13 @@ class HTTPTest(NetTestCase): } } if response: + if self.localOptions.get('withoutbody', 0) is 0: + response_body = _representBody(response_body) + else: + response_body = '' session['response'] = { 'headers': _representHeaders(response.headers), - 'body': _representBody(response_body), + 'body': response_body, 'code': response.code } session['failure'] = None

1 0

[ooni-probe/master] Fixes to the DNS test template
by art＠torproject.org 29 Apr '16

29 Apr '16

commit b13cf26fed638a0de17d87ff1f4910fe98e1a135 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Feb 1 17:05:10 2016 +0100 Fixes to the DNS test template --- ooni/templates/dnst.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index e8ae189..9ee7508 100644 --- a/ooni/templates/dnst.py +++ b/ooni/templates/dnst.py @@ -180,16 +180,16 @@ class DNSTest(NetTestCase): answers = [] addrs = [] for answer in msg: - if answer.type is dnsType: - if dnsType is dns.SOA: - addr = (answer.name.name,answer.payload.serial) - elif dnsType in [dns.NS,dns.PTR]: - addr = answer.payload.name.name - elif dnsType is dns.A: - addr = answer.payload.dottedQuad() - else: - addr = None - addrs.append(addr) + addr = None + if answer.type is dns.SOA: + addr = (answer.name.name,answer.payload.serial) + elif answer.type in [dns.NS, dns.PTR, dns.CNAME]: + addr = answer.payload.name.name + elif answer.type is dns.A: + addr = answer.payload.dottedQuad() + else: + log.debug("Unidentified answer %s" % answer) + addrs.append(addr) answers.append(representAnswer(answer)) if dns_type == 'SOA': @@ -225,9 +225,13 @@ class DNSTest(NetTestCase): def addToReport(self, query, resolver=None, query_type=None, answers=None, failure=None): log.debug("Adding %s to report)" % query) - result = {} - result['resolver_hostname'] = resolver[0] - result['resolver_port'] = resolver[1] + result = { + 'resolver_hostname': None, + 'resolver_port': None + } + if resolver is not None and len(resolver) == 2: + result['resolver_hostname'] = resolver[0] + result['resolver_port'] = resolver[1] result['query_type'] = query_type result['hostname'] = str(query[0].name) result['failure'] = None

1 0

[ooni-probe/master] Adapt unittests to changes in the reporting
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 54529569c99ba670ff1135a3219de0ae19186d40 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Jan 30 22:24:51 2016 +0100 Adapt unittests to changes in the reporting --- ooni/tests/test_reporter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 454fbef..f1f7c3a 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -25,11 +25,12 @@ test_details = { 'software_version': '1.0', 'input_hashes': [], 'probe_asn': 'AS0', + 'probe_cc': 'ZZ', 'start_time': time.time() } oonib_new_report_message = { - 'report_id': "2014-01-29T202038Z_AS0_" + "A" * 50, + 'report_id': "20140129T202038Z_AS0_" + "A" * 50, 'backend_version': "1.0" } @@ -94,6 +95,8 @@ class TestOONIBReporter(unittest.TestCase): @defer.inlineCallbacks def test_write_report_entry(self): + self.mock_response = oonib_new_report_message + yield self.oonib_reporter.createReport() req = {'content': 'something'} yield self.oonib_reporter.writeReportEntry(req) assert self.oonib_reporter.agent.request.called

1 0

[ooni-probe/master] Update the tests to be consistent with the new data formats
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 9bd24ee947194b0f5713b46e845106210e7163c9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jan 28 13:09:36 2016 +0100 Update the tests to be consistent with the new data formats --- ooni/nettests/blocking/dns_consistency.py | 29 +++++++---- ooni/reporter.py | 50 ++++++++++--------- ooni/templates/dnst.py | 83 ++++++++++++++++++++++--------- ooni/templates/httpt.py | 41 +++++++++++---- ooni/templates/scapyt.py | 12 +++-- ooni/tests/test_trueheaders.py | 12 ++--- ooni/utils/__init__.py | 7 +++ ooni/utils/trueheaders.py | 2 +- 8 files changed, 157 insertions(+), 79 deletions(-) diff --git a/ooni/nettests/blocking/dns_consistency.py b/ooni/nettests/blocking/dns_consistency.py index 88e5d5e..93a2d65 100644 --- a/ooni/nettests/blocking/dns_consistency.py +++ b/ooni/nettests/blocking/dns_consistency.py @@ -39,7 +39,7 @@ class DNSConsistencyTest(dnst.DNSTest): name = "DNS Consistency" description = "Checks to see if the DNS responses from a "\ "set of DNS resolvers are consistent." - version = "0.6" + version = "0.7.0" authors = "Arturo Filastò, Isis Lovecruft" inputFile = ['file', 'f', None, @@ -110,7 +110,11 @@ class DNSConsistencyTest(dnst.DNSTest): log.msg("Doing the test lookups on %s" % self.input) hostname = self.input - self.report['tampering'] = {} + self.report['successful'] = [] + self.report['failures'] = [] + self.report['inconsistent'] = [] + + self.report['errors'] = {} try: control_answers = yield self.performALookup(hostname, @@ -121,11 +125,11 @@ class DNSConsistencyTest(dnst.DNSTest): "Got no response from control DNS server %s:%d, " "perhaps the DNS resolver is down?" % self.control_dns_server) - self.report['tampering'][ + self.report['errors'][ "%s:%d" % self.control_dns_server] = 'no_answer' except: - self.report['tampering'][ + self.report['errors'][ "%s:%d" % self.control_dns_server] = 'error' control_answers = None @@ -139,12 +143,14 @@ class DNSConsistencyTest(dnst.DNSTest): test_dns_server) except Exception: log.err("Problem performing the DNS lookup") - self.report['tampering'][test_resolver] = 'dns_lookup_error' + self.report['errors'][test_resolver] = 'dns_lookup_error' + self.report['failures'].append(test_resolver) continue if not experiment_answers: log.err("Got no response, perhaps the DNS resolver is down?") - self.report['tampering'][test_resolver] = 'no_answer' + self.report['errors'][test_resolver] = 'no_answer' + self.report['failures'].append(test_resolver) continue else: log.debug( @@ -165,12 +171,13 @@ class DNSConsistencyTest(dnst.DNSTest): if not control_answers: log.msg("Skipping control resolver comparison") - self.report['tampering'][test_resolver] = None + self.report['errors'][test_resolver] = None elif set(experiment_answers) & set(control_answers): lookup_details() log.msg("tampering: false") - self.report['tampering'][test_resolver] = False + self.report['errors'][test_resolver] = False + self.report['successful'].append(test_resolver) else: log.msg("Trying to do reverse lookup") experiment_reverse = yield self.performPTRLookup(experiment_answers[0], @@ -182,12 +189,14 @@ class DNSConsistencyTest(dnst.DNSTest): log.msg("Further testing has eliminated false positives") lookup_details() log.msg("tampering: reverse_match") - self.report['tampering'][test_resolver] = 'reverse_match' + self.report['errors'][test_resolver] = 'reverse_match' + self.report['successful'].append(test_resolver) else: log.msg("Reverse lookups do not match") lookup_details() log.msg("tampering: true") - self.report['tampering'][test_resolver] = True + self.report['errors'][test_resolver] = True + self.report['inconsistent'].append(test_resolver) def inputProcessor(self, filename=None): """ diff --git a/ooni/reporter.py b/ooni/reporter.py index db6f7c1..3ea1b10 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -236,6 +236,7 @@ class OONIBReporter(OReporter): self.validateCollectorAddress() self.reportID = None + self.supportedFormats = ["yaml"] if self.collectorAddress.startswith('https://'): # not sure if there's something else it needs. Seems to work. @@ -257,27 +258,26 @@ class OONIBReporter(OReporter): if not re.match(regexp, self.collectorAddress): raise errors.InvalidOONIBCollectorAddress - def serializeEntry(self, entry): - if config.report.format == "json": + def serializeEntry(self, entry, serialisation_format="yaml"): + if serialisation_format == "json": if isinstance(entry, Measurement): report_entry = entry.testInstance.report elif isinstance(entry, Failure): - report_entry = entry.value + report_entry = {'failure': entry.value} elif isinstance(entry, dict): report_entry = entry - report_entry["record_type"] = "entry" - report_entry["report_id"] = self.reportID - content = json.dumps(report_entry, ensure_ascii=True) + "\n" + return report_entry else: content = '---\n' if isinstance(entry, Measurement): - content += safe_dump(entry.testInstance.report) + report_entry = entry.testInstance.report elif isinstance(entry, Failure): - content += entry.value + report_entry = {'failure': entry.value} elif isinstance(entry, dict): - content += safe_dump(entry) + report_entry = entry + content += safe_dump(report_entry) content += '...\n' - return content + return content @defer.inlineCallbacks def writeReportEntry(self, entry): @@ -285,8 +285,13 @@ class OONIBReporter(OReporter): url = self.collectorAddress + '/report/' + self.reportID - request = {'format': config.report.format, - 'content': self.serializeEntry(entry)} + if "json" in self.supportedFormats: + serialisation_format = 'json' + else: + serialisation_format = 'yaml' + + request = {'format': serialisation_format, + 'content': self.serializeEntry(entry, serialisation_format)} log.debug("Updating report with id %s (%s)" % (self.reportID, url)) request_json = json.dumps(request) @@ -295,12 +300,11 @@ class OONIBReporter(OReporter): bodyProducer = StringProducer(request_json) try: - yield self.agent.request("PUT", url, + yield self.agent.request("PUT", str(url), bodyProducer=bodyProducer) - except: - # XXX we must trap this in the runner and make sure to report the - # data later. + except Exception as exc: log.err("Error in writing report entry") + log.exception(exc) raise errors.OONIBReportUpdateError @defer.inlineCallbacks @@ -324,21 +328,16 @@ class OONIBReporter(OReporter): url = self.collectorAddress + '/report' - content = '---\n' - content += safe_dump(self.testDetails) - content += '...\n' - request = { 'software_name': self.testDetails['software_name'], 'software_version': self.testDetails['software_version'], 'probe_asn': self.testDetails['probe_asn'], + 'probe_cc': self.testDetails['probe_cc'], 'test_name': self.testDetails['test_name'], 'test_version': self.testDetails['test_version'], + 'start_time': self.testDetails['start_time'], 'input_hashes': self.testDetails['input_hashes'], - # XXX there is a bunch of redundancy in the arguments getting sent - # to the backend. This may need to get changed in the client and - # the backend. - 'content': content + 'format': 'json' } # import values from the environment request.update([(k.lower(),v) for (k,v) in os.environ.iteritems() @@ -395,6 +394,9 @@ class OONIBReporter(OReporter): self.reportID = parsed_response['report_id'] self.backendVersion = parsed_response['backend_version'] + + self.supportedFormats = parsed_response.get('supported_formats', ["yaml"]) + log.debug("Created report with id %s" % parsed_response['report_id']) defer.returnValue(parsed_response['report_id']) diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index 7c9626c..e8ae189 100644 --- a/ooni/templates/dnst.py +++ b/ooni/templates/dnst.py @@ -58,13 +58,36 @@ def connectionLost(self, reason=None): udp.Port.connectionLost = connectionLost def representAnswer(answer): - # We store the resource record and the answer payload in a - # tuple - return (repr(answer), repr(answer.payload)) + answer_types = { + dns.SOA: 'SOA', + dns.NS: 'NS', + dns.PTR: 'PTR', + dns.A: 'A', + dns.CNAME: 'CNAME', + dns.MX: 'MX' + } + answer_type = answer_types.get(answer.type, 'unknown') + represented_answer = { + "answer_type": answer_type + } + if answer_type is 'SOA': + represented_answer['ttl'] = answer.payload.ttl + represented_answer['hostname'] = answer.payload.mname + represented_answer['responsible_name'] = answer.payload.rname + represented_answer['serial_number'] = answer.payload.serial + represented_answer['refresh_interval'] = answer.payload.refresh + represented_answer['retry_interval'] = answer.payload.retry + represented_answer['minimum_ttl'] = answer.payload.minimum + represented_answer['expiration_limit'] = answer.payload.expire + elif answer_type in ['NS', 'PTR', 'CNAME']: + represented_answer['hostname'] = answer.payload.name.name + elif answer_type is 'A': + represented_answer['ipv4'] = answer.payload.dottedQuad() + return represented_answer class DNSTest(NetTestCase): name = "Base DNS Test" - version = 0.1 + version = "0.2.0" requiresRoot = False queryTimeout = [1] @@ -137,18 +160,25 @@ class DNSTest(NetTestCase): :dns_server: is the dns_server that should be used for the lookup as a tuple of ip port (ex. ("127.0.0.1", 53)) """ - types={'NS':dns.NS,'A':dns.A,'SOA':dns.SOA,'PTR':dns.PTR} - dnsType=types[dns_type] + types = { + 'NS': dns.NS, + 'A': dns.A, + 'SOA': dns.SOA, + 'PTR': dns.PTR + } + dnsType = types[dns_type] query = [dns.Query(hostname, dnsType, dns.IN)] def gotResponse(message): - log.debug(dns_type+" Lookup successful") + log.debug(dns_type + " Lookup successful") log.debug(str(message)) - addrs = [] - answers = [] + if dns_server: msg = message.answers else: msg = message[0] + + answers = [] + addrs = [] for answer in msg: if answer.type is dnsType: if dnsType is dns.SOA: @@ -159,46 +189,53 @@ class DNSTest(NetTestCase): addr = answer.payload.dottedQuad() else: addr = None - addrs.append(addr) + addrs.append(addr) answers.append(representAnswer(answer)) - DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type, - answers=answers, addrs=addrs) + if dns_type == 'SOA': + for authority in message.authority: + answers.append(representAnswer(authority)) + + DNSTest.addToReport(self, query, resolver=dns_server, + query_type=dns_type, answers=answers) return addrs def gotError(failure): failure.trap(gaierror, TimeoutError) - DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type, - failure=failure) + DNSTest.addToReport(self, query, resolver=dns_server, + query_type=dns_type, failure=failure) return failure if dns_server: resolver = Resolver(servers=[dns_server]) d = resolver.queryUDP(query, timeout=self.queryTimeout) else: - lookupFunction={'NS':client.lookupNameservers, 'SOA':client.lookupAuthority, 'A':client.lookupAddress, 'PTR':client.lookupPointer} + lookupFunction = { + 'NS': client.lookupNameservers, + 'SOA': client.lookupAuthority, + 'A': client.lookupAddress, + 'PTR': client.lookupPointer + } d = lookupFunction[dns_type](hostname) d.addCallback(gotResponse) d.addErrback(gotError) return d - def addToReport(self, query, resolver=None, query_type=None, - answers=None, name=None, addrs=None, failure=None): + answers=None, failure=None): log.debug("Adding %s to report)" % query) result = {} - result['resolver'] = resolver + result['resolver_hostname'] = resolver[0] + result['resolver_port'] = resolver[1] result['query_type'] = query_type - result['query'] = repr(query) + result['hostname'] = str(query[0].name) + result['failure'] = None if failure: result['failure'] = failureToString(failure) + result['answers'] = [] if answers: result['answers'] = answers - if name: - result['name'] = name - if addrs: - result['addrs'] = addrs self.report['queries'].append(result) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 3702e86..15630d5 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -9,7 +9,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent from ooni.nettest import NetTestCase -from ooni.utils import log +from ooni.utils import log, base64Dict from ooni.settings import config from ooni.utils.net import BodyReceiver, StringProducer, userAgents @@ -122,11 +122,28 @@ class HTTPTest(NetTestCase): failure (instance): An instance of :class:twisted.internet.failure.Failure """ + def _representHeaders(headers): + represented_headers = {} + for name, value in headers.getAllRawHeaders(): + represented_headers[name] = value[0] + return represented_headers + + def _representBody(body): + # XXX perhaps add support for decoding gzip in the future. + try: + body.replace('\0', '') + body = unicode(body, 'ascii') + except UnicodeDecodeError: + try: + body = unicode(body, 'utf-8') + except UnicodeDecodeError: + body = base64Dict(body) + log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers']) - request_response = { + session = { 'request': { - 'headers': list(request_headers.getAllRawHeaders()), + 'headers': _representHeaders(request_headers), 'body': request['body'], 'url': request['url'], 'method': request['method'], @@ -134,15 +151,16 @@ class HTTPTest(NetTestCase): } } if response: - request_response['response'] = { - 'headers': list(response.headers.getAllRawHeaders()), - 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '', + session['response'] = { + 'headers': _representHeaders(response.headers), + 'body': _representBody(response_body), 'code': response.code - } + } + session['failure'] = None if failure_string: - request_response['failure'] = failure_string + session['failure'] = failure_string - self.report['requests'].append(request_response) + self.report['requests'].append(session) def _processResponseBody(self, response_body, request, response, body_processor): log.debug("Processing response body") @@ -302,7 +320,10 @@ class HTTPTest(NetTestCase): request['url'] = url request['headers'] = headers request['body'] = body - request['tor'] = {} + request['tor'] = { + 'exit_ip': None, + 'exit_name': None + } if use_tor: request['tor']['is_tor'] = True else: diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 35dbf5b..f4b6280 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -1,11 +1,17 @@ from ooni.nettest import NetTestCase -from ooni.utils import log +from ooni.utils import log, base64Dict from ooni.settings import config from ooni.utils.net import hasRawSocketPermission from ooni.utils.txscapy import ScapySender, ScapyFactory +def _representPacket(packet): + return { + "raw_packet": base64Dict(str(packet)), + "summary": repr(packet) + } + class BaseScapyTest(NetTestCase): """ @@ -88,8 +94,8 @@ class BaseScapyTest(NetTestCase): sent_packet.src = '127.0.0.1' received_packet.dst = '127.0.0.1' - self.report['sent_packets'].append(sent_packet) - self.report['answered_packets'].append(received_packet) + self.report['sent_packets'].append(_representPacket(sent_packet)) + self.report['answered_packets'].append(_representPacket(received_packet)) return packets def sr(self, packets, timeout=None, *arg, **kw): diff --git a/ooni/tests/test_trueheaders.py b/ooni/tests/test_trueheaders.py index 8e9f053..f6c7812 100644 --- a/ooni/tests/test_trueheaders.py +++ b/ooni/tests/test_trueheaders.py @@ -23,19 +23,15 @@ dummy_headers_dict3 = { class TestTrueHeaders(unittest.TestCase): def test_names_match(self): th = TrueHeaders(dummy_headers_dict) - self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), set()) + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), []) def test_names_not_match(self): th = TrueHeaders(dummy_headers_dict) - self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3'])) + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3']) th = TrueHeaders(dummy_headers_dict3) - self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3', 'Header4'])) + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3', 'Header4']) def test_names_match_expect_ignore(self): th = TrueHeaders(dummy_headers_dict) - self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), set()) - - - - + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), []) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 02904cc..5e9616b 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -5,6 +5,7 @@ import glob import os import gzip +from base64 import b64encode from zipfile import ZipFile from ooni import otime @@ -176,3 +177,9 @@ def gunzip(filename, dst): def get_ooni_root(): script = os.path.join(__file__, '..') return os.path.dirname(os.path.realpath(script)) + +def base64Dict(data): + return { + 'format': 'base64', + 'data': b64encode(data) + } diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index 149b440..b7671b5 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -80,7 +80,7 @@ class TrueHeaders(http_headers.Headers): pass else: diff.add(name) - return diff + return list(diff) def getAllRawHeaders(self): for k, v in self._rawHeaders.iteritems():

1 0

[ooni-probe/master] Fix report uploader to work with JSON
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 061af8e36784ced0f96965eb992b340d63ce027f Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Feb 4 16:39:14 2016 +0100 Fix report uploader to work with JSON * Add better exception handling to deckgen * We should actually be using POST since PUT is indempotent * Debugging of backward and forward data format compatibility * Better integrity checks in web_consistency --- ooni/deckgen/processors/citizenlab_test_lists.py | 5 +++++ ooni/nettest.py | 3 ++- ooni/report/tool.py | 4 ++-- ooni/reporter.py | 14 +++++++++++--- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py index 0a56ec0..086b550 100644 --- a/ooni/deckgen/processors/citizenlab_test_lists.py +++ b/ooni/deckgen/processors/citizenlab_test_lists.py @@ -47,6 +47,11 @@ def generate_global_input(dst): "test-lists-master/lists/" "global.csv") + if not input_list: + print("Could not find the global input list") + print("Perhaps you should run ooniresources --update-inputs") + raise Exception("Could not find the global input list") + load_input(input_list, filename) return filename diff --git a/ooni/nettest.py b/ooni/nettest.py index e347d55..9bdc6d9 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -220,7 +220,8 @@ class NetTestLoader(object): 'input_hashes': input_file_hashes, 'report_id': self.reportID, 'test_helpers': self.testHelpers, - 'annotations': self.annotations + 'annotations': self.annotations, + 'data_format_version': '0.2.0' } return test_details diff --git a/ooni/report/tool.py b/ooni/report/tool.py index a16794a..5946c42 100644 --- a/ooni/report/tool.py +++ b/ooni/report/tool.py @@ -14,7 +14,7 @@ from ooni.oonibclient import OONIBClient def upload(report_file, collector=None, bouncer=None): oonib_report_log = OONIBReportLog() - print "Attempting to upload %s" % report_file + log.msg("Attempting to upload %s" % report_file) with open(config.report_log_file) as f: report_log = yaml.safe_load(f) @@ -47,9 +47,9 @@ def upload(report_file, collector=None, bouncer=None): oonib_reporter = OONIBReporter(report.header, collector) log.msg("Creating report for %s with %s" % (report_file, collector)) report_id = yield oonib_reporter.createReport() + report.header['report_id'] = report_id yield oonib_report_log.created(report_file, collector, report_id) for entry in report: - print "Writing entry" yield oonib_reporter.writeReportEntry(entry) log.msg("Closing report.") yield oonib_reporter.finish() diff --git a/ooni/reporter.py b/ooni/reporter.py index b725540..d9d147c 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -1,3 +1,4 @@ +import uuid import yaml import json import os @@ -198,7 +199,6 @@ class YAMLReporter(OReporter): raise Exception("Failed to serialise entry") content += safe_dump(report_entry) content += '...\n' - report_entry.update(self.testDetails) self._write(content) def createReport(self): @@ -265,11 +265,17 @@ class OONIBReporter(OReporter): if isinstance(entry, Measurement): report_entry = { 'input': entry.testInstance.report.pop('input', None), + 'id': str(uuid.uuid4()), + 'test_start_time': entry.testInstance.report.pop('test_start_time', None), + 'test_runtime': entry.testInstance.report.pop('test_runtime', None), 'test_keys': entry.testInstance.report } elif isinstance(entry, dict): report_entry = { 'input': entry.pop('input', None), + 'id': str(uuid.uuid4()), + 'test_start_time': entry.pop('test_start_time', None), + 'test_runtime': entry.pop('test_runtime', None), 'test_keys': entry } else: @@ -284,7 +290,6 @@ class OONIBReporter(OReporter): report_entry = entry else: raise Exception("Failed to serialise entry") - report_entry.update(self.testDetails) content += safe_dump(report_entry) content += '...\n' return content @@ -312,7 +317,7 @@ class OONIBReporter(OReporter): bodyProducer = StringProducer(request_json) try: - yield self.agent.request("PUT", str(url), + yield self.agent.request("POST", str(url), bodyProducer=bodyProducer) except Exception as exc: log.err("Error in writing report entry") @@ -349,6 +354,7 @@ class OONIBReporter(OReporter): 'test_version': self.testDetails['test_version'], 'start_time': self.testDetails['start_time'], 'input_hashes': self.testDetails['input_hashes'], + 'data_format_version': self.testDetails['data_format_version'], 'format': 'json' } # import values from the environment @@ -618,6 +624,8 @@ class Report(object): self.collector_address) def created(report_id): + self.reportID = report_id + self.test_details['report_id'] = report_id if not self.oonib_reporter: return return self.report_log.created(self.report_filename,

1 0

[ooni-probe/master] We no longer need to avoid twisted 15.5.0
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 87764340881d0afd2091e588141d9aa677660c52 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Dec 3 13:09:39 2015 +0100 We no longer need to avoid twisted 15.5.0 Thanks to @meejah for releasing a fix so quickly --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b052adb..8b8d5ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML>=3.10 -Twisted>=12.2.0,<15.5.0 +Twisted>=12.2.0 ipaddr>=2.1.10 pyOpenSSL>=0.13 geoip

1 0