commit 9bd24ee947194b0f5713b46e845106210e7163c9 Author: Arturo Filastò arturo@filasto.net Date: Thu Jan 28 13:09:36 2016 +0100

Update the tests to be consistent with the new data formats --- ooni/nettests/blocking/dns_consistency.py | 29 +++++++---- ooni/reporter.py | 50 ++++++++++--------- ooni/templates/dnst.py | 83 ++++++++++++++++++++++--------- ooni/templates/httpt.py | 41 +++++++++++---- ooni/templates/scapyt.py | 12 +++-- ooni/tests/test_trueheaders.py | 12 ++--- ooni/utils/__init__.py | 7 +++ ooni/utils/trueheaders.py | 2 +- 8 files changed, 157 insertions(+), 79 deletions(-)

diff --git a/ooni/nettests/blocking/dns_consistency.py b/ooni/nettests/blocking/dns_consistency.py index 88e5d5e..93a2d65 100644 --- a/ooni/nettests/blocking/dns_consistency.py +++ b/ooni/nettests/blocking/dns_consistency.py @@ -39,7 +39,7 @@ class DNSConsistencyTest(dnst.DNSTest): name = "DNS Consistency" description = "Checks to see if the DNS responses from a "\ "set of DNS resolvers are consistent." - version = "0.6" + version = "0.7.0" authors = "Arturo Filastò, Isis Lovecruft"

inputFile = ['file', 'f', None, @@ -110,7 +110,11 @@ class DNSConsistencyTest(dnst.DNSTest): log.msg("Doing the test lookups on %s" % self.input) hostname = self.input

- self.report['tampering'] = {} + self.report['successful'] = [] + self.report['failures'] = [] + self.report['inconsistent'] = [] + + self.report['errors'] = {}

try: control_answers = yield self.performALookup(hostname, @@ -121,11 +125,11 @@ class DNSConsistencyTest(dnst.DNSTest): "Got no response from control DNS server %s:%d, " "perhaps the DNS resolver is down?" % self.control_dns_server) - self.report['tampering'][ + self.report['errors'][ "%s:%d" % self.control_dns_server] = 'no_answer' except: - self.report['tampering'][ + self.report['errors'][ "%s:%d" % self.control_dns_server] = 'error' control_answers = None @@ -139,12 +143,14 @@ class DNSConsistencyTest(dnst.DNSTest): test_dns_server) except Exception: log.err("Problem performing the DNS lookup") - self.report['tampering'][test_resolver] = 'dns_lookup_error' + self.report['errors'][test_resolver] = 'dns_lookup_error' + self.report['failures'].append(test_resolver) continue

if not experiment_answers: log.err("Got no response, perhaps the DNS resolver is down?") - self.report['tampering'][test_resolver] = 'no_answer' + self.report['errors'][test_resolver] = 'no_answer' + self.report['failures'].append(test_resolver) continue else: log.debug( @@ -165,12 +171,13 @@ class DNSConsistencyTest(dnst.DNSTest):

if not control_answers: log.msg("Skipping control resolver comparison") - self.report['tampering'][test_resolver] = None + self.report['errors'][test_resolver] = None

elif set(experiment_answers) & set(control_answers): lookup_details() log.msg("tampering: false") - self.report['tampering'][test_resolver] = False + self.report['errors'][test_resolver] = False + self.report['successful'].append(test_resolver) else: log.msg("Trying to do reverse lookup") experiment_reverse = yield self.performPTRLookup(experiment_answers[0], @@ -182,12 +189,14 @@ class DNSConsistencyTest(dnst.DNSTest): log.msg("Further testing has eliminated false positives") lookup_details() log.msg("tampering: reverse_match") - self.report['tampering'][test_resolver] = 'reverse_match' + self.report['errors'][test_resolver] = 'reverse_match' + self.report['successful'].append(test_resolver) else: log.msg("Reverse lookups do not match") lookup_details() log.msg("tampering: true") - self.report['tampering'][test_resolver] = True + self.report['errors'][test_resolver] = True + self.report['inconsistent'].append(test_resolver)

def inputProcessor(self, filename=None): """ diff --git a/ooni/reporter.py b/ooni/reporter.py index db6f7c1..3ea1b10 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -236,6 +236,7 @@ class OONIBReporter(OReporter): self.validateCollectorAddress()

self.reportID = None + self.supportedFormats = ["yaml"]

if self.collectorAddress.startswith('https://'): # not sure if there's something else it needs. Seems to work. @@ -257,27 +258,26 @@ class OONIBReporter(OReporter): if not re.match(regexp, self.collectorAddress): raise errors.InvalidOONIBCollectorAddress

- def serializeEntry(self, entry): - if config.report.format == "json": + def serializeEntry(self, entry, serialisation_format="yaml"): + if serialisation_format == "json": if isinstance(entry, Measurement): report_entry = entry.testInstance.report elif isinstance(entry, Failure): - report_entry = entry.value + report_entry = {'failure': entry.value} elif isinstance(entry, dict): report_entry = entry - report_entry["record_type"] = "entry" - report_entry["report_id"] = self.reportID - content = json.dumps(report_entry, ensure_ascii=True) + "\n" + return report_entry else: content = '---\n' if isinstance(entry, Measurement): - content += safe_dump(entry.testInstance.report) + report_entry = entry.testInstance.report elif isinstance(entry, Failure): - content += entry.value + report_entry = {'failure': entry.value} elif isinstance(entry, dict): - content += safe_dump(entry) + report_entry = entry + content += safe_dump(report_entry) content += '...\n' - return content + return content

@defer.inlineCallbacks def writeReportEntry(self, entry): @@ -285,8 +285,13 @@ class OONIBReporter(OReporter):

url = self.collectorAddress + '/report/' + self.reportID

- request = {'format': config.report.format, - 'content': self.serializeEntry(entry)} + if "json" in self.supportedFormats: + serialisation_format = 'json' + else: + serialisation_format = 'yaml' + + request = {'format': serialisation_format, + 'content': self.serializeEntry(entry, serialisation_format)}

log.debug("Updating report with id %s (%s)" % (self.reportID, url)) request_json = json.dumps(request) @@ -295,12 +300,11 @@ class OONIBReporter(OReporter): bodyProducer = StringProducer(request_json)

try: - yield self.agent.request("PUT", url, + yield self.agent.request("PUT", str(url), bodyProducer=bodyProducer) - except: - # XXX we must trap this in the runner and make sure to report the - # data later. + except Exception as exc: log.err("Error in writing report entry") + log.exception(exc) raise errors.OONIBReportUpdateError

@defer.inlineCallbacks @@ -324,21 +328,16 @@ class OONIBReporter(OReporter):

url = self.collectorAddress + '/report'

- content = '---\n' - content += safe_dump(self.testDetails) - content += '...\n' - request = { 'software_name': self.testDetails['software_name'], 'software_version': self.testDetails['software_version'], 'probe_asn': self.testDetails['probe_asn'], + 'probe_cc': self.testDetails['probe_cc'], 'test_name': self.testDetails['test_name'], 'test_version': self.testDetails['test_version'], + 'start_time': self.testDetails['start_time'], 'input_hashes': self.testDetails['input_hashes'], - # XXX there is a bunch of redundancy in the arguments getting sent - # to the backend. This may need to get changed in the client and - # the backend. - 'content': content + 'format': 'json' } # import values from the environment request.update([(k.lower(),v) for (k,v) in os.environ.iteritems() @@ -395,6 +394,9 @@ class OONIBReporter(OReporter):

self.reportID = parsed_response['report_id'] self.backendVersion = parsed_response['backend_version'] + + self.supportedFormats = parsed_response.get('supported_formats', ["yaml"]) + log.debug("Created report with id %s" % parsed_response['report_id']) defer.returnValue(parsed_response['report_id'])

diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index 7c9626c..e8ae189 100644 --- a/ooni/templates/dnst.py +++ b/ooni/templates/dnst.py @@ -58,13 +58,36 @@ def connectionLost(self, reason=None): udp.Port.connectionLost = connectionLost

def representAnswer(answer): - # We store the resource record and the answer payload in a - # tuple - return (repr(answer), repr(answer.payload)) + answer_types = { + dns.SOA: 'SOA', + dns.NS: 'NS', + dns.PTR: 'PTR', + dns.A: 'A', + dns.CNAME: 'CNAME', + dns.MX: 'MX' + } + answer_type = answer_types.get(answer.type, 'unknown') + represented_answer = { + "answer_type": answer_type + } + if answer_type is 'SOA': + represented_answer['ttl'] = answer.payload.ttl + represented_answer['hostname'] = answer.payload.mname + represented_answer['responsible_name'] = answer.payload.rname + represented_answer['serial_number'] = answer.payload.serial + represented_answer['refresh_interval'] = answer.payload.refresh + represented_answer['retry_interval'] = answer.payload.retry + represented_answer['minimum_ttl'] = answer.payload.minimum + represented_answer['expiration_limit'] = answer.payload.expire + elif answer_type in ['NS', 'PTR', 'CNAME']: + represented_answer['hostname'] = answer.payload.name.name + elif answer_type is 'A': + represented_answer['ipv4'] = answer.payload.dottedQuad() + return represented_answer

class DNSTest(NetTestCase): name = "Base DNS Test" - version = 0.1 + version = "0.2.0"

requiresRoot = False queryTimeout = [1] @@ -137,18 +160,25 @@ class DNSTest(NetTestCase): :dns_server: is the dns_server that should be used for the lookup as a tuple of ip port (ex. ("127.0.0.1", 53)) """ - types={'NS':dns.NS,'A':dns.A,'SOA':dns.SOA,'PTR':dns.PTR} - dnsType=types[dns_type] + types = { + 'NS': dns.NS, + 'A': dns.A, + 'SOA': dns.SOA, + 'PTR': dns.PTR + } + dnsType = types[dns_type] query = [dns.Query(hostname, dnsType, dns.IN)] def gotResponse(message): - log.debug(dns_type+" Lookup successful") + log.debug(dns_type + " Lookup successful") log.debug(str(message)) - addrs = [] - answers = [] + if dns_server: msg = message.answers else: msg = message[0] + + answers = [] + addrs = [] for answer in msg: if answer.type is dnsType: if dnsType is dns.SOA: @@ -159,46 +189,53 @@ class DNSTest(NetTestCase): addr = answer.payload.dottedQuad() else: addr = None - addrs.append(addr) + addrs.append(addr) answers.append(representAnswer(answer))

- DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type, - answers=answers, addrs=addrs) + if dns_type == 'SOA': + for authority in message.authority: + answers.append(representAnswer(authority)) + + DNSTest.addToReport(self, query, resolver=dns_server, + query_type=dns_type, answers=answers) return addrs

def gotError(failure): failure.trap(gaierror, TimeoutError) - DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type, - failure=failure) + DNSTest.addToReport(self, query, resolver=dns_server, + query_type=dns_type, failure=failure) return failure

if dns_server: resolver = Resolver(servers=[dns_server]) d = resolver.queryUDP(query, timeout=self.queryTimeout) else: - lookupFunction={'NS':client.lookupNameservers, 'SOA':client.lookupAuthority, 'A':client.lookupAddress, 'PTR':client.lookupPointer} + lookupFunction = { + 'NS': client.lookupNameservers, + 'SOA': client.lookupAuthority, + 'A': client.lookupAddress, + 'PTR': client.lookupPointer + } d = lookupFunction[dns_type](hostname)

d.addCallback(gotResponse) d.addErrback(gotError) return d

- def addToReport(self, query, resolver=None, query_type=None, - answers=None, name=None, addrs=None, failure=None): + answers=None, failure=None): log.debug("Adding %s to report)" % query) result = {} - result['resolver'] = resolver + result['resolver_hostname'] = resolver[0] + result['resolver_port'] = resolver[1] result['query_type'] = query_type - result['query'] = repr(query) + result['hostname'] = str(query[0].name) + result['failure'] = None if failure: result['failure'] = failureToString(failure)

+ result['answers'] = [] if answers: result['answers'] = answers - if name: - result['name'] = name - if addrs: - result['addrs'] = addrs

self.report['queries'].append(result) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 3702e86..15630d5 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -9,7 +9,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent

from ooni.nettest import NetTestCase -from ooni.utils import log +from ooni.utils import log, base64Dict from ooni.settings import config

from ooni.utils.net import BodyReceiver, StringProducer, userAgents @@ -122,11 +122,28 @@ class HTTPTest(NetTestCase):

failure (instance): An instance of :class:twisted.internet.failure.Failure """ + def _representHeaders(headers): + represented_headers = {} + for name, value in headers.getAllRawHeaders(): + represented_headers[name] = value[0] + return represented_headers + + def _representBody(body): + # XXX perhaps add support for decoding gzip in the future. + try: + body.replace('\0', '') + body = unicode(body, 'ascii') + except UnicodeDecodeError: + try: + body = unicode(body, 'utf-8') + except UnicodeDecodeError: + body = base64Dict(body) + log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers']) - request_response = { + session = { 'request': { - 'headers': list(request_headers.getAllRawHeaders()), + 'headers': _representHeaders(request_headers), 'body': request['body'], 'url': request['url'], 'method': request['method'], @@ -134,15 +151,16 @@ class HTTPTest(NetTestCase): } } if response: - request_response['response'] = { - 'headers': list(response.headers.getAllRawHeaders()), - 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '', + session['response'] = { + 'headers': _representHeaders(response.headers), + 'body': _representBody(response_body), 'code': response.code - } + } + session['failure'] = None if failure_string: - request_response['failure'] = failure_string + session['failure'] = failure_string

- self.report['requests'].append(request_response) + self.report['requests'].append(session)

def _processResponseBody(self, response_body, request, response, body_processor): log.debug("Processing response body") @@ -302,7 +320,10 @@ class HTTPTest(NetTestCase): request['url'] = url request['headers'] = headers request['body'] = body - request['tor'] = {} + request['tor'] = { + 'exit_ip': None, + 'exit_name': None + } if use_tor: request['tor']['is_tor'] = True else: diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 35dbf5b..f4b6280 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -1,11 +1,17 @@ from ooni.nettest import NetTestCase -from ooni.utils import log +from ooni.utils import log, base64Dict from ooni.settings import config from ooni.utils.net import hasRawSocketPermission

from ooni.utils.txscapy import ScapySender, ScapyFactory

+def _representPacket(packet): + return { + "raw_packet": base64Dict(str(packet)), + "summary": repr(packet) + } + class BaseScapyTest(NetTestCase):

""" @@ -88,8 +94,8 @@ class BaseScapyTest(NetTestCase): sent_packet.src = '127.0.0.1' received_packet.dst = '127.0.0.1'

- self.report['sent_packets'].append(sent_packet) - self.report['answered_packets'].append(received_packet) + self.report['sent_packets'].append(_representPacket(sent_packet)) + self.report['answered_packets'].append(_representPacket(received_packet)) return packets

def sr(self, packets, timeout=None, *arg, **kw): diff --git a/ooni/tests/test_trueheaders.py b/ooni/tests/test_trueheaders.py index 8e9f053..f6c7812 100644 --- a/ooni/tests/test_trueheaders.py +++ b/ooni/tests/test_trueheaders.py @@ -23,19 +23,15 @@ dummy_headers_dict3 = { class TestTrueHeaders(unittest.TestCase): def test_names_match(self): th = TrueHeaders(dummy_headers_dict) - self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), set()) + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), [])

def test_names_not_match(self): th = TrueHeaders(dummy_headers_dict) - self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3'])) + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3'])

th = TrueHeaders(dummy_headers_dict3) - self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3', 'Header4'])) + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3', 'Header4'])

def test_names_match_expect_ignore(self): th = TrueHeaders(dummy_headers_dict) - self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), set()) - - - - + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), []) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 02904cc..5e9616b 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -5,6 +5,7 @@ import glob import os

import gzip +from base64 import b64encode from zipfile import ZipFile

from ooni import otime @@ -176,3 +177,9 @@ def gunzip(filename, dst): def get_ooni_root(): script = os.path.join(__file__, '..') return os.path.dirname(os.path.realpath(script)) + +def base64Dict(data): + return { + 'format': 'base64', + 'data': b64encode(data) + } diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index 149b440..b7671b5 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -80,7 +80,7 @@ class TrueHeaders(http_headers.Headers): pass else: diff.add(name) - return diff + return list(diff)

def getAllRawHeaders(self): for k, v in self._rawHeaders.iteritems():