[tor-commits] [ooni-probe/master] Update the tests to be consistent with the new data formats
art at torproject.org
art at torproject.org
Fri Apr 29 09:42:24 UTC 2016
commit 9bd24ee947194b0f5713b46e845106210e7163c9
Author: Arturo Filastò <arturo at filasto.net>
Date: Thu Jan 28 13:09:36 2016 +0100
Update the tests to be consistent with the new data formats
---
ooni/nettests/blocking/dns_consistency.py | 29 +++++++----
ooni/reporter.py | 50 ++++++++++---------
ooni/templates/dnst.py | 83 ++++++++++++++++++++++---------
ooni/templates/httpt.py | 41 +++++++++++----
ooni/templates/scapyt.py | 12 +++--
ooni/tests/test_trueheaders.py | 12 ++---
ooni/utils/__init__.py | 7 +++
ooni/utils/trueheaders.py | 2 +-
8 files changed, 157 insertions(+), 79 deletions(-)
diff --git a/ooni/nettests/blocking/dns_consistency.py b/ooni/nettests/blocking/dns_consistency.py
index 88e5d5e..93a2d65 100644
--- a/ooni/nettests/blocking/dns_consistency.py
+++ b/ooni/nettests/blocking/dns_consistency.py
@@ -39,7 +39,7 @@ class DNSConsistencyTest(dnst.DNSTest):
name = "DNS Consistency"
description = "Checks to see if the DNS responses from a "\
"set of DNS resolvers are consistent."
- version = "0.6"
+ version = "0.7.0"
authors = "Arturo Filastò, Isis Lovecruft"
inputFile = ['file', 'f', None,
@@ -110,7 +110,11 @@ class DNSConsistencyTest(dnst.DNSTest):
log.msg("Doing the test lookups on %s" % self.input)
hostname = self.input
- self.report['tampering'] = {}
+ self.report['successful'] = []
+ self.report['failures'] = []
+ self.report['inconsistent'] = []
+
+ self.report['errors'] = {}
try:
control_answers = yield self.performALookup(hostname,
@@ -121,11 +125,11 @@ class DNSConsistencyTest(dnst.DNSTest):
"Got no response from control DNS server %s:%d, "
"perhaps the DNS resolver is down?" %
self.control_dns_server)
- self.report['tampering'][
+ self.report['errors'][
"%s:%d" %
self.control_dns_server] = 'no_answer'
except:
- self.report['tampering'][
+ self.report['errors'][
"%s:%d" %
self.control_dns_server] = 'error'
control_answers = None
@@ -139,12 +143,14 @@ class DNSConsistencyTest(dnst.DNSTest):
test_dns_server)
except Exception:
log.err("Problem performing the DNS lookup")
- self.report['tampering'][test_resolver] = 'dns_lookup_error'
+ self.report['errors'][test_resolver] = 'dns_lookup_error'
+ self.report['failures'].append(test_resolver)
continue
if not experiment_answers:
log.err("Got no response, perhaps the DNS resolver is down?")
- self.report['tampering'][test_resolver] = 'no_answer'
+ self.report['errors'][test_resolver] = 'no_answer'
+ self.report['failures'].append(test_resolver)
continue
else:
log.debug(
@@ -165,12 +171,13 @@ class DNSConsistencyTest(dnst.DNSTest):
if not control_answers:
log.msg("Skipping control resolver comparison")
- self.report['tampering'][test_resolver] = None
+ self.report['errors'][test_resolver] = None
elif set(experiment_answers) & set(control_answers):
lookup_details()
log.msg("tampering: false")
- self.report['tampering'][test_resolver] = False
+ self.report['errors'][test_resolver] = False
+ self.report['successful'].append(test_resolver)
else:
log.msg("Trying to do reverse lookup")
experiment_reverse = yield self.performPTRLookup(experiment_answers[0],
@@ -182,12 +189,14 @@ class DNSConsistencyTest(dnst.DNSTest):
log.msg("Further testing has eliminated false positives")
lookup_details()
log.msg("tampering: reverse_match")
- self.report['tampering'][test_resolver] = 'reverse_match'
+ self.report['errors'][test_resolver] = 'reverse_match'
+ self.report['successful'].append(test_resolver)
else:
log.msg("Reverse lookups do not match")
lookup_details()
log.msg("tampering: true")
- self.report['tampering'][test_resolver] = True
+ self.report['errors'][test_resolver] = True
+ self.report['inconsistent'].append(test_resolver)
def inputProcessor(self, filename=None):
"""
diff --git a/ooni/reporter.py b/ooni/reporter.py
index db6f7c1..3ea1b10 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -236,6 +236,7 @@ class OONIBReporter(OReporter):
self.validateCollectorAddress()
self.reportID = None
+ self.supportedFormats = ["yaml"]
if self.collectorAddress.startswith('https://'):
# not sure if there's something else it needs. Seems to work.
@@ -257,27 +258,26 @@ class OONIBReporter(OReporter):
if not re.match(regexp, self.collectorAddress):
raise errors.InvalidOONIBCollectorAddress
- def serializeEntry(self, entry):
- if config.report.format == "json":
+ def serializeEntry(self, entry, serialisation_format="yaml"):
+ if serialisation_format == "json":
if isinstance(entry, Measurement):
report_entry = entry.testInstance.report
elif isinstance(entry, Failure):
- report_entry = entry.value
+ report_entry = {'failure': entry.value}
elif isinstance(entry, dict):
report_entry = entry
- report_entry["record_type"] = "entry"
- report_entry["report_id"] = self.reportID
- content = json.dumps(report_entry, ensure_ascii=True) + "\n"
+ return report_entry
else:
content = '---\n'
if isinstance(entry, Measurement):
- content += safe_dump(entry.testInstance.report)
+ report_entry = entry.testInstance.report
elif isinstance(entry, Failure):
- content += entry.value
+ report_entry = {'failure': entry.value}
elif isinstance(entry, dict):
- content += safe_dump(entry)
+ report_entry = entry
+ content += safe_dump(report_entry)
content += '...\n'
- return content
+ return content
@defer.inlineCallbacks
def writeReportEntry(self, entry):
@@ -285,8 +285,13 @@ class OONIBReporter(OReporter):
url = self.collectorAddress + '/report/' + self.reportID
- request = {'format': config.report.format,
- 'content': self.serializeEntry(entry)}
+ if "json" in self.supportedFormats:
+ serialisation_format = 'json'
+ else:
+ serialisation_format = 'yaml'
+
+ request = {'format': serialisation_format,
+ 'content': self.serializeEntry(entry, serialisation_format)}
log.debug("Updating report with id %s (%s)" % (self.reportID, url))
request_json = json.dumps(request)
@@ -295,12 +300,11 @@ class OONIBReporter(OReporter):
bodyProducer = StringProducer(request_json)
try:
- yield self.agent.request("PUT", url,
+ yield self.agent.request("PUT", str(url),
bodyProducer=bodyProducer)
- except:
- # XXX we must trap this in the runner and make sure to report the
- # data later.
+ except Exception as exc:
log.err("Error in writing report entry")
+ log.exception(exc)
raise errors.OONIBReportUpdateError
@defer.inlineCallbacks
@@ -324,21 +328,16 @@ class OONIBReporter(OReporter):
url = self.collectorAddress + '/report'
- content = '---\n'
- content += safe_dump(self.testDetails)
- content += '...\n'
-
request = {
'software_name': self.testDetails['software_name'],
'software_version': self.testDetails['software_version'],
'probe_asn': self.testDetails['probe_asn'],
+ 'probe_cc': self.testDetails['probe_cc'],
'test_name': self.testDetails['test_name'],
'test_version': self.testDetails['test_version'],
+ 'start_time': self.testDetails['start_time'],
'input_hashes': self.testDetails['input_hashes'],
- # XXX there is a bunch of redundancy in the arguments getting sent
- # to the backend. This may need to get changed in the client and
- # the backend.
- 'content': content
+ 'format': 'json'
}
# import values from the environment
request.update([(k.lower(),v) for (k,v) in os.environ.iteritems()
@@ -395,6 +394,9 @@ class OONIBReporter(OReporter):
self.reportID = parsed_response['report_id']
self.backendVersion = parsed_response['backend_version']
+
+ self.supportedFormats = parsed_response.get('supported_formats', ["yaml"])
+
log.debug("Created report with id %s" % parsed_response['report_id'])
defer.returnValue(parsed_response['report_id'])
diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py
index 7c9626c..e8ae189 100644
--- a/ooni/templates/dnst.py
+++ b/ooni/templates/dnst.py
@@ -58,13 +58,36 @@ def connectionLost(self, reason=None):
udp.Port.connectionLost = connectionLost
def representAnswer(answer):
- # We store the resource record and the answer payload in a
- # tuple
- return (repr(answer), repr(answer.payload))
+ answer_types = {
+ dns.SOA: 'SOA',
+ dns.NS: 'NS',
+ dns.PTR: 'PTR',
+ dns.A: 'A',
+ dns.CNAME: 'CNAME',
+ dns.MX: 'MX'
+ }
+ answer_type = answer_types.get(answer.type, 'unknown')
+ represented_answer = {
+ "answer_type": answer_type
+ }
+ if answer_type is 'SOA':
+ represented_answer['ttl'] = answer.payload.ttl
+ represented_answer['hostname'] = answer.payload.mname
+ represented_answer['responsible_name'] = answer.payload.rname
+ represented_answer['serial_number'] = answer.payload.serial
+ represented_answer['refresh_interval'] = answer.payload.refresh
+ represented_answer['retry_interval'] = answer.payload.retry
+ represented_answer['minimum_ttl'] = answer.payload.minimum
+ represented_answer['expiration_limit'] = answer.payload.expire
+ elif answer_type in ['NS', 'PTR', 'CNAME']:
+ represented_answer['hostname'] = answer.payload.name.name
+ elif answer_type is 'A':
+ represented_answer['ipv4'] = answer.payload.dottedQuad()
+ return represented_answer
class DNSTest(NetTestCase):
name = "Base DNS Test"
- version = 0.1
+ version = "0.2.0"
requiresRoot = False
queryTimeout = [1]
@@ -137,18 +160,25 @@ class DNSTest(NetTestCase):
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
- types={'NS':dns.NS,'A':dns.A,'SOA':dns.SOA,'PTR':dns.PTR}
- dnsType=types[dns_type]
+ types = {
+ 'NS': dns.NS,
+ 'A': dns.A,
+ 'SOA': dns.SOA,
+ 'PTR': dns.PTR
+ }
+ dnsType = types[dns_type]
query = [dns.Query(hostname, dnsType, dns.IN)]
def gotResponse(message):
- log.debug(dns_type+" Lookup successful")
+ log.debug(dns_type + " Lookup successful")
log.debug(str(message))
- addrs = []
- answers = []
+
if dns_server:
msg = message.answers
else:
msg = message[0]
+
+ answers = []
+ addrs = []
for answer in msg:
if answer.type is dnsType:
if dnsType is dns.SOA:
@@ -159,46 +189,53 @@ class DNSTest(NetTestCase):
addr = answer.payload.dottedQuad()
else:
addr = None
- addrs.append(addr)
+ addrs.append(addr)
answers.append(representAnswer(answer))
- DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type,
- answers=answers, addrs=addrs)
+ if dns_type == 'SOA':
+ for authority in message.authority:
+ answers.append(representAnswer(authority))
+
+ DNSTest.addToReport(self, query, resolver=dns_server,
+ query_type=dns_type, answers=answers)
return addrs
def gotError(failure):
failure.trap(gaierror, TimeoutError)
- DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type,
- failure=failure)
+ DNSTest.addToReport(self, query, resolver=dns_server,
+ query_type=dns_type, failure=failure)
return failure
if dns_server:
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
else:
- lookupFunction={'NS':client.lookupNameservers, 'SOA':client.lookupAuthority, 'A':client.lookupAddress, 'PTR':client.lookupPointer}
+ lookupFunction = {
+ 'NS': client.lookupNameservers,
+ 'SOA': client.lookupAuthority,
+ 'A': client.lookupAddress,
+ 'PTR': client.lookupPointer
+ }
d = lookupFunction[dns_type](hostname)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
-
def addToReport(self, query, resolver=None, query_type=None,
- answers=None, name=None, addrs=None, failure=None):
+ answers=None, failure=None):
log.debug("Adding %s to report)" % query)
result = {}
- result['resolver'] = resolver
+ result['resolver_hostname'] = resolver[0]
+ result['resolver_port'] = resolver[1]
result['query_type'] = query_type
- result['query'] = repr(query)
+ result['hostname'] = str(query[0].name)
+ result['failure'] = None
if failure:
result['failure'] = failureToString(failure)
+ result['answers'] = []
if answers:
result['answers'] = answers
- if name:
- result['name'] = name
- if addrs:
- result['addrs'] = addrs
self.report['queries'].append(result)
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 3702e86..15630d5 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -9,7 +9,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent
from ooni.nettest import NetTestCase
-from ooni.utils import log
+from ooni.utils import log, base64Dict
from ooni.settings import config
from ooni.utils.net import BodyReceiver, StringProducer, userAgents
@@ -122,11 +122,28 @@ class HTTPTest(NetTestCase):
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
+ def _representHeaders(headers):
+ represented_headers = {}
+ for name, value in headers.getAllRawHeaders():
+ represented_headers[name] = value[0]
+ return represented_headers
+
+ def _representBody(body):
+ # XXX perhaps add support for decoding gzip in the future.
+ try:
+ body.replace('\0', '')
+ body = unicode(body, 'ascii')
+ except UnicodeDecodeError:
+ try:
+ body = unicode(body, 'utf-8')
+ except UnicodeDecodeError:
+ body = base64Dict(body)
+
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
- request_response = {
+ session = {
'request': {
- 'headers': list(request_headers.getAllRawHeaders()),
+ 'headers': _representHeaders(request_headers),
'body': request['body'],
'url': request['url'],
'method': request['method'],
@@ -134,15 +151,16 @@ class HTTPTest(NetTestCase):
}
}
if response:
- request_response['response'] = {
- 'headers': list(response.headers.getAllRawHeaders()),
- 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '',
+ session['response'] = {
+ 'headers': _representHeaders(response.headers),
+ 'body': _representBody(response_body),
'code': response.code
- }
+ }
+ session['failure'] = None
if failure_string:
- request_response['failure'] = failure_string
+ session['failure'] = failure_string
- self.report['requests'].append(request_response)
+ self.report['requests'].append(session)
def _processResponseBody(self, response_body, request, response, body_processor):
log.debug("Processing response body")
@@ -302,7 +320,10 @@ class HTTPTest(NetTestCase):
request['url'] = url
request['headers'] = headers
request['body'] = body
- request['tor'] = {}
+ request['tor'] = {
+ 'exit_ip': None,
+ 'exit_name': None
+ }
if use_tor:
request['tor']['is_tor'] = True
else:
diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py
index 35dbf5b..f4b6280 100644
--- a/ooni/templates/scapyt.py
+++ b/ooni/templates/scapyt.py
@@ -1,11 +1,17 @@
from ooni.nettest import NetTestCase
-from ooni.utils import log
+from ooni.utils import log, base64Dict
from ooni.settings import config
from ooni.utils.net import hasRawSocketPermission
from ooni.utils.txscapy import ScapySender, ScapyFactory
+def _representPacket(packet):
+ return {
+ "raw_packet": base64Dict(str(packet)),
+ "summary": repr(packet)
+ }
+
class BaseScapyTest(NetTestCase):
"""
@@ -88,8 +94,8 @@ class BaseScapyTest(NetTestCase):
sent_packet.src = '127.0.0.1'
received_packet.dst = '127.0.0.1'
- self.report['sent_packets'].append(sent_packet)
- self.report['answered_packets'].append(received_packet)
+ self.report['sent_packets'].append(_representPacket(sent_packet))
+ self.report['answered_packets'].append(_representPacket(received_packet))
return packets
def sr(self, packets, timeout=None, *arg, **kw):
diff --git a/ooni/tests/test_trueheaders.py b/ooni/tests/test_trueheaders.py
index 8e9f053..f6c7812 100644
--- a/ooni/tests/test_trueheaders.py
+++ b/ooni/tests/test_trueheaders.py
@@ -23,19 +23,15 @@ dummy_headers_dict3 = {
class TestTrueHeaders(unittest.TestCase):
def test_names_match(self):
th = TrueHeaders(dummy_headers_dict)
- self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), set())
+ self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), [])
def test_names_not_match(self):
th = TrueHeaders(dummy_headers_dict)
- self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3']))
+ self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3'])
th = TrueHeaders(dummy_headers_dict3)
- self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3', 'Header4']))
+ self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3', 'Header4'])
def test_names_match_expect_ignore(self):
th = TrueHeaders(dummy_headers_dict)
- self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), set())
-
-
-
-
+ self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), [])
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 02904cc..5e9616b 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -5,6 +5,7 @@ import glob
import os
import gzip
+from base64 import b64encode
from zipfile import ZipFile
from ooni import otime
@@ -176,3 +177,9 @@ def gunzip(filename, dst):
def get_ooni_root():
script = os.path.join(__file__, '..')
return os.path.dirname(os.path.realpath(script))
+
+def base64Dict(data):
+ return {
+ 'format': 'base64',
+ 'data': b64encode(data)
+ }
diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py
index 149b440..b7671b5 100644
--- a/ooni/utils/trueheaders.py
+++ b/ooni/utils/trueheaders.py
@@ -80,7 +80,7 @@ class TrueHeaders(http_headers.Headers):
pass
else:
diff.add(name)
- return diff
+ return list(diff)
def getAllRawHeaders(self):
for k, v in self._rawHeaders.iteritems():
More information about the tor-commits
mailing list