commit d92a95b628167043a84ceabac77e45f3cd319d3d Author: Arturo Filastò arturo@filasto.net Date: Wed Feb 3 19:36:11 2016 +0100

Implement basic web connectivity test

* Various hacky fixes to enforce correct report format and handling of report_id --- ooni/director.py | 5 ++ ooni/nettest.py | 6 ++- ooni/nettests/blocking/web_connectivity.py | 86 ++++++++++++++++++++++++------ ooni/reporter.py | 7 ++- ooni/tasks.py | 3 ++ ooni/templates/httpt.py | 19 +++---- 6 files changed, 94 insertions(+), 32 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py index 02619c2..13a4503 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -258,6 +258,11 @@ class Director(object):

yield net_test.report.open()

+ # XXX this needs some serious refactoring + net_test_loader.reportID = report.reportID + net_test.reportID = report.reportID + net_test.testDetails['report_id'] = report.reportID + yield net_test.initializeInputProcessor() try: self.activeNetTests.append(net_test) diff --git a/ooni/nettest.py b/ooni/nettest.py index 074e8c2..166451a 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -161,6 +161,8 @@ class NetTestLoader(object): method_prefix = 'test' collector = None yamloo = True + requiresTor = False + reportID = None

def __init__(self, options, test_file=None, test_string=None, annotations={}): @@ -568,10 +570,12 @@ class NetTest(object):

@defer.inlineCallbacks def initializeInputProcessor(self): - for test_class, test_method in self.testCases: + for test_class, _ in self.testCases: test_class.inputs = yield defer.maybeDeferred( test_class().getInputProcessor ) + if not test_class.inputs: + test_class.inputs = [None]

def generateMeasurements(self): """ diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index bc23f6d..8d83ad9 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -3,6 +3,8 @@ import json from urlparse import urlparse

+from ipaddr import IPv4Address, AddressValueError + from twisted.internet import reactor from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint @@ -10,6 +12,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet import defer from twisted.python import usage

+from ooni import geoip from ooni.utils import log

from ooni.utils.net import StringProducer, BodyReceiver @@ -34,6 +37,16 @@ class UsageOptions(usage.Options): ]

+def is_public_ipv4_address(address): + try: + ip_address = IPv4Address(address) + if not any([ip_address.is_private, + ip_address.is_loopback]): + return True + return False + except AddressValueError: + return None + class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): """ Web connectivity @@ -51,8 +64,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): ]

requiredTestHelpers = { - 'backend': 'web_connectivity', - 'dns-discovery': 'dns_discovery' + 'backend': 'web-connectivity', + 'dns-discovery': 'dns-discovery' } requiresRoot = False requiresTor = False @@ -78,8 +91,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['control_failure'] = None self.report['experiment_failure'] = None

- self.report['tcp_connect'] = [ - ] + self.report['tcp_connect'] = [] + self.report['control'] = {}

self.hostname = urlparse(self.input).netloc if not self.hostname: @@ -147,6 +160,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): response.deliverBody(BodyReceiver(finished, content_length)) body = yield finished self.control = json.loads(body) + self.report['control'] = self.control

def experiment_http_get_request(self): return self.doRequest(self.input) @@ -168,33 +182,48 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['body_proportion'] = rel if rel > float(self.factor): self.report['body_length_match'] = True - return None + return True else: self.report['body_length_match'] = False - return 'http' + return False

def compare_dns_experiments(self, experiment_dns_answers): control_ips = set(self.control['dns']['ips']) experiment_ips = set(experiment_dns_answers)

+ for experiment_ip in experiment_ips: + if is_public_ipv4_address(experiment_ip) is False: + self.report['dns_consistency'] = 'inconsistent' + return False + if len(control_ips.intersection(experiment_ips)) > 0: self.report['dns_consistency'] = 'consistent' - else: - self.report['dns_consistency'] = 'inconsistent' + return True + + experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], + experiment_ips)) + control_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], + control_ips)) + + if len(control_asns.intersection(experiment_asns)) > 0: + self.report['dns_consistency'] = 'consistent' + return True + + self.report['dns_consistency'] = 'inconsistent' + return False

def compare_tcp_experiments(self): - blocking = False + success = True for idx, result in enumerate(self.report['tcp_connect']): socket = "%s:%s" % (result['ip'], result['port']) control_status = self.control['tcp_connect'][socket] - log.debug(str(result)) if result['status']['success'] == False and \ control_status['status'] == True: self.report['tcp_connect'][idx]['status']['blocked'] = True - blocking = 'tcp_ip' + success = False else: self.report['tcp_connect'][idx]['status']['blocked'] = False - return blocking + return success

@defer.inlineCallbacks def test_web_connectivity(self): @@ -208,8 +237,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['client_resolver'] = results[0][1][1]

experiment_dns_answers = results[1][1] - - sockets = map(lambda x: "%s:80" % x, results[1][1]) + sockets = [] + for answer in experiment_dns_answers: + if is_public_ipv4_address(answer) is True: + sockets.append("%s:80" % answer)

control_request = self.control_request(sockets) @control_request.addErrback @@ -228,12 +259,33 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):

experiment_http_response = yield experiment_http

+ blocking = None + body_length_match = None + dns_consistent = None + tcp_connect = None + if self.report['control_failure'] is None and \ self.report['experiment_failure'] is None: - self.compare_body_lenghts(experiment_http_response) + body_length_match = self.compare_body_lengths(experiment_http_response)

if self.report['control_failure'] is None: - self.compare_dns_experiments(experiment_dns_answers) + dns_consistent = self.compare_dns_experiments(experiment_dns_answers)

if self.report['control_failure'] is None: - self.compare_tcp_experiments() + tcp_connect = self.compare_tcp_experiments() + + if dns_consistent == True and tcp_connect == False: + blocking = 'tcp_ip' + + elif dns_consistent == True and \ + tcp_connect == True and body_length_match == False: + blocking = 'http' + + elif dns_consistent == False: + blocking = 'dns' + + self.report['blocking'] = blocking + if blocking is not None: + log.msg("Blocking detected on %s due to %s" % (self.input, blocking)) + else: + log.msg("No blocking detected on %s" % self.input) diff --git a/ooni/reporter.py b/ooni/reporter.py index 66a3e86..49df443 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -4,6 +4,8 @@ import json import os import re

+from copy import deepcopy + from datetime import datetime from contextlib import contextmanager

@@ -192,9 +194,9 @@ class YAMLReporter(OReporter): log.debug("Writing report with YAML reporter") content = '---\n' if isinstance(entry, Measurement): - report_entry = entry.testInstance.report + report_entry = deepcopy(entry.testInstance.report) elif isinstance(entry, dict): - report_entry = entry + report_entry = deepcopy(entry) else: raise Exception("Failed to serialise entry") content += safe_dump(report_entry) @@ -576,6 +578,7 @@ class OONIBReportLog(object):

class Report(object): + reportID = None

def __init__(self, test_details, report_filename, reportEntryManager, collector_address=None, diff --git a/ooni/tasks.py b/ooni/tasks.py index 72e211f..2c4f07d 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -115,6 +115,9 @@ class Measurement(TaskWithTimeout):

if 'input' not in self.testInstance.report.keys(): self.testInstance.report['input'] = test_input + if 'test_start_time' not in self.testInstance.report.keys(): + start_time = otime.epochToNewTimestamp(self.testInstance._start_time) + self.testInstance.report['test_start_time'] = start_time

self.testInstance.setUp()

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 73bd5db..2b280f0 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,10 +1,9 @@ import re import random

-from twisted.internet import defer - from txtorcon.interface import StreamListenerMixin

+from twisted.web.client import readBody, PartialDownloadError from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent @@ -13,7 +12,7 @@ from ooni.nettest import NetTestCase from ooni.utils import log, base64Dict from ooni.settings import config

-from ooni.utils.net import BodyReceiver, StringProducer, userAgents +from ooni.utils.net import StringProducer, userAgents from ooni.utils.trueheaders import TrueHeaders from ooni.errors import handleAllFailures

@@ -197,6 +196,8 @@ class HTTPTest(NetTestCase): return response

def _processResponseBodyFail(self, failure, request, response): + if failure.check(PartialDownloadError): + return failure.value.response failure_string = handleAllFailures(failure) HTTPTest.addToReport(self, request, response, failure_string=failure_string) @@ -281,17 +282,11 @@ class HTTPTest(NetTestCase): else: self.processResponseHeaders(response_headers_dict)

- try: - content_length = int(response.headers.getRawHeaders('content-length')[0]) - except Exception: - content_length = None - - finished = defer.Deferred() - response.deliverBody(BodyReceiver(finished, content_length)) - finished.addCallback(self._processResponseBody, request, - response, body_processor) + finished = readBody(response) finished.addErrback(self._processResponseBodyFail, request, response) + finished.addCallback(self._processResponseBody, request, + response, body_processor) return finished

def doRequest(self, url, method="GET",