[tor-commits] [ooni-probe/master] Implement basic web connectivity test

art at torproject.org art at torproject.org
Mon May 30 16:28:32 UTC 2016


commit d92a95b628167043a84ceabac77e45f3cd319d3d
Author: Arturo Filastò <arturo at filasto.net>
Date:   Wed Feb 3 19:36:11 2016 +0100

    Implement basic web connectivity test
    
    * Various hacky fixes to enforce correct report format and handling of report_id
---
 ooni/director.py                           |  5 ++
 ooni/nettest.py                            |  6 ++-
 ooni/nettests/blocking/web_connectivity.py | 86 ++++++++++++++++++++++++------
 ooni/reporter.py                           |  7 ++-
 ooni/tasks.py                              |  3 ++
 ooni/templates/httpt.py                    | 19 +++----
 6 files changed, 94 insertions(+), 32 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py
index 02619c2..13a4503 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -258,6 +258,11 @@ class Director(object):
 
         yield net_test.report.open()
 
+        # XXX this needs some serious refactoring
+        net_test_loader.reportID = report.reportID
+        net_test.reportID = report.reportID
+        net_test.testDetails['report_id'] = report.reportID
+
         yield net_test.initializeInputProcessor()
         try:
             self.activeNetTests.append(net_test)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 074e8c2..166451a 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -161,6 +161,8 @@ class NetTestLoader(object):
     method_prefix = 'test'
     collector = None
     yamloo = True
+    requiresTor = False
+    reportID = None
 
     def __init__(self, options, test_file=None, test_string=None,
                  annotations={}):
@@ -568,10 +570,12 @@ class NetTest(object):
 
     @defer.inlineCallbacks
     def initializeInputProcessor(self):
-        for test_class, test_method in self.testCases:
+        for test_class, _ in self.testCases:
             test_class.inputs = yield defer.maybeDeferred(
                 test_class().getInputProcessor
             )
+            if not test_class.inputs:
+                test_class.inputs = [None]
 
     def generateMeasurements(self):
         """
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index bc23f6d..8d83ad9 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -3,6 +3,8 @@
 import json
 from urlparse import urlparse
 
+from ipaddr import IPv4Address, AddressValueError
+
 from twisted.internet import reactor
 from twisted.internet.protocol import Factory, Protocol
 from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -10,6 +12,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet import defer
 from twisted.python import usage
 
+from ooni import geoip
 from ooni.utils import log
 
 from ooni.utils.net import StringProducer, BodyReceiver
@@ -34,6 +37,16 @@ class UsageOptions(usage.Options):
     ]
 
 
+def is_public_ipv4_address(address):
+    try:
+        ip_address = IPv4Address(address)
+        if not any([ip_address.is_private,
+                    ip_address.is_loopback]):
+            return True
+        return False
+    except AddressValueError:
+        return None
+
 class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     """
     Web connectivity
@@ -51,8 +64,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     ]
 
     requiredTestHelpers = {
-        'backend': 'web_connectivity',
-        'dns-discovery': 'dns_discovery'
+        'backend': 'web-connectivity',
+        'dns-discovery': 'dns-discovery'
     }
     requiresRoot = False
     requiresTor = False
@@ -78,8 +91,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         self.report['control_failure'] = None
         self.report['experiment_failure'] = None
 
-        self.report['tcp_connect'] = [
-        ]
+        self.report['tcp_connect'] = []
+        self.report['control'] = {}
 
         self.hostname = urlparse(self.input).netloc
         if not self.hostname:
@@ -147,6 +160,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         response.deliverBody(BodyReceiver(finished, content_length))
         body = yield finished
         self.control = json.loads(body)
+        self.report['control'] = self.control
 
     def experiment_http_get_request(self):
         return self.doRequest(self.input)
@@ -168,33 +182,48 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         self.report['body_proportion'] = rel
         if rel > float(self.factor):
             self.report['body_length_match'] = True
-            return None
+            return True
         else:
             self.report['body_length_match'] = False
-            return 'http'
+            return False
 
     def compare_dns_experiments(self, experiment_dns_answers):
         control_ips = set(self.control['dns']['ips'])
         experiment_ips = set(experiment_dns_answers)
 
+        for experiment_ip in experiment_ips:
+            if is_public_ipv4_address(experiment_ip) is False:
+                self.report['dns_consistency'] = 'inconsistent'
+                return False
+
         if len(control_ips.intersection(experiment_ips)) > 0:
             self.report['dns_consistency'] = 'consistent'
-        else:
-            self.report['dns_consistency'] = 'inconsistent'
+            return True
+
+        experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
+                              experiment_ips))
+        control_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
+                           control_ips))
+
+        if len(control_asns.intersection(experiment_asns)) > 0:
+            self.report['dns_consistency'] = 'consistent'
+            return True
+
+        self.report['dns_consistency'] = 'inconsistent'
+        return False
 
     def compare_tcp_experiments(self):
-        blocking = False
+        success = True
         for idx, result in enumerate(self.report['tcp_connect']):
             socket = "%s:%s" % (result['ip'], result['port'])
             control_status = self.control['tcp_connect'][socket]
-            log.debug(str(result))
             if result['status']['success'] == False and \
                     control_status['status'] == True:
                 self.report['tcp_connect'][idx]['status']['blocked'] = True
-                blocking = 'tcp_ip'
+                success = False
             else:
                 self.report['tcp_connect'][idx]['status']['blocked'] = False
-        return blocking
+        return success
 
     @defer.inlineCallbacks
     def test_web_connectivity(self):
@@ -208,8 +237,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
             self.report['client_resolver']  = results[0][1][1]
 
         experiment_dns_answers = results[1][1]
-
-        sockets = map(lambda x: "%s:80" % x, results[1][1])
+        sockets = []
+        for answer in experiment_dns_answers:
+            if is_public_ipv4_address(answer) is True:
+                sockets.append("%s:80" % answer)
 
         control_request = self.control_request(sockets)
         @control_request.addErrback
@@ -228,12 +259,33 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
         experiment_http_response = yield experiment_http
 
+        blocking = None
+        body_length_match = None
+        dns_consistent = None
+        tcp_connect = None
+
         if self.report['control_failure'] is None and \
                 self.report['experiment_failure'] is None:
-            self.compare_body_lenghts(experiment_http_response)
+            body_length_match = self.compare_body_lengths(experiment_http_response)
 
         if self.report['control_failure'] is None:
-            self.compare_dns_experiments(experiment_dns_answers)
+            dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
 
         if self.report['control_failure'] is None:
-            self.compare_tcp_experiments()
+            tcp_connect = self.compare_tcp_experiments()
+
+        if dns_consistent == True and tcp_connect == False:
+            blocking = 'tcp_ip'
+
+        elif dns_consistent == True and \
+                tcp_connect == True and body_length_match == False:
+            blocking = 'http'
+
+        elif dns_consistent == False:
+            blocking = 'dns'
+
+        self.report['blocking'] = blocking
+        if blocking is not None:
+            log.msg("Blocking detected on %s due to %s" % (self.input, blocking))
+        else:
+            log.msg("No blocking detected on %s" % self.input)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 66a3e86..49df443 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -4,6 +4,8 @@ import json
 import os
 import re
 
+from copy import deepcopy
+
 from datetime import datetime
 from contextlib import contextmanager
 
@@ -192,9 +194,9 @@ class YAMLReporter(OReporter):
         log.debug("Writing report with YAML reporter")
         content = '---\n'
         if isinstance(entry, Measurement):
-            report_entry = entry.testInstance.report
+            report_entry = deepcopy(entry.testInstance.report)
         elif isinstance(entry, dict):
-            report_entry = entry
+            report_entry = deepcopy(entry)
         else:
             raise Exception("Failed to serialise entry")
         content += safe_dump(report_entry)
@@ -576,6 +578,7 @@ class OONIBReportLog(object):
 
 
 class Report(object):
+    reportID = None
 
     def __init__(self, test_details, report_filename,
                  reportEntryManager, collector_address=None,
diff --git a/ooni/tasks.py b/ooni/tasks.py
index 72e211f..2c4f07d 100644
--- a/ooni/tasks.py
+++ b/ooni/tasks.py
@@ -115,6 +115,9 @@ class Measurement(TaskWithTimeout):
 
         if 'input' not in self.testInstance.report.keys():
             self.testInstance.report['input'] = test_input
+        if 'test_start_time' not in self.testInstance.report.keys():
+            start_time = otime.epochToNewTimestamp(self.testInstance._start_time)
+            self.testInstance.report['test_start_time'] = start_time
 
         self.testInstance.setUp()
 
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 73bd5db..2b280f0 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -1,10 +1,9 @@
 import re
 import random
 
-from twisted.internet import defer
-
 from txtorcon.interface import StreamListenerMixin
 
+from twisted.web.client import readBody, PartialDownloadError
 from twisted.internet import reactor
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent
@@ -13,7 +12,7 @@ from ooni.nettest import NetTestCase
 from ooni.utils import log, base64Dict
 from ooni.settings import config
 
-from ooni.utils.net import BodyReceiver, StringProducer, userAgents
+from ooni.utils.net import StringProducer, userAgents
 from ooni.utils.trueheaders import TrueHeaders
 from ooni.errors import handleAllFailures
 
@@ -197,6 +196,8 @@ class HTTPTest(NetTestCase):
         return response
 
     def _processResponseBodyFail(self, failure, request, response):
+        if failure.check(PartialDownloadError):
+            return failure.value.response
         failure_string = handleAllFailures(failure)
         HTTPTest.addToReport(self, request, response,
                              failure_string=failure_string)
@@ -281,17 +282,11 @@ class HTTPTest(NetTestCase):
         else:
             self.processResponseHeaders(response_headers_dict)
 
-        try:
-            content_length = int(response.headers.getRawHeaders('content-length')[0])
-        except Exception:
-            content_length = None
-
-        finished = defer.Deferred()
-        response.deliverBody(BodyReceiver(finished, content_length))
-        finished.addCallback(self._processResponseBody, request,
-                response, body_processor)
+        finished = readBody(response)
         finished.addErrback(self._processResponseBodyFail, request,
                             response)
+        finished.addCallback(self._processResponseBody, request,
+                response, body_processor)
         return finished
 
     def doRequest(self, url, method="GET",





More information about the tor-commits mailing list