[tor-commits] [ooni-probe/master] Add support for computing header difference and follow redirects

[art at torproject.org]

commit 8c5fdd48ebb760b72b1b02a83fc96a19f92293c7
Author: Arturo Filastò <arturo at filasto.net>
Date:   Sat Apr 16 14:30:29 2016 +0200

    Add support for computing header difference and follow redirects
---
 ooni/nettests/blocking/web_connectivity.py | 54 +++++++++++++++++++++---------
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index 175c6d3..4b9b6c2 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -81,6 +81,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     }
     requiresRoot = False
     requiresTor = False
+    followRedirects = True
 
     # Factor used to determine HTTP blockpage detection
     factor = 0.8
@@ -112,6 +113,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         self.report['client_resolver'] = self.resolverIp
         self.report['dns_consistency'] = None
         self.report['body_length_match'] = None
+        self.report['headers_match'] = None
+
         self.report['accessible'] = None
         self.report['blocking'] = None
 
@@ -139,6 +142,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         }
 
     def experiment_dns_query(self):
+        log.msg("Doing DNS query for {}".format(self.hostname))
         return self.performALookup(self.hostname)
 
     def tcp_connect(self, socket):
@@ -197,6 +201,22 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     def experiment_http_get_request(self):
         return self.doRequest(self.input, headers=REQUEST_HEADERS)
 
+    def compare_headers(self, experiment_http_response):
+        count = 0
+        control_headers_lower = {k.lower(): v for k, v in
+                self.report['control']['http_request']['headers'].items()}
+
+        for header_name, header_value in \
+                experiment_http_response.headers.getAllRawHeaders():
+            try:
+                control_headers_lower[header_name.lower()]
+            except KeyError:
+                log.msg("Did not find the key {}".format(header_name))
+                return False
+            count += 1
+
+        return count == len(self.report['control']['http_request']['headers'])
+
     def compare_body_lengths(self, experiment_http_response):
         control_body_length = self.control['http_request']['body_length']
         experiment_body_length = len(experiment_http_response.body)
@@ -213,10 +233,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
         self.report['body_proportion'] = rel
         if rel > float(self.factor):
-            self.report['body_length_match'] = True
             return True
         else:
-            self.report['body_length_match'] = False
             return False
 
     def compare_dns_experiments(self, experiment_dns_answers):
@@ -229,16 +247,13 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         experiment_ips = set(experiment_dns_answers)
 
         if control_ips == experiment_ips:
-            self.report['dns_consistency'] = 'consistent'
             return True
 
         for experiment_ip in experiment_ips:
             if is_public_ipv4_address(experiment_ip) is False:
-                self.report['dns_consistency'] = 'inconsistent'
                 return False
 
         if len(control_ips.intersection(experiment_ips)) > 0:
-            self.report['dns_consistency'] = 'consistent'
             return True
 
         experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
@@ -247,10 +262,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
                            control_ips))
 
         if len(control_asns.intersection(experiment_asns)) > 0:
-            self.report['dns_consistency'] = 'consistent'
             return True
 
-        self.report['dns_consistency'] = 'inconsistent'
         return False
 
     def compare_tcp_experiments(self):
@@ -268,22 +281,26 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
     def determine_blocking(self, experiment_http_response, experiment_dns_answers):
         blocking = False
-        body_length_match = None
-        dns_consistent = None
-        tcp_connect = None
 
         if (self.report['http_experiment_failure'] is None and
                     self.report['control']['http_request']['failure'] is None):
-            body_length_match = self.compare_body_lengths(experiment_http_response)
+            self.report['body_length_match'] = self.compare_body_lengths(
+                experiment_http_response)
+            self.report['headers_match'] = self.compare_headers(
+                experiment_http_response)
 
         dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
+        if dns_consistent is True:
+            self.report['dns_consistency'] = 'consistent'
+        else:
+            self.report['dns_consistency'] = 'inconsistent'
         tcp_connect = self.compare_tcp_experiments()
 
         if dns_consistent == True and tcp_connect == False:
             blocking = 'tcp_ip'
 
         elif (dns_consistent == True and tcp_connect == True and
-                      body_length_match == False):
+                      self.report['body_length_match'] == False):
             blocking = 'http'
 
         elif dns_consistent == False:
@@ -302,10 +319,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
             return []
         experiment_dns_answers = yield experiment_dns
 
+        port = 80
+        parsed_url = urlparse(self.input)
+        if parsed_url.port:
+            port = parsed_url.port
+        elif parsed_url.scheme == 'https':
+            port = 443
+
         sockets = []
-        for answer in experiment_dns_answers:
-            if is_public_ipv4_address(answer) is True:
-                sockets.append("%s:80" % answer)
+        for ip_address in experiment_dns_answers:
+            if is_public_ipv4_address(ip_address) is True:
+                sockets.append("{}:{}".format(ip_address, port))
 
         # STEALTH in here we should make changes to make the test more stealth
         dl = []