[tor-commits] [ooni-probe/master] Add support for Gzip Decoding in web connectivity test

[art at torproject.org]

commit f3cb68e06118ab7c20de9559404fae76715a5500
Author: Arturo Filastò <arturo at filasto.net>
Date:   Mon May 23 12:10:32 2016 +0200

    Add support for Gzip Decoding in web connectivity test
---
 ooni/nettests/blocking/web_connectivity.py | 36 +++++++++++++++++++++---------
 ooni/templates/httpt.py                    | 14 ++++++++++++
 2 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index 97cde60..f68b5aa 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -6,6 +6,7 @@ from urlparse import urlparse
 
 from ipaddr import IPv4Address, AddressValueError
 
+from twisted.web.client import GzipDecoder
 from twisted.internet import reactor
 from twisted.internet.protocol import Factory, Protocol
 from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -72,6 +73,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     author = "Arturo Filastò"
     version = "0.1.0"
 
+    contentDecoders = [('gzip', GzipDecoder)]
+
     usageOptions = UsageOptions
 
     inputFile = [
@@ -156,6 +159,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         self.report['dns_consistency'] = None
         self.report['body_length_match'] = None
         self.report['headers_match'] = None
+        self.report['status_code_match'] = None
 
         self.report['accessible'] = None
         self.report['blocking'] = None
@@ -177,8 +181,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
                 'ips': []
             },
             'http_request': {
-                'body_length': None,
-                'failure': True,
+                'body_length': -1,
+                'failure': None,
+                'status_code': -1,
                 'headers': {}
             }
         }
@@ -270,6 +275,19 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         else:
             return False
 
+    def compare_http_experiments(self, experiment_http_response):
+
+        self.report['body_length_match'] = \
+            self.compare_body_lengths(experiment_http_response)
+
+        self.report['headers_match'] = \
+            self.compare_headers(experiment_http_response)
+
+        self.report['status_code_match'] = (
+            experiment_http_response.code ==
+            self.control['http_request']['status_code']
+        )
+
     def compare_dns_experiments(self, experiment_dns_answers):
         if self.control['dns']['failure'] is not None and \
                 self.control['dns']['failure'] == self.report['dns_experiment_failure']:
@@ -319,7 +337,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     def determine_blocking(self, experiment_http_response, experiment_dns_answers):
         blocking = False
 
-        control_http_failure = self.report['control']['http_request']['failure']
+        control_http_failure = self.control['http_request']['failure']
         if control_http_failure is not None:
             control_http_failure = control_http_failure.split(" ")[0]
 
@@ -328,10 +346,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
             experiment_http_failure = experiment_http_failure.split(" ")[0]
 
         if (experiment_http_failure is None and control_http_failure is None):
-            self.report['body_length_match'] = self.compare_body_lengths(
-                experiment_http_response)
-            self.report['headers_match'] = self.compare_headers(
-                experiment_http_response)
+            self.compare_http_experiments(experiment_http_response)
 
         dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
         if dns_consistent is True:
@@ -341,7 +356,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         tcp_connect = self.compare_tcp_experiments()
 
         got_expected_web_page = (self.report['body_length_match'] or
-                                 self.report['headers_match'])
+                                 self.report['headers_match']) and \
+                                self.report['status_code_match']
 
         if (dns_consistent == True and tcp_connect == False and
                 experiment_http_failure is not None):
@@ -488,7 +504,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
             for reason, urls in summary['blocked'].items():
                 log.msg("")
-                log.msg("URLS blocked due to {}".format(reason))
-                log.msg("--------------------"+'-'*len(reason))
+                log.msg("URLS possibly blocked due to {}".format(reason))
+                log.msg("-----------------------------"+'-'*len(reason))
                 for url in urls:
                     log.msg("* {}".format(url))
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 7cbfd1d..ca0306b 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -5,8 +5,11 @@ import random
 from txtorcon.interface import StreamListenerMixin
 
 from twisted.web.client import readBody, PartialDownloadError
+from twisted.web.client import ContentDecoderAgent
+
 from twisted.internet import reactor
 from twisted.internet.endpoints import TCP4ClientEndpoint
+
 from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent
 from ooni.utils.trueheaders import FixedRedirectAgent
 
@@ -96,6 +99,12 @@ class HTTPTest(NetTestCase):
     randomizeUA = False
     followRedirects = False
 
+    # You can specify a list of tuples in the format of (CONTENT_TYPE,
+    # DECODER)
+    # For example to support Gzip decoding you should specify
+    # contentDecoders = [('gzip', GzipDecoder)]
+    contentDecoders = []
+
     baseParameters = [['socksproxy', 's', None,
         'Specify a socks proxy to use for requests (ip:port)']]
 
@@ -138,6 +147,11 @@ class HTTPTest(NetTestCase):
                         "(<= 10.1). I will not be able to follow redirects."\
                         "This may make the testing less precise.")
 
+        if len(self.contentDecoders) > 0:
+            self.control_agent = ContentDecoderAgent(self.control_agent,
+                                                     self.contentDecoders)
+            self.agent = ContentDecoderAgent(self.agent,
+                                             self.contentDecoders)
         self.processInputs()
         log.debug("Finished test setup")