commit f3cb68e06118ab7c20de9559404fae76715a5500 Author: Arturo Filastò arturo@filasto.net Date: Mon May 23 12:10:32 2016 +0200
Add support for Gzip Decoding in web connectivity test --- ooni/nettests/blocking/web_connectivity.py | 36 +++++++++++++++++++++--------- ooni/templates/httpt.py | 14 ++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-)
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 97cde60..f68b5aa 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -6,6 +6,7 @@ from urlparse import urlparse
from ipaddr import IPv4Address, AddressValueError
+from twisted.web.client import GzipDecoder from twisted.internet import reactor from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint @@ -72,6 +73,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): author = "Arturo Filastò" version = "0.1.0"
+ contentDecoders = [('gzip', GzipDecoder)] + usageOptions = UsageOptions
inputFile = [ @@ -156,6 +159,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = None self.report['body_length_match'] = None self.report['headers_match'] = None + self.report['status_code_match'] = None
self.report['accessible'] = None self.report['blocking'] = None @@ -177,8 +181,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): 'ips': [] }, 'http_request': { - 'body_length': None, - 'failure': True, + 'body_length': -1, + 'failure': None, + 'status_code': -1, 'headers': {} } } @@ -270,6 +275,19 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): else: return False
+ def compare_http_experiments(self, experiment_http_response): + + self.report['body_length_match'] = \ + self.compare_body_lengths(experiment_http_response) + + self.report['headers_match'] = \ + self.compare_headers(experiment_http_response) + + self.report['status_code_match'] = ( + experiment_http_response.code == + self.control['http_request']['status_code'] + ) + def compare_dns_experiments(self, experiment_dns_answers): if self.control['dns']['failure'] is not None and \ self.control['dns']['failure'] == self.report['dns_experiment_failure']: @@ -319,7 +337,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def determine_blocking(self, experiment_http_response, experiment_dns_answers): blocking = False
- control_http_failure = self.report['control']['http_request']['failure'] + control_http_failure = self.control['http_request']['failure'] if control_http_failure is not None: control_http_failure = control_http_failure.split(" ")[0]
@@ -328,10 +346,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http_failure = experiment_http_failure.split(" ")[0]
if (experiment_http_failure is None and control_http_failure is None): - self.report['body_length_match'] = self.compare_body_lengths( - experiment_http_response) - self.report['headers_match'] = self.compare_headers( - experiment_http_response) + self.compare_http_experiments(experiment_http_response)
dns_consistent = self.compare_dns_experiments(experiment_dns_answers) if dns_consistent is True: @@ -341,7 +356,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): tcp_connect = self.compare_tcp_experiments()
got_expected_web_page = (self.report['body_length_match'] or - self.report['headers_match']) + self.report['headers_match']) and \ + self.report['status_code_match']
if (dns_consistent == True and tcp_connect == False and experiment_http_failure is not None): @@ -488,7 +504,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
for reason, urls in summary['blocked'].items(): log.msg("") - log.msg("URLS blocked due to {}".format(reason)) - log.msg("--------------------"+'-'*len(reason)) + log.msg("URLS possibly blocked due to {}".format(reason)) + log.msg("-----------------------------"+'-'*len(reason)) for url in urls: log.msg("* {}".format(url)) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 7cbfd1d..ca0306b 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -5,8 +5,11 @@ import random from txtorcon.interface import StreamListenerMixin
from twisted.web.client import readBody, PartialDownloadError +from twisted.web.client import ContentDecoderAgent + from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint + from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent from ooni.utils.trueheaders import FixedRedirectAgent
@@ -96,6 +99,12 @@ class HTTPTest(NetTestCase): randomizeUA = False followRedirects = False
+ # You can specify a list of tuples in the format of (CONTENT_TYPE, + # DECODER) + # For example to support Gzip decoding you should specify + # contentDecoders = [('gzip', GzipDecoder)] + contentDecoders = [] + baseParameters = [['socksproxy', 's', None, 'Specify a socks proxy to use for requests (ip:port)']]
@@ -138,6 +147,11 @@ class HTTPTest(NetTestCase): "(<= 10.1). I will not be able to follow redirects."\ "This may make the testing less precise.")
+ if len(self.contentDecoders) > 0: + self.control_agent = ContentDecoderAgent(self.control_agent, + self.contentDecoders) + self.agent = ContentDecoderAgent(self.agent, + self.contentDecoders) self.processInputs() log.debug("Finished test setup")