[tor-commits] [ooni-probe/master] Many improvements to web_connectivity test
art at torproject.org
art at torproject.org
Mon May 30 16:28:32 UTC 2016
commit 7e1ee49e44480f62ac93a9f63aa90a64342df2ed
Author: Arturo Filastò <arturo at filasto.net>
Date: Thu Apr 14 22:54:49 2016 +0200
Many improvements to web_connectivity test
* Run the probe resolver detection inside of the setupClass
* Display a summary at the end of a test run summarising the results
* Make the blocking detection logic more robust
---
ooni/nettests/blocking/web_connectivity.py | 152 ++++++++++++++++++++---------
1 file changed, 108 insertions(+), 44 deletions(-)
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index d8276b3..320f614 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -8,6 +8,7 @@ from ipaddr import IPv4Address, AddressValueError
from twisted.internet import reactor
from twisted.internet.protocol import Factory, Protocol
from twisted.internet.endpoints import TCP4ClientEndpoint
+from twisted.names import client, dns
from twisted.internet import defer
from twisted.python import usage
@@ -19,6 +20,17 @@ from ooni.utils.net import StringProducer, BodyReceiver
from ooni.templates import httpt, dnst
from ooni.errors import failureToString
+REQUEST_HEADERS = {
+ 'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, '
+ 'like Gecko) Chrome/47.0.2526.106 Safari/537.36'],
+ 'Accept-Language': ['en-US;q=0.8,en;q=0.5'],
+ 'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,'
+ '*/*;q=0.8']
+}
+
+class InvalidControlResponse(Exception):
+ pass
+
class TCPConnectProtocol(Protocol):
def connectionMade(self):
self.transport.loseConnection()
@@ -72,6 +84,21 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
# Factor used to determine HTTP blockpage detection
factor = 0.8
+ resolverIp = None
+
+ @classmethod
+ @defer.inlineCallbacks
+ def setUpClass(cls):
+ try:
+ answers = yield client.lookupAddress(
+ cls.localOptions['dns-discovery']
+ )
+ assert len(answers) > 0
+ assert len(answers[0]) > 0
+ cls.resolverIp = answers[0][0].payload.dottedQuad()
+ except Exception as exc:
+ log.exception(exc)
+ log.err("Failed to lookup the resolver IP address")
def setUp(self):
"""
@@ -82,7 +109,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
if not self.input:
raise Exception("No input specified")
- self.report['client_resolver'] = None
+ self.report['client_resolver'] = self.resolverIp
self.report['dns_consistency'] = None
self.report['body_length_match'] = None
self.report['accessible'] = None
@@ -111,9 +138,6 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
}
}
- def dns_discovery(self):
- return self.performALookup(self.localOptions['dns-discovery'])
-
def experiment_dns_query(self):
return self.performALookup(self.hostname)
@@ -161,11 +185,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
finished = defer.Deferred()
response.deliverBody(BodyReceiver(finished, content_length))
body = yield finished
- self.control = json.loads(body)
+ try:
+ self.control = json.loads(body)
+ assert 'http_request' in self.control.keys()
+ assert 'tcp_connect' in self.control.keys()
+ assert 'dns' in self.control.keys()
+ except AssertionError, ValueError:
+ raise InvalidControlResponse(body)
self.report['control'] = self.control
def experiment_http_get_request(self):
- return self.doRequest(self.input)
+ return self.doRequest(self.input, headers=REQUEST_HEADERS)
def compare_body_lengths(self, experiment_http_response):
control_body_length = self.control['http_request']['body_length']
@@ -233,27 +263,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
return success
def determine_blocking(self, experiment_http_response, experiment_dns_answers):
- blocking = None
+ blocking = False
body_length_match = None
dns_consistent = None
tcp_connect = None
- if self.report['control_failure'] is None and \
- self.report['http_experiment_failure'] is None and \
- self.report['control']['http_request']['failure'] is None:
+ if (self.report['http_experiment_failure'] is None and
+ self.report['control']['http_request']['failure'] is None):
body_length_match = self.compare_body_lengths(experiment_http_response)
- if self.report['control_failure'] is None:
- dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
-
- if self.report['control_failure'] is None:
- tcp_connect = self.compare_tcp_experiments()
+ dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
+ tcp_connect = self.compare_tcp_experiments()
if dns_consistent == True and tcp_connect == False:
blocking = 'tcp_ip'
- elif dns_consistent == True and \
- tcp_connect == True and body_length_match == False:
+ elif (dns_consistent == True and tcp_connect == True and
+ body_length_match == False):
blocking = 'http'
elif dns_consistent == False:
@@ -270,28 +296,15 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
def dns_experiment_err(failure):
self.report['dns_experiment_failure'] = failureToString(failure)
return []
+ experiment_dns_answers = yield experiment_dns
- results = yield defer.DeferredList([
- self.dns_discovery(),
- experiment_dns
- ])
-
- self.report['client_resolver'] = None
- if results[0][0] == True:
- self.report['client_resolver'] = results[0][1][0]
-
- experiment_dns_answers = results[1][1]
sockets = []
for answer in experiment_dns_answers:
if is_public_ipv4_address(answer) is True:
sockets.append("%s:80" % answer)
- control_request = self.control_request(sockets)
- @control_request.addErrback
- def control_err(failure):
- self.report['control_failure'] = failureToString(failure)
-
- dl = [control_request]
+ # STEALTH in here we should make changes to make the test more stealth
+ dl = []
for socket in sockets:
dl.append(self.tcp_connect(socket))
results = yield defer.DeferredList(dl)
@@ -303,20 +316,71 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
experiment_http_response = yield experiment_http
- blocking = self.determine_blocking(experiment_http_response, experiment_dns_answers)
- self.report['blocking'] = blocking
+ control_request = self.control_request(sockets)
+ @control_request.addErrback
+ def control_err(failure):
+ log.err("Failed to perform control lookup")
+ self.report['control_failure'] = failureToString(failure)
+
+ yield control_request
- if blocking is not None:
- log.msg("%s: BLOCKING DETECTED due to %s" % (self.input, blocking))
+ if self.report['control_failure'] is None:
+ self.report['blocking'] = self.determine_blocking(experiment_http_response, experiment_dns_answers)
+
+ log.msg("")
+ log.msg("Result for %s" % self.input)
+ log.msg("-----------" + "-"*len(self.input))
+
+ if self.report['blocking'] is None:
+ log.msg("* Could not determine status of blocking due to "
+ "failing control request")
+ elif self.report['blocking'] is False:
+ log.msg("* No blocking detected")
else:
- log.msg("%s: No blocking detected" % self.input)
+ log.msg("* BLOCKING DETECTED due to %s" % (self.report['blocking']))
- if all(map(lambda x: x == None, [self.report['http_experiment_failure'],
- self.report['dns_experiment_failure'],
- blocking])):
- log.msg("")
+ if (self.report['http_experiment_failure'] == None and
+ self.report['dns_experiment_failure'] == None and
+ self.report['blocking'] in (False, None)):
self.report['accessible'] = True
- log.msg("%s: is accessible" % self.input)
+ log.msg("* Is accessible")
else:
- log.msg("%s: is NOT accessible" % self.input)
+ log.msg("* Is NOT accessible")
self.report['accessible'] = False
+
+ def postProcessor(self, measurements):
+ self.summary['accessible'] = self.summary.get('accessible', [])
+ self.summary['not-accessible'] = self.summary.get('not-accessible', [])
+ self.summary['blocked'] = self.summary.get('blocked', [])
+
+ if self.report['blocking'] not in (False, None):
+ self.summary['blocked'].append((self.input,
+ self.report['blocking']))
+ if self.report['accessible'] is True:
+ self.summary['accessible'].append(self.input)
+ else:
+ self.summary['not-accessible'].append(self.input)
+ return self.report
+
+ def displaySummary(self, summary):
+
+ if len(summary['accessible']) > 0:
+ log.msg("")
+ log.msg("Accessible URLS")
+ log.msg("---------------")
+ for url in summary['accessible']:
+ log.msg("* {}".format(url))
+
+ if len(summary['not-accessible']) > 0:
+ log.msg("")
+ log.msg("Not accessible URLS")
+ log.msg("---------------")
+ for url in summary['not-accessible']:
+ log.msg("* {}".format(url))
+
+ if len(summary['blocked']) > 0:
+ log.msg("")
+ log.msg("Blocked URLS")
+ log.msg("------------")
+ for url, reason in summary['blocked']:
+ log.msg("* {} due to {}".format(url, reason))
More information about the tor-commits
mailing list