May 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Add support for Gzip Decoding in web connectivity test
by art＠torproject.org 30 May '16

30 May '16

commit f3cb68e06118ab7c20de9559404fae76715a5500 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 23 12:10:32 2016 +0200 Add support for Gzip Decoding in web connectivity test --- ooni/nettests/blocking/web_connectivity.py | 36 +++++++++++++++++++++--------- ooni/templates/httpt.py | 14 ++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 97cde60..f68b5aa 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -6,6 +6,7 @@ from urlparse import urlparse from ipaddr import IPv4Address, AddressValueError +from twisted.web.client import GzipDecoder from twisted.internet import reactor from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint @@ -72,6 +73,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): author = "Arturo Filastò" version = "0.1.0" + contentDecoders = [('gzip', GzipDecoder)] + usageOptions = UsageOptions inputFile = [ @@ -156,6 +159,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = None self.report['body_length_match'] = None self.report['headers_match'] = None + self.report['status_code_match'] = None self.report['accessible'] = None self.report['blocking'] = None @@ -177,8 +181,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): 'ips': [] }, 'http_request': { - 'body_length': None, - 'failure': True, + 'body_length': -1, + 'failure': None, + 'status_code': -1, 'headers': {} } } @@ -270,6 +275,19 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): else: return False + def compare_http_experiments(self, experiment_http_response): + + self.report['body_length_match'] = \ + self.compare_body_lengths(experiment_http_response) + + self.report['headers_match'] = \ + self.compare_headers(experiment_http_response) + + self.report['status_code_match'] = ( + experiment_http_response.code == + self.control['http_request']['status_code'] + ) + def compare_dns_experiments(self, experiment_dns_answers): if self.control['dns']['failure'] is not None and \ self.control['dns']['failure'] == self.report['dns_experiment_failure']: @@ -319,7 +337,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def determine_blocking(self, experiment_http_response, experiment_dns_answers): blocking = False - control_http_failure = self.report['control']['http_request']['failure'] + control_http_failure = self.control['http_request']['failure'] if control_http_failure is not None: control_http_failure = control_http_failure.split(" ")[0] @@ -328,10 +346,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http_failure = experiment_http_failure.split(" ")[0] if (experiment_http_failure is None and control_http_failure is None): - self.report['body_length_match'] = self.compare_body_lengths( - experiment_http_response) - self.report['headers_match'] = self.compare_headers( - experiment_http_response) + self.compare_http_experiments(experiment_http_response) dns_consistent = self.compare_dns_experiments(experiment_dns_answers) if dns_consistent is True: @@ -341,7 +356,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): tcp_connect = self.compare_tcp_experiments() got_expected_web_page = (self.report['body_length_match'] or - self.report['headers_match']) + self.report['headers_match']) and \ + self.report['status_code_match'] if (dns_consistent == True and tcp_connect == False and experiment_http_failure is not None): @@ -488,7 +504,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): for reason, urls in summary['blocked'].items(): log.msg("") - log.msg("URLS blocked due to {}".format(reason)) - log.msg("--------------------"+'-'*len(reason)) + log.msg("URLS possibly blocked due to {}".format(reason)) + log.msg("-----------------------------"+'-'*len(reason)) for url in urls: log.msg("* {}".format(url)) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 7cbfd1d..ca0306b 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -5,8 +5,11 @@ import random from txtorcon.interface import StreamListenerMixin from twisted.web.client import readBody, PartialDownloadError +from twisted.web.client import ContentDecoderAgent + from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint + from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent from ooni.utils.trueheaders import FixedRedirectAgent @@ -96,6 +99,12 @@ class HTTPTest(NetTestCase): randomizeUA = False followRedirects = False + # You can specify a list of tuples in the format of (CONTENT_TYPE, + # DECODER) + # For example to support Gzip decoding you should specify + # contentDecoders = [('gzip', GzipDecoder)] + contentDecoders = [] + baseParameters = [['socksproxy', 's', None, 'Specify a socks proxy to use for requests (ip:port)']] @@ -138,6 +147,11 @@ class HTTPTest(NetTestCase): "(<= 10.1). I will not be able to follow redirects."\ "This may make the testing less precise.") + if len(self.contentDecoders) > 0: + self.control_agent = ContentDecoderAgent(self.control_agent, + self.contentDecoders) + self.agent = ContentDecoderAgent(self.agent, + self.contentDecoders) self.processInputs() log.debug("Finished test setup")

1 0

[ooni-probe/master] Don't require the bouncer key to be present
by art＠torproject.org 30 May '16

30 May '16

commit 156ce3f630eb58a061841dfa6d6f6f6cf6755d83 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue May 17 20:24:15 2016 +0200 Don't require the bouncer key to be present --- ooni/deck.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 404a4d0..7a3b697 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -136,11 +136,11 @@ class Deck(InputFile): net_test_loader = NetTestLoader(test['options']['subargs'], annotations=test['options'].get('annotations', {}), test_file=nettest_path) - if test['options']['collector'] is not None: + if test['options'].get('collector', None) is not None: net_test_loader.collector = CollectorClient( test['options']['collector'] ) - if test['options']['bouncer'] is not None: + if test['options'].get('bouncer', None) is not None: self.bouncer = test['options']['bouncer'] self.insert(net_test_loader)

1 0

[ooni-probe/master] Various improvement related to backend_client
by art＠torproject.org 30 May '16

30 May '16

commit 24ea012b508526165b1a7d1eef1d76475bd313e6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue May 17 20:02:09 2016 +0200 Various improvement related to backend_client * Include web_connectivity in default generated deck * Fix serialisation of HTTP Headers * Support setting collector and bouncer in oonireport upload --- ooni/deck.py | 4 +++- ooni/deckgen/cli.py | 22 +++++++--------------- ooni/report/cli.py | 2 +- ooni/report/tool.py | 30 +++++++++++++++++++++--------- ooni/reporter.py | 3 ++- ooni/templates/httpt.py | 2 +- 6 files changed, 35 insertions(+), 28 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 007097e..404a4d0 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -136,10 +136,12 @@ class Deck(InputFile): net_test_loader = NetTestLoader(test['options']['subargs'], annotations=test['options'].get('annotations', {}), test_file=nettest_path) - if test['options']['collector']: + if test['options']['collector'] is not None: net_test_loader.collector = CollectorClient( test['options']['collector'] ) + if test['options']['bouncer'] is not None: + self.bouncer = test['options']['bouncer'] self.insert(net_test_loader) def insert(self, net_test_loader): diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index 2e0e976..a6046f3 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -27,6 +27,7 @@ class Options(usage.Options): "generate the deck."], ["collector", None, None, "Specify a custom collector to use when " "submitting reports"], + ["bouncer", None, None, "Specify a custom bouncer to use"], ["output", "o", None, "Specify the directory where to write output."] ] @@ -44,7 +45,6 @@ class Deck(object): "annotations": None, "collector": None, - # XXX setting this is currently not supported "bouncer": None, "reportfile": None, @@ -56,13 +56,15 @@ class Deck(object): } } - def __init__(self, collector=None): + def __init__(self, collector=None, bouncer=None): self.deck_entries = [] self.collector = collector + self.bouncer = bouncer def add_test(self, test_file, subargs=[]): deck_entry = copy.deepcopy(self._base_entry) deck_entry['options']['collector'] = self.collector + deck_entry['options']['bouncer'] = self.bouncer deck_entry['options']['test_file'] = test_file deck_entry['options']['subargs'] = subargs self.deck_entries.append(deck_entry) @@ -89,24 +91,14 @@ def generate_deck(options): url_list_global = citizenlab_test_lists.generate_global_input( options['output'] ) - dns_servers = namebench_dns_servers.generate_country_input( - options['country-code'], - options['output'] - ) - deck = Deck(collector=options['collector']) + deck = Deck(collector=options['collector'], bouncer=options['bouncer']) deck.add_test('manipulation/http_invalid_request_line') deck.add_test('manipulation/http_header_field_manipulation') if url_list_country is not None: - deck.add_test('blocking/http_requests', ['-f', url_list_country]) - deck.add_test('blocking/http_requests', ['-f', url_list_global]) - - if url_list_country is not None: - deck.add_test('blocking/dns_consistency', - ['-f', url_list_country, '-T', dns_servers]) - deck.add_test('blocking/dns_consistency', - ['-f', url_list_global, '-T', dns_servers]) + deck.add_test('blocking/web_connectivity', ['-f', url_list_country]) + deck.add_test('blocking/web_connectivity', ['-f', url_list_global]) if config.advanced.debug: deck.pprint() diff --git a/ooni/report/cli.py b/ooni/report/cli.py index c1b4a22..e52b022 100644 --- a/ooni/report/cli.py +++ b/ooni/report/cli.py @@ -20,7 +20,7 @@ class Options(usage.Options): "Specify the configuration file to use."], ["collector", "c", None, "Specify the collector to upload the result to."], - ["bouncer", "b", "httpo://nkvphnp3p6agi5qq.onion", + ["bouncer", "b", None, "Specify the bouncer to query for a collector."] ] diff --git a/ooni/report/tool.py b/ooni/report/tool.py index fd504a6..d80a8e0 100644 --- a/ooni/report/tool.py +++ b/ooni/report/tool.py @@ -9,12 +9,15 @@ from ooni.reporter import OONIBReporter, OONIBReportLog from ooni.utils import log from ooni.report import parser from ooni.settings import config -from ooni.backend_client import BouncerClient +from ooni.backend_client import BouncerClient, CollectorClient @defer.inlineCallbacks def upload(report_file, collector=None, bouncer=None): oonib_report_log = OONIBReportLog() + collector_client = None + if collector: + collector_client = CollectorClient(address=collector) log.msg("Attempting to upload %s" % report_file) @@ -22,7 +25,7 @@ def upload(report_file, collector=None, bouncer=None): report_log = yaml.safe_load(f) report = parser.ReportLoader(report_file) - if bouncer and not collector: + if bouncer and collector_client is None: oonib_client = BouncerClient(bouncer) net_tests = [{ 'test-helpers': [], @@ -33,24 +36,33 @@ def upload(report_file, collector=None, bouncer=None): result = yield oonib_client.lookupTestCollector( net_tests ) - collector = str(result['net-tests'][0]['collector']) + collector_client = CollectorClient( + address=result['net-tests'][0]['collector'] + ) - if collector is None: + if collector_client is None: try: - collector = report_log[report_file]['collector'] - if collector is None: + collector_settings = report_log[report_file]['collector'] + if collector_settings is None: raise KeyError + elif isinstance(collector_settings, dict): + collector_client = CollectorClient(settings=collector_settings) + elif isinstance(collector_settings, str): + collector_client = CollectorClient(address=collector_settings) except KeyError: raise Exception( "No collector or bouncer specified" " and collector not in report log." ) - oonib_reporter = OONIBReporter(report.header, collector) - log.msg("Creating report for %s with %s" % (report_file, collector)) + oonib_reporter = OONIBReporter(report.header, collector_client) + log.msg("Creating report for %s with %s" % (report_file, + collector_client.settings)) report_id = yield oonib_reporter.createReport() report.header['report_id'] = report_id - yield oonib_report_log.created(report_file, collector, report_id) + yield oonib_report_log.created(report_file, + collector_client.settings, + report_id) log.msg("Writing report entries") for entry in report: yield oonib_reporter.writeReportEntry(entry) diff --git a/ooni/reporter.py b/ooni/reporter.py index 70c2f56..f76fada 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -454,7 +454,8 @@ class OONIBReportLog(object): def _closed(self, report_file): with self.edit_log() as report: - if report[report_file]['status'] != "created": + rs = report[report_file]['status'] + if rs != "created" and rs != "incomplete": raise errors.ReportNotCreated() del report[report_file] diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index a4844c8..7cbfd1d 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -42,7 +42,7 @@ class StreamListener(StreamListenerMixin): def _representHeaders(headers): represented_headers = {} for name, value in headers.getAllRawHeaders(): - represented_headers[name] = value[0] + represented_headers[name] = unicode(value[0], errors='ignore') return represented_headers def _representBody(body):

1 0

[ooni-probe/master] Add tor to the apt-get command.
by art＠torproject.org 30 May '16

30 May '16

commit 07eef0862220b7a1d7f651e2ce5511707cdc1d4a Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat May 21 14:07:49 2016 +0200 Add tor to the apt-get command. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5227cc6..1c2ec82 100644 --- a/README.rst +++ b/README.rst @@ -109,7 +109,7 @@ Make sure you have installed the following dependencies: On debian based systems this can generally be done by running:: - sudo apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip + sudo apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip tor Then you should be able to install ooniprobe by running::

1 0

[ooni-probe/master] Update the tests to be consistent with the new data formats
by art＠torproject.org 30 May '16

30 May '16

commit ff21910bcb2d2a3e54e5fc33be6b6d73ed0b60ee Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jan 28 13:09:36 2016 +0100 Update the tests to be consistent with the new data formats --- ooni/templates/httpt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index e608f8f..73bd5db 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -183,6 +183,7 @@ class HTTPTest(NetTestCase): session['failure'] = None if failure_string: session['failure'] = failure_string + self.report['requests'].append(session) def _processResponseBody(self, response_body, request, response, body_processor):

1 0

[ooni-probe/master] Maintain backward compatibility
by art＠torproject.org 30 May '16

30 May '16

commit d9656bc7187a10774bac3851b33fe4118b10c7a9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Jan 30 12:36:49 2016 +0100 Maintain backward compatibility --- ooni/reporter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooni/reporter.py b/ooni/reporter.py index 63d6f15..66a3e86 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -199,6 +199,7 @@ class YAMLReporter(OReporter): raise Exception("Failed to serialise entry") content += safe_dump(report_entry) content += '...\n' + report_entry.update(self.testDetails) self._write(content) def createReport(self): @@ -292,6 +293,7 @@ class OONIBReporter(OReporter): report_entry = entry else: raise Exception("Failed to serialise entry") + report_entry.update(self.testDetails) content += safe_dump(report_entry) content += '...\n' return content

1 0

[ooni-probe/master] Set the default DNS discovery test helper
by art＠torproject.org 30 May '16

30 May '16

commit 984276859853c192f4a5952728d9ea25deedf909 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Feb 16 13:11:41 2016 +0100 Set the default DNS discovery test helper --- ooni/nettests/blocking/web_connectivity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 6734618..d8276b3 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -32,7 +32,7 @@ class TCPConnectFactory(Factory): class UsageOptions(usage.Options): optParameters = [ ['url', 'u', None, 'Specify a single URL to test'], - ['dns-discovery', 'd', None, 'Specify the dns discovery test helper'], + ['dns-discovery', 'd', 'whoami.akamai.net', 'Specify the dns discovery test helper'], ['backend', 'b', None, 'The web_consistency backend test helper'], ] @@ -278,7 +278,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['client_resolver'] = None if results[0][0] == True: - self.report['client_resolver'] = results[0][1][1] + self.report['client_resolver'] = results[0][1][0] experiment_dns_answers = results[1][1] sockets = []

1 0

[ooni-probe/master] Fix report uploader to work with JSON
by art＠torproject.org 30 May '16

30 May '16

commit 613a02cdfe44c14fc8a1e91003a80000617d913f Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Feb 4 16:39:14 2016 +0100 Fix report uploader to work with JSON * Add better exception handling to deckgen * We should actually be using POST since PUT is indempotent * Debugging of backward and forward data format compatibility * Better integrity checks in web_consistency --- ooni/nettests/blocking/web_connectivity.py | 4 +++- ooni/reporter.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index a61819a..6734618 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -106,6 +106,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): }, 'http_request': { 'body_length': None, + 'failure': True, 'headers': {} } } @@ -238,7 +239,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): tcp_connect = None if self.report['control_failure'] is None and \ - self.report['http_experiment_failure'] is None: + self.report['http_experiment_failure'] is None and \ + self.report['control']['http_request']['failure'] is None: body_length_match = self.compare_body_lengths(experiment_http_response) if self.report['control_failure'] is None: diff --git a/ooni/reporter.py b/ooni/reporter.py index 49df443..4318fd5 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -201,7 +201,6 @@ class YAMLReporter(OReporter): raise Exception("Failed to serialise entry") content += safe_dump(report_entry) content += '...\n' - report_entry.update(self.testDetails) self._write(content) def createReport(self): @@ -295,7 +294,6 @@ class OONIBReporter(OReporter): report_entry = entry else: raise Exception("Failed to serialise entry") - report_entry.update(self.testDetails) content += safe_dump(report_entry) content += '...\n' return content

1 0

[ooni-probe/master] First stab at implementing the web_connectivity test
by art＠torproject.org 30 May '16

30 May '16

commit a710467d95627c196f78f553e11055e16d9b9e7f Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Feb 1 17:29:08 2016 +0100 First stab at implementing the web_connectivity test --- ooni/nettests/blocking/web_connectivity.py | 217 +++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py new file mode 100644 index 0000000..bcc3a28 --- /dev/null +++ b/ooni/nettests/blocking/web_connectivity.py @@ -0,0 +1,217 @@ +# -*- encoding: utf-8 -*- + +import json +from urlparse import urlparse + +from twisted.internet import reactor +from twisted.internet.protocol import Factory, Protocol +from twisted.internet.endpoints import TCP4ClientEndpoint + +from twisted.internet import defer +from twisted.python import usage + +from ooni.utils import log + +from ooni.utils.net import StringProducer, BodyReceiver +from ooni.templates import httpt, dnst, tcpt +from ooni.errors import failureToString + +class TCPConnectProtocol(Protocol): + def connectionMade(self): + self.transport.loseConnection() + +class TCPConnectFactory(Factory): + def buildProtocol(self, addr): + return TCPConnectProtocol() + + +class UsageOptions(usage.Options): + optParameters = [ + ['url', 'u', None, 'Specify a single URL to test'], + ['dns-discovery', 'd', None, 'Specify the dns discovery test helper'], + ['backend', 'b', None, 'The web_consistency backend test helper'], + ] + + +class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): + """ + Web connectivity + """ + name = "Web connectivity" + description = ("Performs a HTTP GET request over Tor and one over the " + "local network and compares the two results.") + author = "Arturo Filastò" + version = "0.1.0" + + usageOptions = UsageOptions + + inputFile = [ + 'file', 'f', None, 'List of URLS to perform GET requests to' + ] + + requiredTestHelpers = { + 'backend': 'web_connectivity', + 'dns-discovery': 'dns_discovery' + } + requiresRoot = False + requiresTor = False + + # Factor used to determine HTTP blockpage detection + factor = 0.8 + + def setUp(self): + """ + Check for inputs. + """ + if self.localOptions['url']: + self.input = self.localOptions['url'] + if not self.input: + raise Exception("No input specified") + + self.report['client_resolver'] = None + self.report['dns_consistency'] = None + self.report['body_length_match'] = None + self.report['accessible'] = None + self.report['blocking'] = None + + self.report['tcp_connect'] = [ + ] + + self.hostname = urlparse(self.input).netloc + if not self.hostname: + raise Exception("Invalid input") + + self.control = { + 'tcp_connect': {}, + 'dns_consistency': [], + 'http_requests': { + 'body_length': None, + 'headers': {} + } + } + + def dns_discovery(self): + return self.performALookup(self.localOptions['dns-discovery']) + + def dns_consistency(self): + return self.performALookup(self.hostname) + + def tcp_connect(self, socket): + ip_address, port = socket.split(":") + port = int(port) + result = { + 'ip': ip_address, + 'port': port, + 'status': { + 'success': None, + 'failure': None, + 'blocked': None + } + } + point = TCP4ClientEndpoint(reactor, ip_address, port) + d = point.connect(TCPConnectFactory()) + @d.addCallback + def cb(p): + result['status']['success'] = True + result['status']['blocked'] = False + self.report['tcp_connect'].append(result) + + @d.addErrback + def eb(failure): + result['status']['success'] = False + self.report['tcp_connect'].append(result) + return d + + @defer.inlineCallbacks + def control_request(self, sockets): + bodyProducer = StringProducer(json.dumps({ + 'http_request': self.input, + 'tcp_connect': sockets + })) + response = yield self.agent.request("POST", + str(self.localOptions['backend']), + bodyProducer=bodyProducer) + try: + content_length = int(response.headers.getRawHeaders('content-length')[0]) + except Exception: + content_length = None + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished, content_length)) + body = yield finished + self.control = json.loads(body) + + def experiment_http_get_request(self): + return self.doRequest(self.input) + + def compare_control_experiment(self, experiment_http_response, + experiment_dns_answers): + + blocking = None + + control_body_length = self.control['http_request']['body_length'] + experiment_body_length = len(experiment_http_response.body) + + if control_body_length == experiment_body_length: + rel = float(1) + elif control_body_length == 0 or experiment_body_length == 0: + rel = float(0) + else: + rel = float(control_body_length) / float(experiment_body_length) + + if rel > 1: + rel = 1/rel + + self.report['body_proportion'] = rel + if rel > float(self.factor): + self.report['body_length_match'] = True + else: + blocking = 'http' + self.report['body_length_match'] = False + + control_ips = set(self.control['dns']['ips']) + experiment_ips = set(experiment_dns_answers) + + if len(control_ips.intersection(experiment_ips)) > 0: + self.report['dns_consistency'] = 'consistent' + else: + if blocking is not None: + blocking = 'dns' + self.report['dns_consistency'] = 'inconsistent' + + for idx, result in enumerate(self.report['tcp_connect']): + socket = "%s:%s" % (result['ip'], result['port']) + control_status = self.control['tcp_connect'][socket] + log.debug(str(result)) + if result['status']['success'] == False and \ + control_status['status'] == True: + self.report['tcp_connect'][idx]['status']['blocked'] = True + blocking = 'tcp_ip' + else: + self.report['tcp_connect'][idx]['status']['blocked'] = False + + self.report['blocking'] = blocking + + @defer.inlineCallbacks + def test_web_connectivity(self): + results = yield defer.DeferredList([ + self.dns_discovery(), + self.dns_consistency() + ]) + + self.report['client_resolver'] = None + if results[0][0] == True: + self.report['client_resolver'] = results[0][1][1] + + experiment_dns_answers = results[1][1] + + sockets = map(lambda x: "%s:80" % x, results[1][1]) + + dl = [self.control_request(sockets)] + for socket in sockets: + dl.append(self.tcp_connect(socket)) + yield defer.DeferredList(dl) + + experiment_http_response = yield self.experiment_http_get_request() + self.compare_control_experiment(experiment_http_response, + experiment_dns_answers)

1 0

[ooni-probe/master] Add detection for accessibility of the website
by art＠torproject.org 30 May '16

30 May '16

commit bcd87425f035a2f6d37278007a668cd5a0349bf8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Feb 4 16:26:16 2016 +0100 Add detection for accessibility of the website * Minor refactoring of the test --- ooni/nettests/blocking/web_connectivity.py | 91 ++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 8d83ad9..a61819a 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -89,7 +89,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['blocking'] = None self.report['control_failure'] = None - self.report['experiment_failure'] = None + self.report['http_experiment_failure'] = None + self.report['dns_experiment_failure'] = None self.report['tcp_connect'] = [] self.report['control'] = {} @@ -188,6 +189,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): return False def compare_dns_experiments(self, experiment_dns_answers): + if self.control['dns']['failure'] is not None and \ + self.control['dns']['failure'] == self.report['dns_experiment_failure']: + self.report['dns_consistency'] = 'consistent' + return True + control_ips = set(self.control['dns']['ips']) experiment_ips = set(experiment_dns_answers) @@ -225,11 +231,47 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['tcp_connect'][idx]['status']['blocked'] = False return success + def determine_blocking(self, experiment_http_response, experiment_dns_answers): + blocking = None + body_length_match = None + dns_consistent = None + tcp_connect = None + + if self.report['control_failure'] is None and \ + self.report['http_experiment_failure'] is None: + body_length_match = self.compare_body_lengths(experiment_http_response) + + if self.report['control_failure'] is None: + dns_consistent = self.compare_dns_experiments(experiment_dns_answers) + + if self.report['control_failure'] is None: + tcp_connect = self.compare_tcp_experiments() + + if dns_consistent == True and tcp_connect == False: + blocking = 'tcp_ip' + + elif dns_consistent == True and \ + tcp_connect == True and body_length_match == False: + blocking = 'http' + + elif dns_consistent == False: + blocking = 'dns' + + return blocking + + @defer.inlineCallbacks def test_web_connectivity(self): + experiment_dns = self.experiment_dns_query() + + @experiment_dns.addErrback + def dns_experiment_err(failure): + self.report['dns_experiment_failure'] = failureToString(failure) + return [] + results = yield defer.DeferredList([ self.dns_discovery(), - self.experiment_dns_query() + experiment_dns ]) self.report['client_resolver'] = None @@ -254,38 +296,25 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http = self.experiment_http_get_request() @experiment_http.addErrback - def experiment_err(failure): - self.report['experiment_failure'] = failureToString(failure) + def http_experiment_err(failure): + self.report['http_experiment_failure'] = failureToString(failure) experiment_http_response = yield experiment_http - blocking = None - body_length_match = None - dns_consistent = None - tcp_connect = None - - if self.report['control_failure'] is None and \ - self.report['experiment_failure'] is None: - body_length_match = self.compare_body_lengths(experiment_http_response) - - if self.report['control_failure'] is None: - dns_consistent = self.compare_dns_experiments(experiment_dns_answers) - - if self.report['control_failure'] is None: - tcp_connect = self.compare_tcp_experiments() - - if dns_consistent == True and tcp_connect == False: - blocking = 'tcp_ip' - - elif dns_consistent == True and \ - tcp_connect == True and body_length_match == False: - blocking = 'http' - - elif dns_consistent == False: - blocking = 'dns' - + blocking = self.determine_blocking(experiment_http_response, experiment_dns_answers) self.report['blocking'] = blocking + if blocking is not None: - log.msg("Blocking detected on %s due to %s" % (self.input, blocking)) + log.msg("%s: BLOCKING DETECTED due to %s" % (self.input, blocking)) + else: + log.msg("%s: No blocking detected" % self.input) + + if all(map(lambda x: x == None, [self.report['http_experiment_failure'], + self.report['dns_experiment_failure'], + blocking])): + log.msg("") + self.report['accessible'] = True + log.msg("%s: is accessible" % self.input) else: - log.msg("No blocking detected on %s" % self.input) + log.msg("%s: is NOT accessible" % self.input) + self.report['accessible'] = False

1 0