May 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Progress on implementing web_connectivity test
by art＠torproject.org 30 May '16

30 May '16

commit 9ab068b324290df91d4296f2f3388071df3bbac4 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Feb 2 16:07:04 2016 +0100 Progress on implementing web_connectivity test --- ooni/nettests/blocking/web_connectivity.py | 62 ++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index bcc3a28..bc23f6d 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -13,7 +13,7 @@ from twisted.python import usage from ooni.utils import log from ooni.utils.net import StringProducer, BodyReceiver -from ooni.templates import httpt, dnst, tcpt +from ooni.templates import httpt, dnst from ooni.errors import failureToString class TCPConnectProtocol(Protocol): @@ -21,6 +21,7 @@ class TCPConnectProtocol(Protocol): self.transport.loseConnection() class TCPConnectFactory(Factory): + noisy = False def buildProtocol(self, addr): return TCPConnectProtocol() @@ -74,6 +75,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['accessible'] = None self.report['blocking'] = None + self.report['control_failure'] = None + self.report['experiment_failure'] = None + self.report['tcp_connect'] = [ ] @@ -83,8 +87,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.control = { 'tcp_connect': {}, - 'dns_consistency': [], - 'http_requests': { + 'dns': { + 'ips': [] + }, + 'http_request': { 'body_length': None, 'headers': {} } @@ -93,7 +99,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def dns_discovery(self): return self.performALookup(self.localOptions['dns-discovery']) - def dns_consistency(self): + def experiment_dns_query(self): return self.performALookup(self.hostname) def tcp_connect(self, socket): @@ -119,6 +125,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): @d.addErrback def eb(failure): result['status']['success'] = False + result['status']['failure'] = failureToString(failure) self.report['tcp_connect'].append(result) return d @@ -144,11 +151,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def experiment_http_get_request(self): return self.doRequest(self.input) - def compare_control_experiment(self, experiment_http_response, - experiment_dns_answers): - - blocking = None - + def compare_body_lengths(self, experiment_http_response): control_body_length = self.control['http_request']['body_length'] experiment_body_length = len(experiment_http_response.body) @@ -165,20 +168,22 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['body_proportion'] = rel if rel > float(self.factor): self.report['body_length_match'] = True + return None else: - blocking = 'http' self.report['body_length_match'] = False + return 'http' + def compare_dns_experiments(self, experiment_dns_answers): control_ips = set(self.control['dns']['ips']) experiment_ips = set(experiment_dns_answers) if len(control_ips.intersection(experiment_ips)) > 0: self.report['dns_consistency'] = 'consistent' else: - if blocking is not None: - blocking = 'dns' self.report['dns_consistency'] = 'inconsistent' + def compare_tcp_experiments(self): + blocking = False for idx, result in enumerate(self.report['tcp_connect']): socket = "%s:%s" % (result['ip'], result['port']) control_status = self.control['tcp_connect'][socket] @@ -189,14 +194,13 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): blocking = 'tcp_ip' else: self.report['tcp_connect'][idx]['status']['blocked'] = False - - self.report['blocking'] = blocking + return blocking @defer.inlineCallbacks def test_web_connectivity(self): results = yield defer.DeferredList([ self.dns_discovery(), - self.dns_consistency() + self.experiment_dns_query() ]) self.report['client_resolver'] = None @@ -207,11 +211,29 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): sockets = map(lambda x: "%s:80" % x, results[1][1]) - dl = [self.control_request(sockets)] + control_request = self.control_request(sockets) + @control_request.addErrback + def control_err(failure): + self.report['control_failure'] = failureToString(failure) + + dl = [control_request] for socket in sockets: dl.append(self.tcp_connect(socket)) - yield defer.DeferredList(dl) + results = yield defer.DeferredList(dl) + + experiment_http = self.experiment_http_get_request() + @experiment_http.addErrback + def experiment_err(failure): + self.report['experiment_failure'] = failureToString(failure) + + experiment_http_response = yield experiment_http + + if self.report['control_failure'] is None and \ + self.report['experiment_failure'] is None: + self.compare_body_lenghts(experiment_http_response) + + if self.report['control_failure'] is None: + self.compare_dns_experiments(experiment_dns_answers) - experiment_http_response = yield self.experiment_http_get_request() - self.compare_control_experiment(experiment_http_response, - experiment_dns_answers) + if self.report['control_failure'] is None: + self.compare_tcp_experiments()

1 0

[ooni-probe/master] [feature] Add setUpClass method to NetTestCase
by art＠torproject.org 30 May '16

30 May '16

commit 9367f7f0398cfd0e2105596719c47009a5e65924 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 14 22:54:25 2016 +0200 [feature] Add setUpClass method to NetTestCase --- ooni/director.py | 2 +- ooni/nettest.py | 18 +++++++++++++++++- ooni/tests/test_nettest.py | 6 ++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 64b5a98..43ca36c 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -257,7 +257,7 @@ class Director(object): net_test = NetTest(test_cases, test_details, report) net_test.director = self - yield net_test.initializeInputProcessor() + yield net_test.initialize() try: self.activeNetTests.append(net_test) self.measurementManager.schedule(net_test.generateMeasurements()) diff --git a/ooni/nettest.py b/ooni/nettest.py index d01b3e4..67d3e27 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -566,14 +566,20 @@ class NetTest(object): return measurement @defer.inlineCallbacks - def initializeInputProcessor(self): + def initialize(self): for test_class, _ in self.testCases: + # Initialize Input Processor test_class.inputs = yield defer.maybeDeferred( test_class().getInputProcessor ) if not test_class.inputs: test_class.inputs = [None] + # Run the setupClass method + yield defer.maybeDeferred( + test_class.setUpClass + ) + def generateMeasurements(self): """ This is a generator that yields measurements and registers the @@ -701,6 +707,16 @@ class NetTestCase(object): localOptions = {} + @classmethod + def setUpClass(cls): + """ + You can override this hook with logic that should be run once before + any test method in the NetTestCase is run. + This can be useful to populate class attribute that should be valid + for all the runtime of the NetTest. + """ + pass + def _setUp(self): """ This is the internal setup method to be overwritten by templates. diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 91e663f..660fd3e 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -253,12 +253,13 @@ class TestNetTest(unittest.TestCase): ntl.loadNetTestString(net_test_string_with_required_option) self.assertRaises(MissingRequiredOption, ntl.checkOptions) + @defer.inlineCallbacks def test_net_test_inputs(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() nt = NetTest(ntl.getTestCases(), ntl.getTestDetails(), None) - nt.initializeInputProcessor() + yield nt.initialize() # XXX: if you use the same test_class twice you will have consumed all # of its inputs! @@ -275,6 +276,7 @@ class TestNetTest(unittest.TestCase): ntl.checkOptions() self.assertEqual(dict(ntl.localOptions), dummyOptions) + @defer.inlineCallbacks def test_generate_measurements_size(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) @@ -282,7 +284,7 @@ class TestNetTest(unittest.TestCase): net_test = NetTest(ntl.getTestCases(), ntl.getTestDetails(), None) - net_test.initializeInputProcessor() + yield net_test.initialize() measurements = list(net_test.generateMeasurements()) self.assertEqual(len(measurements), 20)

1 0

[ooni-probe/master] Implement basic web connectivity test
by art＠torproject.org 30 May '16

30 May '16

commit d92a95b628167043a84ceabac77e45f3cd319d3d Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Feb 3 19:36:11 2016 +0100 Implement basic web connectivity test * Various hacky fixes to enforce correct report format and handling of report_id --- ooni/director.py | 5 ++ ooni/nettest.py | 6 ++- ooni/nettests/blocking/web_connectivity.py | 86 ++++++++++++++++++++++++------ ooni/reporter.py | 7 ++- ooni/tasks.py | 3 ++ ooni/templates/httpt.py | 19 +++---- 6 files changed, 94 insertions(+), 32 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 02619c2..13a4503 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -258,6 +258,11 @@ class Director(object): yield net_test.report.open() + # XXX this needs some serious refactoring + net_test_loader.reportID = report.reportID + net_test.reportID = report.reportID + net_test.testDetails['report_id'] = report.reportID + yield net_test.initializeInputProcessor() try: self.activeNetTests.append(net_test) diff --git a/ooni/nettest.py b/ooni/nettest.py index 074e8c2..166451a 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -161,6 +161,8 @@ class NetTestLoader(object): method_prefix = 'test' collector = None yamloo = True + requiresTor = False + reportID = None def __init__(self, options, test_file=None, test_string=None, annotations={}): @@ -568,10 +570,12 @@ class NetTest(object): @defer.inlineCallbacks def initializeInputProcessor(self): - for test_class, test_method in self.testCases: + for test_class, _ in self.testCases: test_class.inputs = yield defer.maybeDeferred( test_class().getInputProcessor ) + if not test_class.inputs: + test_class.inputs = [None] def generateMeasurements(self): """ diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index bc23f6d..8d83ad9 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -3,6 +3,8 @@ import json from urlparse import urlparse +from ipaddr import IPv4Address, AddressValueError + from twisted.internet import reactor from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint @@ -10,6 +12,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet import defer from twisted.python import usage +from ooni import geoip from ooni.utils import log from ooni.utils.net import StringProducer, BodyReceiver @@ -34,6 +37,16 @@ class UsageOptions(usage.Options): ] +def is_public_ipv4_address(address): + try: + ip_address = IPv4Address(address) + if not any([ip_address.is_private, + ip_address.is_loopback]): + return True + return False + except AddressValueError: + return None + class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): """ Web connectivity @@ -51,8 +64,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): ] requiredTestHelpers = { - 'backend': 'web_connectivity', - 'dns-discovery': 'dns_discovery' + 'backend': 'web-connectivity', + 'dns-discovery': 'dns-discovery' } requiresRoot = False requiresTor = False @@ -78,8 +91,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['control_failure'] = None self.report['experiment_failure'] = None - self.report['tcp_connect'] = [ - ] + self.report['tcp_connect'] = [] + self.report['control'] = {} self.hostname = urlparse(self.input).netloc if not self.hostname: @@ -147,6 +160,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): response.deliverBody(BodyReceiver(finished, content_length)) body = yield finished self.control = json.loads(body) + self.report['control'] = self.control def experiment_http_get_request(self): return self.doRequest(self.input) @@ -168,33 +182,48 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['body_proportion'] = rel if rel > float(self.factor): self.report['body_length_match'] = True - return None + return True else: self.report['body_length_match'] = False - return 'http' + return False def compare_dns_experiments(self, experiment_dns_answers): control_ips = set(self.control['dns']['ips']) experiment_ips = set(experiment_dns_answers) + for experiment_ip in experiment_ips: + if is_public_ipv4_address(experiment_ip) is False: + self.report['dns_consistency'] = 'inconsistent' + return False + if len(control_ips.intersection(experiment_ips)) > 0: self.report['dns_consistency'] = 'consistent' - else: - self.report['dns_consistency'] = 'inconsistent' + return True + + experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], + experiment_ips)) + control_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], + control_ips)) + + if len(control_asns.intersection(experiment_asns)) > 0: + self.report['dns_consistency'] = 'consistent' + return True + + self.report['dns_consistency'] = 'inconsistent' + return False def compare_tcp_experiments(self): - blocking = False + success = True for idx, result in enumerate(self.report['tcp_connect']): socket = "%s:%s" % (result['ip'], result['port']) control_status = self.control['tcp_connect'][socket] - log.debug(str(result)) if result['status']['success'] == False and \ control_status['status'] == True: self.report['tcp_connect'][idx]['status']['blocked'] = True - blocking = 'tcp_ip' + success = False else: self.report['tcp_connect'][idx]['status']['blocked'] = False - return blocking + return success @defer.inlineCallbacks def test_web_connectivity(self): @@ -208,8 +237,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['client_resolver'] = results[0][1][1] experiment_dns_answers = results[1][1] - - sockets = map(lambda x: "%s:80" % x, results[1][1]) + sockets = [] + for answer in experiment_dns_answers: + if is_public_ipv4_address(answer) is True: + sockets.append("%s:80" % answer) control_request = self.control_request(sockets) @control_request.addErrback @@ -228,12 +259,33 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http_response = yield experiment_http + blocking = None + body_length_match = None + dns_consistent = None + tcp_connect = None + if self.report['control_failure'] is None and \ self.report['experiment_failure'] is None: - self.compare_body_lenghts(experiment_http_response) + body_length_match = self.compare_body_lengths(experiment_http_response) if self.report['control_failure'] is None: - self.compare_dns_experiments(experiment_dns_answers) + dns_consistent = self.compare_dns_experiments(experiment_dns_answers) if self.report['control_failure'] is None: - self.compare_tcp_experiments() + tcp_connect = self.compare_tcp_experiments() + + if dns_consistent == True and tcp_connect == False: + blocking = 'tcp_ip' + + elif dns_consistent == True and \ + tcp_connect == True and body_length_match == False: + blocking = 'http' + + elif dns_consistent == False: + blocking = 'dns' + + self.report['blocking'] = blocking + if blocking is not None: + log.msg("Blocking detected on %s due to %s" % (self.input, blocking)) + else: + log.msg("No blocking detected on %s" % self.input) diff --git a/ooni/reporter.py b/ooni/reporter.py index 66a3e86..49df443 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -4,6 +4,8 @@ import json import os import re +from copy import deepcopy + from datetime import datetime from contextlib import contextmanager @@ -192,9 +194,9 @@ class YAMLReporter(OReporter): log.debug("Writing report with YAML reporter") content = '---\n' if isinstance(entry, Measurement): - report_entry = entry.testInstance.report + report_entry = deepcopy(entry.testInstance.report) elif isinstance(entry, dict): - report_entry = entry + report_entry = deepcopy(entry) else: raise Exception("Failed to serialise entry") content += safe_dump(report_entry) @@ -576,6 +578,7 @@ class OONIBReportLog(object): class Report(object): + reportID = None def __init__(self, test_details, report_filename, reportEntryManager, collector_address=None, diff --git a/ooni/tasks.py b/ooni/tasks.py index 72e211f..2c4f07d 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -115,6 +115,9 @@ class Measurement(TaskWithTimeout): if 'input' not in self.testInstance.report.keys(): self.testInstance.report['input'] = test_input + if 'test_start_time' not in self.testInstance.report.keys(): + start_time = otime.epochToNewTimestamp(self.testInstance._start_time) + self.testInstance.report['test_start_time'] = start_time self.testInstance.setUp() diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 73bd5db..2b280f0 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,10 +1,9 @@ import re import random -from twisted.internet import defer - from txtorcon.interface import StreamListenerMixin +from twisted.web.client import readBody, PartialDownloadError from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent @@ -13,7 +12,7 @@ from ooni.nettest import NetTestCase from ooni.utils import log, base64Dict from ooni.settings import config -from ooni.utils.net import BodyReceiver, StringProducer, userAgents +from ooni.utils.net import StringProducer, userAgents from ooni.utils.trueheaders import TrueHeaders from ooni.errors import handleAllFailures @@ -197,6 +196,8 @@ class HTTPTest(NetTestCase): return response def _processResponseBodyFail(self, failure, request, response): + if failure.check(PartialDownloadError): + return failure.value.response failure_string = handleAllFailures(failure) HTTPTest.addToReport(self, request, response, failure_string=failure_string) @@ -281,17 +282,11 @@ class HTTPTest(NetTestCase): else: self.processResponseHeaders(response_headers_dict) - try: - content_length = int(response.headers.getRawHeaders('content-length')[0]) - except Exception: - content_length = None - - finished = defer.Deferred() - response.deliverBody(BodyReceiver(finished, content_length)) - finished.addCallback(self._processResponseBody, request, - response, body_processor) + finished = readBody(response) finished.addErrback(self._processResponseBodyFail, request, response) + finished.addCallback(self._processResponseBody, request, + response, body_processor) return finished def doRequest(self, url, method="GET",

1 0

[ooni-probe/master] Clean up how we handle the reportId and remove unused functions
by art＠torproject.org 30 May '16

30 May '16

commit da51f6d069ac55ebf430ebb0adfa6a70110fda68 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Apr 12 17:09:51 2016 +0200 Clean up how we handle the reportId and remove unused functions --- ooni/deck.py | 52 ++++------------------------------- ooni/director.py | 8 +----- ooni/reporter.py | 66 +++++++++++++++++++-------------------------- ooni/tasks.py | 3 --- ooni/tests/mocks.py | 19 +++++++++++++ ooni/tests/test_deck.py | 4 +-- ooni/tests/test_reporter.py | 2 +- 7 files changed, 55 insertions(+), 99 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 4b6d894..b13385d 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -171,10 +171,10 @@ class Deck(InputFile): if self.bouncer: log.msg("Looking up collector and test helpers") - yield self.lookupCollector() + yield self.lookupCollectorAndTestHelpers() @defer.inlineCallbacks - def lookupCollector(self): + def lookupCollectorAndTestHelpers(self): self.oonibclient.address = self.bouncer required_nettests = [] @@ -221,9 +221,9 @@ class Deck(InputFile): net_test_loader.testName) collector, test_helpers = \ - find_collector_and_test_helpers(net_test_loader.testName, - net_test_loader.testVersion, - net_test_loader.inputFiles) + find_collector_and_test_helpers(test_name=net_test_loader.testName, + test_version=net_test_loader.testVersion, + input_files=net_test_loader.inputFiles) for option, name in net_test_loader.missingTestHelpers: test_helper_address = test_helpers[name].encode('utf-8') @@ -234,48 +234,6 @@ class Deck(InputFile): net_test_loader.collector = collector.encode('utf-8') @defer.inlineCallbacks - def lookupTestHelpers(self): - self.oonibclient.address = self.bouncer - - required_test_helpers = [] - requires_collector = [] - for net_test_loader in self.netTestLoaders: - if not net_test_loader.collector and not self.no_collector: - requires_collector.append(net_test_loader) - - required_test_helpers += map(lambda x: x[1], - net_test_loader.missingTestHelpers) - - if not required_test_helpers and not requires_collector: - defer.returnValue(None) - - response = yield self.oonibclient.lookupTestHelpers(required_test_helpers) - - for net_test_loader in self.netTestLoaders: - log.msg("Setting collector and test helpers for %s" % - net_test_loader.testName) - - # Only set the collector if the no collector has been specified - # from the command line or via the test deck. - if len(net_test_loader.missingTestHelpers) == 0 and \ - net_test_loader in requires_collector: - log.msg("Using the default collector: %s" % - response['default']['collector']) - net_test_loader.collector = response['default']['collector'].encode('utf-8') - continue - - for option, name in net_test_loader.missingTestHelpers: - test_helper_address = response[name]['address'].encode('utf-8') - test_helper_collector = \ - response[name]['collector'].encode('utf-8') - - log.msg("Using this helper: %s" % test_helper_address) - net_test_loader.localOptions[option] = test_helper_address - net_test_loader.testHelpers[option] = test_helper_address - if net_test_loader in requires_collector: - net_test_loader.collector = test_helper_collector - - @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): """ fetch and verify a single NetTest's inputs """ log.debug("Fetching and verifying inputs") diff --git a/ooni/director.py b/ooni/director.py index 13a4503..64b5a98 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -253,16 +253,10 @@ class Director(object): self.reportEntryManager, collector_address, no_yamloo) + yield report.open() net_test = NetTest(test_cases, test_details, report) net_test.director = self - yield net_test.report.open() - - # XXX this needs some serious refactoring - net_test_loader.reportID = report.reportID - net_test.reportID = report.reportID - net_test.testDetails['report_id'] = report.reportID - yield net_test.initializeInputProcessor() try: self.activeNetTests.append(net_test) diff --git a/ooni/reporter.py b/ooni/reporter.py index 4318fd5..aa6f5ee 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -163,7 +163,10 @@ class YAMLReporter(OReporter): if not os.path.isdir(report_destination): raise errors.InvalidDestination - report_filename = generate_filename(test_details, filename=report_filename, prefix='report', extension='yamloo') + report_filename = generate_filename(test_details, + filename=report_filename, + prefix='report', + extension='yamloo') report_path = os.path.join(self.reportDestination, report_filename) @@ -239,7 +242,7 @@ class OONIBReporter(OReporter): self.collectorAddress = collector_address self.validateCollectorAddress() - self.reportID = None + self.reportId = None self.supportedFormats = ["yaml"] if self.collectorAddress.startswith('https://'): @@ -302,7 +305,7 @@ class OONIBReporter(OReporter): def writeReportEntry(self, entry): log.debug("Writing report with OONIB reporter") - url = self.collectorAddress + '/report/' + self.reportID + url = self.collectorAddress + '/report/' + self.reportId if "json" in self.supportedFormats: serialisation_format = 'json' @@ -314,7 +317,7 @@ class OONIBReporter(OReporter): 'content': self.serializeEntry(entry, serialisation_format) } - log.debug("Updating report with id %s (%s)" % (self.reportID, url)) + log.debug("Updating report with id %s (%s)" % (self.reportId, url)) request_json = json.dumps(request) log.debug("Sending %s" % request_json) @@ -414,7 +417,7 @@ class OONIBReporter(OReporter): "different collector.") raise errors.OONIBReportCreationError - self.reportID = parsed_response['report_id'] + self.reportId = parsed_response['report_id'] self.backendVersion = parsed_response['backend_version'] self.supportedFormats = parsed_response.get('supported_formats', ["yaml"]) @@ -423,7 +426,7 @@ class OONIBReporter(OReporter): defer.returnValue(parsed_response['report_id']) def finish(self): - url = self.collectorAddress + '/report/' + self.reportID + '/close' + url = self.collectorAddress + '/report/' + self.reportId + '/close' log.debug("Closing the report %s" % url) return self.agent.request("POST", str(url)) @@ -538,6 +541,7 @@ class OONIBReportLog(object): 'collector': collector_address, 'report_id': report_id } + return report_id def created(self, report_file, collector_address, report_id): return self.run(self._created, report_file, @@ -576,7 +580,7 @@ class OONIBReportLog(object): class Report(object): - reportID = None + reportId = None def __init__(self, test_details, report_filename, reportEntryManager, collector_address=None, @@ -605,19 +609,13 @@ class Report(object): """ self.test_details = test_details self.collector_address = collector_address + self.report_filename = report_filename self.report_log = OONIBReportLog() self.yaml_reporter = None - self.report_filename = None - if not no_yamloo: - self.yaml_reporter = YAMLReporter(test_details, report_filename=report_filename) - self.report_filename = self.yaml_reporter.report_path - self.oonib_reporter = None - if collector_address: - self.oonib_reporter = OONIBReporter(test_details, - collector_address) + self.no_yamloo = no_yamloo self.done = defer.Deferred() self.reportEntryManager = reportEntryManager @@ -641,36 +639,26 @@ class Report(object): d.addCallback(created) return d + @defer.inlineCallbacks def open(self): """ This will create all the reports that need to be created and fires the created callback of the reporter whose report got created. """ - d = defer.Deferred() - deferreds = [] - - def yaml_report_failed(failure): - d.errback(failure) - - def all_reports_openned(result): - if not d.called: - d.callback(None) - - if self.oonib_reporter: - deferreds.append(self.open_oonib_reporter()) - else: - if self.yaml_reporter: - deferreds.append(self.report_log.not_created(self.report_filename)) - - if self.yaml_reporter: - yaml_report_created = \ - defer.maybeDeferred(self.yaml_reporter.createReport) - yaml_report_created.addErrback(yaml_report_failed) - - dl = defer.DeferredList(deferreds) - dl.addCallback(all_reports_openned) + if self.collector_address: + self.oonib_reporter = OONIBReporter(self.net_test_details, + self.collector_address) + self.net_test_details['report_id'] = yield self.open_oonib_reporter() + + if not self.no_yamloo: + self.yaml_reporter = YAMLReporter(self.net_test_details, + report_filename=self.report_filename) + self.report_filename = self.yaml_reporter.report_path + if not self.oonib_reporter: + yield self.report_log.not_created(self.report_filename) + yield defer.maybeDeferred(self.yaml_reporter.createReport) - return d + defer.returnValue(self.reportId) def write(self, measurement): """ diff --git a/ooni/tasks.py b/ooni/tasks.py index 2c4f07d..72e211f 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -115,9 +115,6 @@ class Measurement(TaskWithTimeout): if 'input' not in self.testInstance.report.keys(): self.testInstance.report['input'] = test_input - if 'test_start_time' not in self.testInstance.report.keys(): - start_time = otime.epochToNewTimestamp(self.testInstance._start_time) - self.testInstance.report['test_start_time'] = start_time self.testInstance.setUp() diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py index 42f548e..4f51f32 100644 --- a/ooni/tests/mocks.py +++ b/ooni/tests/mocks.py @@ -203,3 +203,22 @@ class MockOONIBClient(object): 'collector': 'httpo://thirteenchars1234.onion' } return defer.succeed(ret) + + def lookupTestCollector(self, net_tests): + ret = { + 'net-tests': [ + ] + } + for net_test in net_tests: + test_helpers ={} + for test_helper in net_test['test-helpers']: + test_helpers[test_helper] = '127.0.0.1' + + ret['net-tests'].append({ + 'name': net_test['name'], + 'version': net_test['version'], + 'input-hashes': net_test['input-hashes'], + 'collector': 'httpo://thirteenchars1234.onion', + 'test-helpers': test_helpers + }) + return defer.succeed(ret) diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index 7b423af..2c3bc04 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -148,7 +148,7 @@ class TestDeck(BaseTestCase): deck.verify() @defer.inlineCallbacks - def test_lookuptest_helpers(self): + def test_lookup_test_helpers_and_collector(self): deck = Deck(decks_directory=".") deck.bouncer = "httpo://foo.onion" deck.oonibclient = MockOONIBClient() @@ -156,7 +156,7 @@ class TestDeck(BaseTestCase): self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1) - yield deck.lookupTestHelpers() + yield deck.lookupCollectorAndTestHelpers() self.assertEqual(deck.netTestLoaders[0].collector, 'httpo://thirteenchars1234.onion') diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 9774449..b4e7592 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -92,7 +92,7 @@ class TestOONIBReporter(unittest.TestCase): def test_create_report(self): self.mock_response = oonib_new_report_message yield self.oonib_reporter.createReport() - assert self.oonib_reporter.reportID == oonib_new_report_message[ + assert self.oonib_reporter.reportId == oonib_new_report_message[ 'report_id'] @defer.inlineCallbacks

1 0

[ooni-probe/master] Many improvements to web_connectivity test
by art＠torproject.org 30 May '16

30 May '16

commit 7e1ee49e44480f62ac93a9f63aa90a64342df2ed Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 14 22:54:49 2016 +0200 Many improvements to web_connectivity test * Run the probe resolver detection inside of the setupClass * Display a summary at the end of a test run summarising the results * Make the blocking detection logic more robust --- ooni/nettests/blocking/web_connectivity.py | 152 ++++++++++++++++++++--------- 1 file changed, 108 insertions(+), 44 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index d8276b3..320f614 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -8,6 +8,7 @@ from ipaddr import IPv4Address, AddressValueError from twisted.internet import reactor from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.names import client, dns from twisted.internet import defer from twisted.python import usage @@ -19,6 +20,17 @@ from ooni.utils.net import StringProducer, BodyReceiver from ooni.templates import httpt, dnst from ooni.errors import failureToString +REQUEST_HEADERS = { + 'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, ' + 'like Gecko) Chrome/47.0.2526.106 Safari/537.36'], + 'Accept-Language': ['en-US;q=0.8,en;q=0.5'], + 'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,' + '*/*;q=0.8'] +} + +class InvalidControlResponse(Exception): + pass + class TCPConnectProtocol(Protocol): def connectionMade(self): self.transport.loseConnection() @@ -72,6 +84,21 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): # Factor used to determine HTTP blockpage detection factor = 0.8 + resolverIp = None + + @classmethod + @defer.inlineCallbacks + def setUpClass(cls): + try: + answers = yield client.lookupAddress( + cls.localOptions['dns-discovery'] + ) + assert len(answers) > 0 + assert len(answers[0]) > 0 + cls.resolverIp = answers[0][0].payload.dottedQuad() + except Exception as exc: + log.exception(exc) + log.err("Failed to lookup the resolver IP address") def setUp(self): """ @@ -82,7 +109,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): if not self.input: raise Exception("No input specified") - self.report['client_resolver'] = None + self.report['client_resolver'] = self.resolverIp self.report['dns_consistency'] = None self.report['body_length_match'] = None self.report['accessible'] = None @@ -111,9 +138,6 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): } } - def dns_discovery(self): - return self.performALookup(self.localOptions['dns-discovery']) - def experiment_dns_query(self): return self.performALookup(self.hostname) @@ -161,11 +185,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): finished = defer.Deferred() response.deliverBody(BodyReceiver(finished, content_length)) body = yield finished - self.control = json.loads(body) + try: + self.control = json.loads(body) + assert 'http_request' in self.control.keys() + assert 'tcp_connect' in self.control.keys() + assert 'dns' in self.control.keys() + except AssertionError, ValueError: + raise InvalidControlResponse(body) self.report['control'] = self.control def experiment_http_get_request(self): - return self.doRequest(self.input) + return self.doRequest(self.input, headers=REQUEST_HEADERS) def compare_body_lengths(self, experiment_http_response): control_body_length = self.control['http_request']['body_length'] @@ -233,27 +263,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): return success def determine_blocking(self, experiment_http_response, experiment_dns_answers): - blocking = None + blocking = False body_length_match = None dns_consistent = None tcp_connect = None - if self.report['control_failure'] is None and \ - self.report['http_experiment_failure'] is None and \ - self.report['control']['http_request']['failure'] is None: + if (self.report['http_experiment_failure'] is None and + self.report['control']['http_request']['failure'] is None): body_length_match = self.compare_body_lengths(experiment_http_response) - if self.report['control_failure'] is None: - dns_consistent = self.compare_dns_experiments(experiment_dns_answers) - - if self.report['control_failure'] is None: - tcp_connect = self.compare_tcp_experiments() + dns_consistent = self.compare_dns_experiments(experiment_dns_answers) + tcp_connect = self.compare_tcp_experiments() if dns_consistent == True and tcp_connect == False: blocking = 'tcp_ip' - elif dns_consistent == True and \ - tcp_connect == True and body_length_match == False: + elif (dns_consistent == True and tcp_connect == True and + body_length_match == False): blocking = 'http' elif dns_consistent == False: @@ -270,28 +296,15 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def dns_experiment_err(failure): self.report['dns_experiment_failure'] = failureToString(failure) return [] + experiment_dns_answers = yield experiment_dns - results = yield defer.DeferredList([ - self.dns_discovery(), - experiment_dns - ]) - - self.report['client_resolver'] = None - if results[0][0] == True: - self.report['client_resolver'] = results[0][1][0] - - experiment_dns_answers = results[1][1] sockets = [] for answer in experiment_dns_answers: if is_public_ipv4_address(answer) is True: sockets.append("%s:80" % answer) - control_request = self.control_request(sockets) - @control_request.addErrback - def control_err(failure): - self.report['control_failure'] = failureToString(failure) - - dl = [control_request] + # STEALTH in here we should make changes to make the test more stealth + dl = [] for socket in sockets: dl.append(self.tcp_connect(socket)) results = yield defer.DeferredList(dl) @@ -303,20 +316,71 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http_response = yield experiment_http - blocking = self.determine_blocking(experiment_http_response, experiment_dns_answers) - self.report['blocking'] = blocking + control_request = self.control_request(sockets) + @control_request.addErrback + def control_err(failure): + log.err("Failed to perform control lookup") + self.report['control_failure'] = failureToString(failure) + + yield control_request - if blocking is not None: - log.msg("%s: BLOCKING DETECTED due to %s" % (self.input, blocking)) + if self.report['control_failure'] is None: + self.report['blocking'] = self.determine_blocking(experiment_http_response, experiment_dns_answers) + + log.msg("") + log.msg("Result for %s" % self.input) + log.msg("-----------" + "-"*len(self.input)) + + if self.report['blocking'] is None: + log.msg("* Could not determine status of blocking due to " + "failing control request") + elif self.report['blocking'] is False: + log.msg("* No blocking detected") else: - log.msg("%s: No blocking detected" % self.input) + log.msg("* BLOCKING DETECTED due to %s" % (self.report['blocking'])) - if all(map(lambda x: x == None, [self.report['http_experiment_failure'], - self.report['dns_experiment_failure'], - blocking])): - log.msg("") + if (self.report['http_experiment_failure'] == None and + self.report['dns_experiment_failure'] == None and + self.report['blocking'] in (False, None)): self.report['accessible'] = True - log.msg("%s: is accessible" % self.input) + log.msg("* Is accessible") else: - log.msg("%s: is NOT accessible" % self.input) + log.msg("* Is NOT accessible") self.report['accessible'] = False + + def postProcessor(self, measurements): + self.summary['accessible'] = self.summary.get('accessible', []) + self.summary['not-accessible'] = self.summary.get('not-accessible', []) + self.summary['blocked'] = self.summary.get('blocked', []) + + if self.report['blocking'] not in (False, None): + self.summary['blocked'].append((self.input, + self.report['blocking'])) + if self.report['accessible'] is True: + self.summary['accessible'].append(self.input) + else: + self.summary['not-accessible'].append(self.input) + return self.report + + def displaySummary(self, summary): + + if len(summary['accessible']) > 0: + log.msg("") + log.msg("Accessible URLS") + log.msg("---------------") + for url in summary['accessible']: + log.msg("* {}".format(url)) + + if len(summary['not-accessible']) > 0: + log.msg("") + log.msg("Not accessible URLS") + log.msg("---------------") + for url in summary['not-accessible']: + log.msg("* {}".format(url)) + + if len(summary['blocked']) > 0: + log.msg("") + log.msg("Blocked URLS") + log.msg("------------") + for url, reason in summary['blocked']: + log.msg("* {} due to {}".format(url, reason))

1 0

[ooni-probe/master] Add check on equality of control and experiment DNS responses
by art＠torproject.org 30 May '16

30 May '16

commit 6a4ed0161a961daaaccfd8b082a7328683f3b2fc Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Apr 15 00:05:07 2016 +0200 Add check on equality of control and experiment DNS responses --- ooni/nettests/blocking/web_connectivity.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 320f614..175c6d3 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -228,6 +228,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): control_ips = set(self.control['dns']['ips']) experiment_ips = set(experiment_dns_answers) + if control_ips == experiment_ips: + self.report['dns_consistency'] = 'consistent' + return True + for experiment_ip in experiment_ips: if is_public_ipv4_address(experiment_ip) is False: self.report['dns_consistency'] = 'inconsistent'

1 0

[ooni-probe/master] [refactoring] Change signature of Deck to be more robust
by art＠torproject.org 30 May '16

30 May '16

commit aa8314973df0f6e458aa1e0de786faaad8264dc3 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Apr 15 15:45:03 2016 +0200 [refactoring] Change signature of Deck to be more robust Do more sanity checks on bouncer address at instantiation time --- ooni/deck.py | 30 ++++++++++++++---------------- ooni/oonibclient.py | 15 +++++++++++++-- ooni/oonicli.py | 4 ++-- ooni/tests/mocks.py | 3 +++ ooni/tests/test_deck.py | 14 +++++++------- ooni/tests/test_templates.py | 3 +++ 6 files changed, 42 insertions(+), 27 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index b13385d..4b2c502 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -95,28 +95,27 @@ def nettest_to_path(path, allow_arbitrary_paths=False): class Deck(InputFile): + # this exists so we can mock it out in unittests + _OONIBClient = OONIBClient + def __init__(self, deck_hash=None, - deckFile=None, + bouncer=None, decks_directory=config.decks_directory, no_collector=False): self.id = deck_hash - self.requiresTor = False self.no_collector = no_collector - self.bouncer = '' + self.bouncer = bouncer + + self.requiresTor = False + self.netTestLoaders = [] self.inputs = [] - self.oonibclient = OONIBClient(self.bouncer) - self.decksDirectory = os.path.abspath(decks_directory) - self.deckHash = deck_hash - - if deckFile: - self.loadDeck(deckFile) @property def cached_file(self): - return os.path.join(self.decksDirectory, self.deckHash) + return os.path.join(self.decksDirectory, self.id) @property def cached_descriptor(self): @@ -124,7 +123,7 @@ class Deck(InputFile): def loadDeck(self, deckFile): with open(deckFile) as f: - self.deckHash = sha256(f.read()).hexdigest() + self.id = sha256(f.read()).hexdigest() f.seek(0) test_deck = yaml.safe_load(f) @@ -175,7 +174,7 @@ class Deck(InputFile): @defer.inlineCallbacks def lookupCollectorAndTestHelpers(self): - self.oonibclient.address = self.bouncer + oonibclient = self._OONIBClient(self.bouncer) required_nettests = [] @@ -201,8 +200,7 @@ class Deck(InputFile): if not requires_test_helpers and not requires_collector: defer.returnValue(None) - log.debug("Looking up {}".format(required_nettests)) - response = yield self.oonibclient.lookupTestCollector(required_nettests) + response = yield oonibclient.lookupTestCollector(required_nettests) provided_net_tests = response['net-tests'] def find_collector_and_test_helpers(test_name, test_version, input_files): @@ -240,10 +238,10 @@ class Deck(InputFile): for i in net_test_loader.inputFiles: if i['url']: log.debug("Downloading %s" % i['url']) - self.oonibclient.address = i['address'] + oonibclient = self._OONIBClient(i['address']) try: - input_file = yield self.oonibclient.downloadInput(i['hash']) + input_file = yield oonibclient.downloadInput(i['hash']) except: raise e.UnableToLoadDeckInput diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 388bb68..336ae4e 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -1,5 +1,7 @@ import json +from urlparse import urljoin + from twisted.web.client import Agent from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint @@ -15,6 +17,15 @@ class OONIBClient(object): retries = 3 def __init__(self, address): + if address.startswith("https://"): + log.err("HTTPS bouncers are currently not supported!") + raise e.InvalidOONIBBouncerAddress + elif address.startswith("http://"): + log.msg("Warning using plaintext bouncer!") + elif address.startswith("httpo://"): + log.debug("Using Tor hidden service bouncer: {}".format(address)) + else: + raise e.InvalidOONIBBouncerAddress self.address = address def _request(self, method, urn, genReceiver, bodyProducer=None): @@ -30,7 +41,7 @@ class OONIBClient(object): raise e.InvalidOONIBBouncerAddress elif self.address.startswith('http://'): - log.msg("Warning using unencrypted backend") + log.msg("Warning using unencrypted bouncer") agent = Agent(reactor) attempts = 0 @@ -38,7 +49,7 @@ class OONIBClient(object): finished = defer.Deferred() def perform_request(attempts): - uri = address + urn + uri = urljoin(address, urn) d = agent.request(method, uri, bodyProducer=bodyProducer) @d.addCallback diff --git a/ooni/oonicli.py b/ooni/oonicli.py index e2d78e1..281ff46 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -245,8 +245,8 @@ def createDeck(global_options, url=None): if global_options['no-yamloo']: log.msg("Will not write to a yamloo report file") - deck = Deck(no_collector=global_options['no-collector']) - deck.bouncer = global_options['bouncer'] + deck = Deck(bouncer=global_options['bouncer'], + no_collector=global_options['no-collector']) try: if global_options['testdeck']: diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py index 4f51f32..19b4692 100644 --- a/ooni/tests/mocks.py +++ b/ooni/tests/mocks.py @@ -190,6 +190,9 @@ class MockTaskManager(TaskManager): class MockOONIBClient(object): + def __init__(self, *args, **kw): + pass + def lookupTestHelpers(self, required_test_helpers): ret = { 'default': { diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index 2c3bc04..3e2a322 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -127,14 +127,14 @@ class TestDeck(BaseTestCase): os.remove(self.filename) def test_open_deck(self): - deck = Deck(decks_directory=".") - deck.bouncer = "httpo://foo.onion" + deck = Deck(bouncer="httpo://foo.onion", + decks_directory=".") deck.loadDeck(self.deck_file) assert len(deck.netTestLoaders) == 1 def test_save_deck_descriptor(self): - deck = Deck(decks_directory=".") - deck.bouncer = "httpo://foo.onion" + deck = Deck(bouncer="httpo://foo.onion", + decks_directory=".") deck.loadDeck(self.deck_file) deck.load({'name': 'spam', 'id': 'spam', @@ -149,9 +149,9 @@ class TestDeck(BaseTestCase): @defer.inlineCallbacks def test_lookup_test_helpers_and_collector(self): - deck = Deck(decks_directory=".") - deck.bouncer = "httpo://foo.onion" - deck.oonibclient = MockOONIBClient() + deck = Deck(bouncer="httpo://foo.onion", + decks_directory=".") + deck._OONIBClient = MockOONIBClient deck.loadDeck(self.deck_file) self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1) diff --git a/ooni/tests/test_templates.py b/ooni/tests/test_templates.py index e7452df..5b2c77a 100644 --- a/ooni/tests/test_templates.py +++ b/ooni/tests/test_templates.py @@ -1,6 +1,7 @@ from ooni.templates import httpt, dnst from ooni.tests import is_internet_connected + from twisted.names import dns from twisted.internet.error import DNSLookupError from twisted.internet import reactor, defer, base @@ -92,6 +93,8 @@ class TestDNST(unittest.TestCase): @defer.inlineCallbacks def test_perform_a_lookup(self): + if not is_internet_connected(): + self.skipTest("You must be connected to the internet to run this test") dns_test = dnst.DNSTest() dns_test._setUp() result = yield dns_test.performALookup('example.com', dns_server=('8.8.8.8', 53))

1 0

[ooni-probe/master] Make the blocking detection heuristics more strict
by art＠torproject.org 30 May '16

30 May '16

commit ff77593d59dc7e25e4214e15571d616d7e374dce Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Apr 18 18:45:09 2016 +0200 Make the blocking detection heuristics more strict --- ooni/nettests/blocking/web_connectivity.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 4b9b6c2..28ee942 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -84,7 +84,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): followRedirects = True # Factor used to determine HTTP blockpage detection - factor = 0.8 + # the factor 0.7 comes from http://www3.cs.stonybrook.edu/~phillipa/papers/JLFG14.pdf + factor = 0.7 resolverIp = None @classmethod @@ -296,14 +297,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = 'inconsistent' tcp_connect = self.compare_tcp_experiments() - if dns_consistent == True and tcp_connect == False: + if (dns_consistent == True and tcp_connect == False and + self.report['http_experiment_failure'] is not None): blocking = 'tcp_ip' + # XXX we may want to have different codes for these two types of + # blocking elif (dns_consistent == True and tcp_connect == True and self.report['body_length_match'] == False): blocking = 'http' + elif (dns_consistent == True and tcp_connect == True and + self.report['http_experiment_failure'] is not None and + self.report['control']['http_request']['failure'] != self.report['http_experiment_failure']): + blocking = 'http' - elif dns_consistent == False: + elif (dns_consistent == False and + (self.report['http_experiment_failure'] is not None or + self.report['body_length_match'] == False)): blocking = 'dns' return blocking

1 0

[ooni-probe/master] Add support for computing header difference and follow redirects
by art＠torproject.org 30 May '16

30 May '16

commit 8c5fdd48ebb760b72b1b02a83fc96a19f92293c7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Apr 16 14:30:29 2016 +0200 Add support for computing header difference and follow redirects --- ooni/nettests/blocking/web_connectivity.py | 54 +++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 175c6d3..4b9b6c2 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -81,6 +81,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): } requiresRoot = False requiresTor = False + followRedirects = True # Factor used to determine HTTP blockpage detection factor = 0.8 @@ -112,6 +113,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['client_resolver'] = self.resolverIp self.report['dns_consistency'] = None self.report['body_length_match'] = None + self.report['headers_match'] = None + self.report['accessible'] = None self.report['blocking'] = None @@ -139,6 +142,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): } def experiment_dns_query(self): + log.msg("Doing DNS query for {}".format(self.hostname)) return self.performALookup(self.hostname) def tcp_connect(self, socket): @@ -197,6 +201,22 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def experiment_http_get_request(self): return self.doRequest(self.input, headers=REQUEST_HEADERS) + def compare_headers(self, experiment_http_response): + count = 0 + control_headers_lower = {k.lower(): v for k, v in + self.report['control']['http_request']['headers'].items()} + + for header_name, header_value in \ + experiment_http_response.headers.getAllRawHeaders(): + try: + control_headers_lower[header_name.lower()] + except KeyError: + log.msg("Did not find the key {}".format(header_name)) + return False + count += 1 + + return count == len(self.report['control']['http_request']['headers']) + def compare_body_lengths(self, experiment_http_response): control_body_length = self.control['http_request']['body_length'] experiment_body_length = len(experiment_http_response.body) @@ -213,10 +233,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['body_proportion'] = rel if rel > float(self.factor): - self.report['body_length_match'] = True return True else: - self.report['body_length_match'] = False return False def compare_dns_experiments(self, experiment_dns_answers): @@ -229,16 +247,13 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_ips = set(experiment_dns_answers) if control_ips == experiment_ips: - self.report['dns_consistency'] = 'consistent' return True for experiment_ip in experiment_ips: if is_public_ipv4_address(experiment_ip) is False: - self.report['dns_consistency'] = 'inconsistent' return False if len(control_ips.intersection(experiment_ips)) > 0: - self.report['dns_consistency'] = 'consistent' return True experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], @@ -247,10 +262,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): control_ips)) if len(control_asns.intersection(experiment_asns)) > 0: - self.report['dns_consistency'] = 'consistent' return True - self.report['dns_consistency'] = 'inconsistent' return False def compare_tcp_experiments(self): @@ -268,22 +281,26 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def determine_blocking(self, experiment_http_response, experiment_dns_answers): blocking = False - body_length_match = None - dns_consistent = None - tcp_connect = None if (self.report['http_experiment_failure'] is None and self.report['control']['http_request']['failure'] is None): - body_length_match = self.compare_body_lengths(experiment_http_response) + self.report['body_length_match'] = self.compare_body_lengths( + experiment_http_response) + self.report['headers_match'] = self.compare_headers( + experiment_http_response) dns_consistent = self.compare_dns_experiments(experiment_dns_answers) + if dns_consistent is True: + self.report['dns_consistency'] = 'consistent' + else: + self.report['dns_consistency'] = 'inconsistent' tcp_connect = self.compare_tcp_experiments() if dns_consistent == True and tcp_connect == False: blocking = 'tcp_ip' elif (dns_consistent == True and tcp_connect == True and - body_length_match == False): + self.report['body_length_match'] == False): blocking = 'http' elif dns_consistent == False: @@ -302,10 +319,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): return [] experiment_dns_answers = yield experiment_dns + port = 80 + parsed_url = urlparse(self.input) + if parsed_url.port: + port = parsed_url.port + elif parsed_url.scheme == 'https': + port = 443 + sockets = [] - for answer in experiment_dns_answers: - if is_public_ipv4_address(answer) is True: - sockets.append("%s:80" % answer) + for ip_address in experiment_dns_answers: + if is_public_ipv4_address(ip_address) is True: + sockets.append("{}:{}".format(ip_address, port)) # STEALTH in here we should make changes to make the test more stealth dl = []

1 0

[translation/tails-openpgp-applet] Update translations for tails-openpgp-applet
by translation＠torproject.org 30 May '16

30 May '16

commit c092e33521ca7ff1dab3ea7a35b89d212d87694b Author: Translation commit bot <translation(a)torproject.org> Date: Mon May 30 14:17:05 2016 +0000 Update translations for tails-openpgp-applet --- et/openpgp-applet.pot | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/et/openpgp-applet.pot b/et/openpgp-applet.pot index b3178a6..32e50ca 100644 --- a/et/openpgp-applet.pot +++ b/et/openpgp-applet.pot @@ -3,13 +3,14 @@ # This file is distributed under the same license as the PACKAGE package. # # Translators: +# Kristjan Räts <kristjanrats(a)gmail.com>, 2016 msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: tails(a)boum.org\n" "POT-Creation-Date: 2015-08-10 15:55+0200\n" -"PO-Revision-Date: 2015-11-23 02:23+0000\n" -"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"PO-Revision-Date: 2016-05-30 13:49+0000\n" +"Last-Translator: Kristjan Räts <kristjanrats(a)gmail.com>\n" "Language-Team: Estonian (http://www.transifex.com/otf/torproject/language/et/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -43,7 +44,7 @@ msgstr "" #: bin/openpgp-applet:243 msgid "_Decrypt/Verify Clipboard" -msgstr "" +msgstr "_Dekrüpteeri/Kontrolli lõikepuhvrit" #: bin/openpgp-applet:247 msgid "_Manage Keys"

1 0