May 2016 - tor-commits - lists.torproject.org

[translation/tor-messenger-fingerdtd_completed] Update translations for tor-messenger-fingerdtd_completed
by translation＠torproject.org 30 May '16

30 May '16

commit 53a12b4dccc1d3445c09978bfc493162e80311ec Author: Translation commit bot <translation(a)torproject.org> Date: Mon May 30 19:46:25 2016 +0000 Update translations for tor-messenger-fingerdtd_completed --- pl/finger.dtd | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pl/finger.dtd b/pl/finger.dtd index fa58403..5943e49 100644 --- a/pl/finger.dtd +++ b/pl/finger.dtd @@ -7,4 +7,9 @@ <!ENTITY finger.protocol "Protokuł"> <!ENTITY finger.verify "Weryfikacja"> <!ENTITY finger.remove "Usuń"> -<!ENTITY finger.intro "Zarządzaj listą kluczy które widziałeś."> \ No newline at end of file +<!ENTITY finger.intro "Zarządzaj listą kluczy które widziałeś."> +<!ENTITY addFingerDialog.title "Dodaj odcisk palca"> +<!ENTITY addFingerDialog.finger "Odcisk palca"> +<!ENTITY addFingerDialog.accept "OK"> +<!ENTITY addFingerDialog.cancel "Pomiń"> +<!ENTITY addFingerDialog.tooltip "Jeśli znasz 40 znaków fingerprinta klucza prywatnego OTR Twojego kontaktu to wpisz go teraz."> \ No newline at end of file

1 0

[translation/tor-messenger-fingerdtd] Update translations for tor-messenger-fingerdtd
by translation＠torproject.org 30 May '16

30 May '16

commit 88390a939a5b56de115f88daa66ee6f690722d87 Author: Translation commit bot <translation(a)torproject.org> Date: Mon May 30 19:46:22 2016 +0000 Update translations for tor-messenger-fingerdtd --- pl/finger.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl/finger.dtd b/pl/finger.dtd index bd371ca..5943e49 100644 --- a/pl/finger.dtd +++ b/pl/finger.dtd @@ -12,4 +12,4 @@ <!ENTITY addFingerDialog.finger "Odcisk palca"> <!ENTITY addFingerDialog.accept "OK"> <!ENTITY addFingerDialog.cancel "Pomiń"> -<!ENTITY addFingerDialog.tooltip "If you know the 40 hex char fingerprint of your contact's OTR private key, enter it now."> \ No newline at end of file +<!ENTITY addFingerDialog.tooltip "Jeśli znasz 40 znaków fingerprinta klucza prywatnego OTR Twojego kontaktu to wpisz go teraz."> \ No newline at end of file

1 0

[tor/master] remove now-irrelevant XXX020 comments in configure.ac
by nickm＠torproject.org 30 May '16

30 May '16

commit 57bf8bb26357444219c07d57caa9260a75ff7681 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon May 30 15:31:19 2016 -0400 remove now-irrelevant XXX020 comments in configure.ac They apply to ancient GCC versions and to an unknown set of configuration options. Notabug. --- configure.ac | 3 --- 1 file changed, 3 deletions(-) diff --git a/configure.ac b/configure.ac index 26f85b5..7259bf7 100644 --- a/configure.ac +++ b/configure.ac @@ -30,7 +30,6 @@ fi # the += operator on it in src/or/Makefile.am CPPFLAGS="$CPPFLAGS -I\${top_srcdir}/src/common" -#XXXX020 We should make these enabled or not, before 0.2.0.x-final AC_ARG_ENABLE(openbsd-malloc, AS_HELP_STRING(--enable-openbsd-malloc, [use malloc code from OpenBSD. Linux only])) AC_ARG_ENABLE(static-openssl, @@ -1710,7 +1709,6 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then if test "x$have_gcc42" = "xyes"; then # These warnings break gcc 4.0.2 and work on gcc 4.2 - # XXXX020 See if any of these work with earlier versions. CFLAGS="$CFLAGS -Waddress -Wmissing-noreturn -Wstrict-overflow=1" # We used to use -Wstrict-overflow=5, but that breaks us heavily under 4.3. @@ -1723,7 +1721,6 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then if test "x$have_gcc43" = "xyes"; then # These warnings break gcc 4.2 and work on gcc 4.3 - # XXXX020 See if any of these work with earlier versions. CFLAGS="$CFLAGS -Wextra -Warray-bounds" fi

1 0

[webwml/master] Add new Tor Browser version: 6.0
by boklm＠torproject.org 30 May '16

30 May '16

commit 9f11ba3c13dd68089b942cb2dbd3b2804e8a3e3c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon May 30 20:56:17 2016 +0200 Add new Tor Browser version: 6.0 --- include/versions.wmi | 22 +++++++++++----------- projects/torbrowser/RecommendedTBBVersions | 4 ++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/include/versions.wmi b/include/versions.wmi index 750b575..e7c4b2c 100644 --- a/include/versions.wmi +++ b/include/versions.wmi @@ -4,27 +4,27 @@ <define-tag version-win32-stable whitespace=delete>0.2.7.6</define-tag> <define-tag version-torbrowserdevelopbranch whitespace=delete>maint-5.5</define-tag> -<define-tag version-torbrowserbundledir whitespace=delete>5.5.5</define-tag> +<define-tag version-torbrowserbundledir whitespace=delete>6.0</define-tag> <define-tag version-torbrowserbundlebetadir whitespace=delete>6.0a5</define-tag> -<define-tag version-torbrowserbundle whitespace=delete>5.5.5</define-tag> -<define-tag releasedate-torbrowserbundle whitespace=delete>2016-04-26</define-tag> +<define-tag version-torbrowserbundle whitespace=delete>6.0</define-tag> +<define-tag releasedate-torbrowserbundle whitespace=delete>2016-05-30</define-tag> <define-tag version-torbrowserbundlebeta whitespace=delete>6.0a5</define-tag> <define-tag releasedate-torbrowserbundlebeta whitespace=delete>2016-04-28</define-tag> -<define-tag version-torbrowserbundlelinux32 whitespace=delete>5.5.5</define-tag> -<define-tag releasedate-torbrowserbundlelinux32 whitespace=delete>2016-04-26</define-tag> -<define-tag version-torbrowserbundlelinux64 whitespace=delete>5.5.5</define-tag> -<define-tag releasedate-torbrowserbundlelinux64 whitespace=delete>2016-04-26</define-tag> +<define-tag version-torbrowserbundlelinux32 whitespace=delete>6.0</define-tag> +<define-tag releasedate-torbrowserbundlelinux32 whitespace=delete>2016-05-30</define-tag> +<define-tag version-torbrowserbundlelinux64 whitespace=delete>6.0</define-tag> +<define-tag releasedate-torbrowserbundlelinux64 whitespace=delete>2016-05-30</define-tag> <define-tag version-torbrowserbundlelinux32beta whitespace=delete>6.0a5</define-tag> <define-tag releasedate-torbrowserbundlelinux32beta whitespace=delete>2016-04-28</define-tag> <define-tag version-torbrowserbundlelinux64beta whitespace=delete>6.0a5</define-tag> <define-tag releasedate-torbrowserbundlelinux64beta whitespace=delete>2016-04-28</define-tag> -<define-tag version-torbrowserbundleosx32 whitespace=delete>5.5.5</define-tag> -<define-tag releasedate-torbrowserbundleosx32 whitespace=delete>2016-04-26</define-tag> -<define-tag version-torbrowserbundleosx64 whitespace=delete>5.5.5</define-tag> -<define-tag releasedate-torbrowserbundleosx64 whitespace=delete>2016-04-26</define-tag> +<define-tag version-torbrowserbundleosx32 whitespace=delete>6.0</define-tag> +<define-tag releasedate-torbrowserbundleosx32 whitespace=delete>2016-05-30</define-tag> +<define-tag version-torbrowserbundleosx64 whitespace=delete>6.0</define-tag> +<define-tag releasedate-torbrowserbundleosx64 whitespace=delete>2016-05-30</define-tag> <define-tag version-torbrowserbundleosx64beta whitespace=delete>6.0a5</define-tag> <define-tag releasedate-torbrowserbundleosx64beta whitespace=delete>2016-04-28</define-tag> diff --git a/projects/torbrowser/RecommendedTBBVersions b/projects/torbrowser/RecommendedTBBVersions index 61ac009..7b9dc5e 100644 --- a/projects/torbrowser/RecommendedTBBVersions +++ b/projects/torbrowser/RecommendedTBBVersions @@ -3,6 +3,10 @@ "5.5.5-Linux", "5.5.5-MacOS", "5.5.5-Windows", +"6.0", +"6.0-Linux", +"6.0-MacOS", +"6.0-Windows", "6.0a5", "6.0a5-Linux", "6.0a5-MacOS",

1 0

[ooni-probe/master] Ensure the input key is properly set
by art＠torproject.org 30 May '16

30 May '16

commit c8df2379870e9210c4fe20541b5f6ca4f65e1cbb Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 23 13:12:36 2016 +0200 Ensure the input key is properly set * Improve readability of the blockpage detection check --- ooni/nettests/blocking/web_connectivity.py | 8 +++++--- ooni/tasks.py | 5 ++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index f68b5aa..4f5f4e4 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -355,9 +355,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = 'inconsistent' tcp_connect = self.compare_tcp_experiments() - got_expected_web_page = (self.report['body_length_match'] or - self.report['headers_match']) and \ - self.report['status_code_match'] + got_expected_web_page = ( + (self.report['body_length_match'] is True or + self.report['headers_match'] is True) + and self.report['status_code_match'] is True + ) if (dns_consistent == True and tcp_connect == False and experiment_http_failure is not None): diff --git a/ooni/tasks.py b/ooni/tasks.py index 72e211f..925089b 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -113,10 +113,9 @@ class Measurement(TaskWithTimeout): if not hasattr(self.testInstance, '_start_time'): self.testInstance._start_time = time.time() - if 'input' not in self.testInstance.report.keys(): - self.testInstance.report['input'] = test_input - self.testInstance.setUp() + if 'input' not in self.testInstance.report.keys(): + self.testInstance.report['input'] = self.testInstance.input self.netTestMethod = getattr(self.testInstance, test_method)

1 0

[ooni-probe/master] Consider valid onion addresses also when they don't use the httpo:// schema
by art＠torproject.org 30 May '16

30 May '16

commit c32bd88b5b99ea7d3fc62c9088d3ac17770ba7df Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue May 24 11:18:10 2016 +0200 Consider valid onion addresses also when they don't use the httpo:// schema Based on feedback from @willscott * Fix invalid onion address in unittests --- ooni/backend_client.py | 16 ++++++++++------ ooni/deck.py | 4 ++-- ooni/tests/mocks.py | 6 +++--- ooni/tests/test_deck.py | 2 +- ooni/tests/test_onion.py | 16 ++++++++++++++++ ooni/utils/onion.py | 8 ++++++++ 6 files changed, 40 insertions(+), 12 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py index 3b551d4..de01ceb 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -14,7 +14,7 @@ _twisted_14_0_2_version = Version('twisted', 14, 0, 2) from ooni import errors as e from ooni.settings import config -from ooni.utils import log +from ooni.utils import log, onion from ooni.utils.net import BodyReceiver, StringProducer, Downloader from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent @@ -42,10 +42,10 @@ class OONIBClient(object): def _guessBackendType(self): if self.base_address is None: raise e.InvalidAddress - if self.base_address.startswith('https://'): - self.backend_type = 'https' - elif self.base_address.startswith('httpo://'): + if onion.is_onion_address(self.base_address): self.backend_type = 'onion' + elif self.base_address.startswith('https://'): + self.backend_type = 'https' elif self.base_address.startswith('http://'): self.backend_type = 'http' else: @@ -54,10 +54,14 @@ class OONIBClient(object): def _setupBaseAddress(self): parsed_address = urlparse(self.base_address) if self.backend_type == 'onion': - if not parsed_address.netloc.endswith(".onion"): + if not onion.is_onion_address(self.base_address): log.err("Invalid onion address.") raise e.InvalidAddress(self.base_address) - self.base_address = ("http://%s" % parsed_address.netloc) + if parsed_address.scheme in ('http', 'httpo'): + self.base_address = ("http://%s" % parsed_address.netloc) + else: + self.base_address = ("%s://%s" % (parsed_address.scheme, + parsed_address.netloc)) elif self.backend_type == 'http': self.base_address = ("http://%s" % parsed_address.netloc) elif self.backend_type in ('https', 'cloudfront'): diff --git a/ooni/deck.py b/ooni/deck.py index 7a3b697..383845c 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -4,7 +4,7 @@ from ooni.backend_client import CollectorClient, BouncerClient from ooni.backend_client import WebConnectivityClient from ooni.nettest import NetTestLoader from ooni.settings import config -from ooni.utils import log +from ooni.utils import log, onion from ooni import errors as e from twisted.python.filepath import FilePath @@ -175,7 +175,7 @@ class Deck(InputFile): https_addresses = [] plaintext_addresses = [] - if priority_address.startswith('httpo://'): + if onion.is_onion_address(priority_address): priority_address = { 'address': priority_address, 'type': 'onion' diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py index f3f852f..db7b154 100644 --- a/ooni/tests/mocks.py +++ b/ooni/tests/mocks.py @@ -197,13 +197,13 @@ class MockBouncerClient(object): ret = { 'default': { 'address': '127.0.0.1', - 'collector': 'httpo://thirteenchars1234.onion' + 'collector': 'httpo://thirteenchars123.onion' } } for required_test_helper in required_test_helpers: ret[required_test_helper] = { 'address': '127.0.0.1', - 'collector': 'httpo://thirteenchars1234.onion' + 'collector': 'httpo://thirteenchars123.onion' } return defer.succeed(ret) @@ -221,7 +221,7 @@ class MockBouncerClient(object): 'name': net_test['name'], 'version': net_test['version'], 'input-hashes': net_test['input-hashes'], - 'collector': 'httpo://thirteenchars1234.onion', + 'collector': 'httpo://thirteenchars123.onion', 'test-helpers': test_helpers }) return defer.succeed(ret) diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index 8d415f0..fa9b0e8 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -160,7 +160,7 @@ class TestDeck(BaseTestCase): yield deck.lookupCollectorAndTestHelpers() self.assertEqual(deck.netTestLoaders[0].collector.settings['address'], - 'http://thirteenchars1234.onion') + 'http://thirteenchars123.onion') self.assertEqual(deck.netTestLoaders[0].localOptions['backend'], '127.0.0.1') diff --git a/ooni/tests/test_onion.py b/ooni/tests/test_onion.py index 30fd26d..944f8f5 100644 --- a/ooni/tests/test_onion.py +++ b/ooni/tests/test_onion.py @@ -48,3 +48,19 @@ class TestOnion(unittest.TestCase): {'version': onion.TorVersion('0.2.3.20')}): self.assertRaises(onion.OutdatedTor, onion.bridge_line, 'fte', '/log.txt') + + def test_is_onion_address(self): + self.assertEqual(onion.is_onion_address( + 'httpo://thirteenchars123.onion'), True) + + self.assertEqual(onion.is_onion_address( + 'thirteenchars123.onion'), True) + + self.assertEqual(onion.is_onion_address( + 'http://thirteenchars123.onion'), True) + + self.assertEqual(onion.is_onion_address( + 'https://thirteenchars123.onion'), True) + + self.assertEqual(onion.is_onion_address( + 'http://thirteenchars123.com'), False) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index ec0a8b4..fea75c2 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -1,3 +1,4 @@ +import re import string import subprocess from distutils.spawn import find_executable @@ -7,6 +8,8 @@ from txtorcon.util import find_tor_binary as tx_find_tor_binary from ooni.settings import config +ONION_ADDRESS_REGEXP = re.compile("^((httpo|http|https)://)?" + "[a-z0-9]{16}\.onion") class TorVersion(LooseVersion): pass @@ -65,6 +68,11 @@ def transport_name(address): return None +def is_onion_address(address): + return ONION_ADDRESS_REGEXP.match(address) != None + + + tor_details = { 'binary': find_tor_binary(), 'version': tor_version()

1 0

[ooni-probe/master] Compare CNAME responses as well
by art＠torproject.org 30 May '16

30 May '16

commit d7a7d0905862b7d75aaee20f32f351f1a66bb926 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue May 24 12:04:36 2016 +0200 Compare CNAME responses as well --- ooni/nettests/blocking/web_connectivity.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 3560cb0..e640a5f 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -181,7 +181,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.control = { 'tcp_connect': {}, 'dns': { - 'ips': [] + 'addrs': [], + 'failure': None, }, 'http_request': { 'body_length': -1, @@ -297,23 +298,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = 'consistent' return True - control_ips = set(self.control['dns']['ips']) - experiment_ips = set(experiment_dns_answers) + control_addrs = set(self.control['dns']['addrs']) + experiment_addrs = set(experiment_dns_answers) - if control_ips == experiment_ips: + if control_addrs == experiment_addrs: return True - for experiment_ip in experiment_ips: - if is_public_ipv4_address(experiment_ip) is False: + for experiment_addr in experiment_addrs: + if is_public_ipv4_address(experiment_addr) is False: return False - if len(control_ips.intersection(experiment_ips)) > 0: + if len(control_addrs.intersection(experiment_addrs)) > 0: return True experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], - experiment_ips)) + experiment_addrs)) control_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'], - control_ips)) + control_addrs)) # Remove the instance of AS0 when we fail to find the ASN control_asns.discard('AS0')

1 0

[ooni-probe/master] Increase the default timeout for the vanilla tor test to 300 seconds (#512)
by art＠torproject.org 30 May '16

30 May '16

commit a62a614de8839aee9341ad5663598cfcdd0bb668 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 23 14:57:18 2016 +0200 Increase the default timeout for the vanilla tor test to 300 seconds (#512) This closes #508 --- ooni/nettests/blocking/vanilla_tor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/nettests/blocking/vanilla_tor.py b/ooni/nettests/blocking/vanilla_tor.py index 19c4805..7ed050f 100644 --- a/ooni/nettests/blocking/vanilla_tor.py +++ b/ooni/nettests/blocking/vanilla_tor.py @@ -18,7 +18,7 @@ class TorIsNotInstalled(Exception): class UsageOptions(usage.Options): optParameters = [ - ['timeout', 't', 200, + ['timeout', 't', 300, 'Specify the timeout after which to consider ' 'the Tor bootstrapping process to have failed.'], ]

1 0

[ooni-probe/master] Address feedback by @bassosimone
by art＠torproject.org 30 May '16

30 May '16

commit 2d5b7988c23697cd6af390c6ed649d69b7380dff Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue May 24 11:34:58 2016 +0200 Address feedback by @bassosimone * Distinguish between http-failure and http-diff * Update description of web_connectivity test * Various consistency and stylistic improvements --- ooni/nettests/blocking/web_connectivity.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 4f5f4e4..3560cb0 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -68,8 +68,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): Web connectivity """ name = "Web connectivity" - description = ("Performs a HTTP GET request over Tor and one over the " - "local network and compares the two results.") + description = ("Identifies the reason for blocking of a given URL by " + "performing DNS resolution of the hostname, doing a TCP " + "connect to the resolved IPs and then fetching the page " + "and comparing all these results with those of a control.") author = "Arturo Filastò" version = "0.1.0" @@ -126,12 +128,12 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): if not l: continue # Skip comment lines - elif l.startswith('#'): + if l.startswith('#'): continue yield l + fh = open(filename) try: - fh = open(filename) line = fh.readline() # Detect the line of the citizenlab input file if line.startswith("url,"): @@ -140,7 +142,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): fh.seek(0) generator = simple_file_generator(fh) for i in generator: - if not i.startswith("http"): + if (not i.startswith("http://") and + not i.startswith("https://")): i = "http://{}/".format(i) yield i finally: @@ -200,7 +203,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): log.msg("* doing DNS query for {}".format(self.hostname)) return self.performALookup(self.hostname) - def tcp_connect(self, socket): + def experiment_tcp_connect(self, socket): log.msg("* connecting to {}".format(socket)) ip_address, port = socket.split(":") port = int(port) @@ -365,12 +368,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http_failure is not None): blocking = 'tcp_ip' - # XXX we may want to have different codes for these two types of - # blocking elif (dns_consistent == True and tcp_connect == True and got_expected_web_page == False): - blocking = 'http' + blocking = 'http-diff' elif (dns_consistent == True and tcp_connect == True and @@ -379,7 +380,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): if experiment_http_failure == 'dns_lookup_error': blocking = 'dns' else: - blocking = 'http' + blocking = 'http-failure' elif (dns_consistent == False and (got_expected_web_page == False or @@ -425,7 +426,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): # STEALTH in here we should make changes to make the test more stealth dl = [] for socket in sockets: - dl.append(self.tcp_connect(socket)) + dl.append(self.experiment_tcp_connect(socket)) results = yield defer.DeferredList(dl) experiment_http = self.experiment_http_get_request()

1 0

[ooni-probe/master] Expand the heuristics for detecting blockpages in HTTP response
by art＠torproject.org 30 May '16

30 May '16

commit 7f8021efd69d28beb034a7cd25c60ea2c0016bf6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue May 24 18:06:00 2016 +0200 Expand the heuristics for detecting blockpages in HTTP response * Extract the title of the response * Consider only common HTTP headers --- ooni/nettests/blocking/web_connectivity.py | 76 ++++++++++++++++++++++-------- ooni/utils/net.py | 39 +++++++++++++++ 2 files changed, 95 insertions(+), 20 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index e640a5f..8f048e1 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -1,7 +1,6 @@ # -*- encoding: utf-8 -*- import csv -import json from urlparse import urlparse from ipaddr import IPv4Address, AddressValueError @@ -20,7 +19,7 @@ from ooni.utils import log from ooni.backend_client import WebConnectivityClient -from ooni.utils.net import StringProducer, BodyReceiver +from ooni.utils.net import COMMON_SERVER_HEADERS, extract_title from ooni.templates import httpt, dnst from ooni.errors import failureToString @@ -50,6 +49,7 @@ class UsageOptions(usage.Options): ['url', 'u', None, 'Specify a single URL to test'], ['dns-discovery', 'd', 'whoami.akamai.net', 'Specify the dns discovery test helper'], ['backend', 'b', None, 'The web_consistency backend test helper'], + ['retries', 'r', 1, 'Number of retries for the HTTP request'], ] @@ -158,6 +158,12 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): if not self.input: raise Exception("No input specified") + try: + self.localOptions['retries'] = int(self.localOptions['retries']) + except ValueError: + self.localOptions['retries'] = 2 + + self.report['retries'] = self.localOptions['retries'] self.report['client_resolver'] = self.resolverIp self.report['dns_consistency'] = None self.report['body_length_match'] = None @@ -188,7 +194,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): 'body_length': -1, 'failure': None, 'status_code': -1, - 'headers': {} + 'headers': {}, + 'title': '' } } if isinstance(self.localOptions['backend'], dict): @@ -240,24 +247,36 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): ) self.report['control'] = self.control + @defer.inlineCallbacks def experiment_http_get_request(self): - return self.doRequest(self.input, headers=REQUEST_HEADERS) + retries = 0 + while True: + try: + result = yield self.doRequest(self.input, + headers=REQUEST_HEADERS) + break + except: + if self.localOptions['retries'] > retries: + raise + retries += 1 + + defer.returnValue(result) def compare_headers(self, experiment_http_response): - count = 0 control_headers_lower = {k.lower(): v for k, v in - self.report['control']['http_request']['headers'].items()} + self.report['control']['http_request']['headers'].items() + } + experiment_headers_lower = {k.lower(): v for k, v in + experiment_http_response.headers.getAllRawHeaders() + } - for header_name, header_value in \ - experiment_http_response.headers.getAllRawHeaders(): - try: - control_headers_lower[header_name.lower()] - except KeyError: - log.debug("Did not find the key {}".format(header_name)) - return False - count += 1 + uncommon_ctrl_headers = (set(control_headers_lower.keys()) - + set(COMMON_SERVER_HEADERS)) + uncommon_exp_headers = (set(experiment_headers_lower.keys()) - + set(COMMON_SERVER_HEADERS)) - return count == len(self.report['control']['http_request']['headers']) + return len(uncommon_ctrl_headers.intersection( + uncommon_exp_headers)) > 0 def compare_body_lengths(self, experiment_http_response): control_body_length = self.control['http_request']['body_length'] @@ -279,6 +298,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): else: return False + def compare_titles(self, experiment_http_response): + experiment_title = extract_title(experiment_http_response.body).strip() + control_title = self.control['http_request']['title'].strip() + first_exp_word = experiment_title.split(' ')[0] + first_ctrl_word = control_title.split(' ')[0] + if len(first_exp_word) < 5: + # We don't consider to match words that are shorter than 5 + # characters (5 is the average word length for english) + return False + return (first_ctrl_word.lower() == first_exp_word.lower()) + def compare_http_experiments(self, experiment_http_response): self.report['body_length_match'] = \ @@ -292,6 +322,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.control['http_request']['status_code'] ) + self.report['title_match'] = self.compare_titles(experiment_http_response) + def compare_dns_experiments(self, experiment_dns_answers): if self.control['dns']['failure'] is not None and \ self.control['dns']['failure'] == self.report['dns_experiment_failure']: @@ -359,11 +391,15 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = 'inconsistent' tcp_connect = self.compare_tcp_experiments() - got_expected_web_page = ( - (self.report['body_length_match'] is True or - self.report['headers_match'] is True) - and self.report['status_code_match'] is True - ) + got_expected_web_page = None + if (experiment_http_failure is None and + control_http_failure is None): + got_expected_web_page = ( + (self.report['body_length_match'] is True or + self.report['headers_match'] is True or + self.report['title_match']) + and self.report['status_code_match'] is True + ) if (dns_consistent == True and tcp_connect == False and experiment_http_failure is not None): diff --git a/ooni/utils/net.py b/ooni/utils/net.py index ad5454e..20f5a42 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -1,3 +1,4 @@ +import re import sys import socket from random import randint @@ -46,10 +47,48 @@ PLATFORMS = {'LINUX': sys.platform.startswith("linux"), 'SOLARIS': sys.platform.startswith("sunos"), 'WINDOWS': sys.platform.startswith("win32")} +# These are the 25 most common server headers for the sites in the +# citizenlab global testing list. +COMMON_SERVER_HEADERS = ( + "date", + "content-type", + "server", + "cache-control", + "vary", + "set-cookie", + "location", + "expires", + "x-powered-by", + "content-encoding", + "last-modified", + "accept-ranges", + "pragma", + "x-frame-options", + "etag", + "x-content-type-options", + "age", + "via", + "p3p", + "x-xss-protection", + "content-language", + "cf-ray", + "strict-transport-security", + "link", + "x-varnish" +) + # This is used as a default for checking if we get the expected result when # fetching URLs over some proxy. GOOGLE_HUMANS = ('http://www.google.com/humans.txt', 'Google is built by a large') +TITLE_REGEXP = re.compile("<title>(.*?)</title>", re.IGNORECASE | re.DOTALL) + +def extract_title(body): + m = TITLE_REGEXP.search(body) + if m: + return m.group(1) + return '' + class StringProducer(object): implements(IBodyProducer)

1 0