commit c32bd88b5b99ea7d3fc62c9088d3ac17770ba7df Author: Arturo Filastò <arturo@filasto.net> Date: Tue May 24 11:18:10 2016 +0200 Consider valid onion addresses also when they don't use the httpo:// schema Based on feedback from @willscott * Fix invalid onion address in unittests --- ooni/backend_client.py | 16 ++++++++++------ ooni/deck.py | 4 ++-- ooni/tests/mocks.py | 6 +++--- ooni/tests/test_deck.py | 2 +- ooni/tests/test_onion.py | 16 ++++++++++++++++ ooni/utils/onion.py | 8 ++++++++ 6 files changed, 40 insertions(+), 12 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py index 3b551d4..de01ceb 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -14,7 +14,7 @@ _twisted_14_0_2_version = Version('twisted', 14, 0, 2) from ooni import errors as e from ooni.settings import config -from ooni.utils import log +from ooni.utils import log, onion from ooni.utils.net import BodyReceiver, StringProducer, Downloader from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent @@ -42,10 +42,10 @@ class OONIBClient(object): def _guessBackendType(self): if self.base_address is None: raise e.InvalidAddress - if self.base_address.startswith('https://'): - self.backend_type = 'https' - elif self.base_address.startswith('httpo://'): + if onion.is_onion_address(self.base_address): self.backend_type = 'onion' + elif self.base_address.startswith('https://'): + self.backend_type = 'https' elif self.base_address.startswith('http://'): self.backend_type = 'http' else: @@ -54,10 +54,14 @@ class OONIBClient(object): def _setupBaseAddress(self): parsed_address = urlparse(self.base_address) if self.backend_type == 'onion': - if not parsed_address.netloc.endswith(".onion"): + if not onion.is_onion_address(self.base_address): log.err("Invalid onion address.") raise e.InvalidAddress(self.base_address) - self.base_address = ("http://%s" % parsed_address.netloc) + if parsed_address.scheme in ('http', 'httpo'): + self.base_address = ("http://%s" % parsed_address.netloc) + else: + self.base_address = ("%s://%s" % (parsed_address.scheme, + parsed_address.netloc)) elif self.backend_type == 'http': self.base_address = ("http://%s" % parsed_address.netloc) elif self.backend_type in ('https', 'cloudfront'): diff --git a/ooni/deck.py b/ooni/deck.py index 7a3b697..383845c 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -4,7 +4,7 @@ from ooni.backend_client import CollectorClient, BouncerClient from ooni.backend_client import WebConnectivityClient from ooni.nettest import NetTestLoader from ooni.settings import config -from ooni.utils import log +from ooni.utils import log, onion from ooni import errors as e from twisted.python.filepath import FilePath @@ -175,7 +175,7 @@ class Deck(InputFile): https_addresses = [] plaintext_addresses = [] - if priority_address.startswith('httpo://'): + if onion.is_onion_address(priority_address): priority_address = { 'address': priority_address, 'type': 'onion' diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py index f3f852f..db7b154 100644 --- a/ooni/tests/mocks.py +++ b/ooni/tests/mocks.py @@ -197,13 +197,13 @@ class MockBouncerClient(object): ret = { 'default': { 'address': '127.0.0.1', - 'collector': 'httpo://thirteenchars1234.onion' + 'collector': 'httpo://thirteenchars123.onion' } } for required_test_helper in required_test_helpers: ret[required_test_helper] = { 'address': '127.0.0.1', - 'collector': 'httpo://thirteenchars1234.onion' + 'collector': 'httpo://thirteenchars123.onion' } return defer.succeed(ret) @@ -221,7 +221,7 @@ class MockBouncerClient(object): 'name': net_test['name'], 'version': net_test['version'], 'input-hashes': net_test['input-hashes'], - 'collector': 'httpo://thirteenchars1234.onion', + 'collector': 'httpo://thirteenchars123.onion', 'test-helpers': test_helpers }) return defer.succeed(ret) diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index 8d415f0..fa9b0e8 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -160,7 +160,7 @@ class TestDeck(BaseTestCase): yield deck.lookupCollectorAndTestHelpers() self.assertEqual(deck.netTestLoaders[0].collector.settings['address'], - 'http://thirteenchars1234.onion') + 'http://thirteenchars123.onion') self.assertEqual(deck.netTestLoaders[0].localOptions['backend'], '127.0.0.1') diff --git a/ooni/tests/test_onion.py b/ooni/tests/test_onion.py index 30fd26d..944f8f5 100644 --- a/ooni/tests/test_onion.py +++ b/ooni/tests/test_onion.py @@ -48,3 +48,19 @@ class TestOnion(unittest.TestCase): {'version': onion.TorVersion('0.2.3.20')}): self.assertRaises(onion.OutdatedTor, onion.bridge_line, 'fte', '/log.txt') + + def test_is_onion_address(self): + self.assertEqual(onion.is_onion_address( + 'httpo://thirteenchars123.onion'), True) + + self.assertEqual(onion.is_onion_address( + 'thirteenchars123.onion'), True) + + self.assertEqual(onion.is_onion_address( + 'http://thirteenchars123.onion'), True) + + self.assertEqual(onion.is_onion_address( + 'https://thirteenchars123.onion'), True) + + self.assertEqual(onion.is_onion_address( + 'http://thirteenchars123.com'), False) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index ec0a8b4..fea75c2 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -1,3 +1,4 @@ +import re import string import subprocess from distutils.spawn import find_executable @@ -7,6 +8,8 @@ from txtorcon.util import find_tor_binary as tx_find_tor_binary from ooni.settings import config +ONION_ADDRESS_REGEXP = re.compile("^((httpo|http|https)://)?" + "[a-z0-9]{16}\.onion") class TorVersion(LooseVersion): pass @@ -65,6 +68,11 @@ def transport_name(address): return None +def is_onion_address(address): + return ONION_ADDRESS_REGEXP.match(address) != None + + + tor_details = { 'binary': find_tor_binary(), 'version': tor_version()