commit 9d52e351e79521175ce2e91e5b00dbe72d8f1814 Author: Arturo Filastò arturo@filasto.net Date: Thu Jun 16 20:03:57 2016 +0300

Add a configuration option that allows to prioritise non-onion backends

This option allows a user to specify that they wish to use either a https, cloudfronted or onion backend server.

Write unittests to use the priority address --- data/ooniprobe.conf.sample | 3 ++ ooni/backend_client.py | 26 +++++------ ooni/constants.py | 5 +++ ooni/deck.py | 109 ++++++++++++++++++++++++++------------------- ooni/errors.py | 4 ++ ooni/oonicli.py | 11 ++--- ooni/tests/bases.py | 10 +++-- ooni/tests/mocks.py | 9 ++++ ooni/tests/test_deck.py | 70 ++++++++++++++++++++++++++--- 9 files changed, 175 insertions(+), 72 deletions(-)

diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index be35364..e310ddc 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -48,7 +48,10 @@ advanced: report_log_file: null inputs_dir: null decks_dir: null + # If we should support communicating to plaintext backends (via HTTP) insecure_backend: false + # The preferred backend type, can be one of onion, https or cloudfront + preferred_backend: onion tor: #socks_port: 8801 #control_port: 8802 diff --git a/ooni/backend_client.py b/ooni/backend_client.py index 0e85dd7..71b774f 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -19,6 +19,18 @@ from ooni.utils.net import BodyReceiver, StringProducer, Downloader from ooni.utils.socks import TrueHeadersSOCKS5Agent

+def guess_backend_type(address): + if address is None: + raise e.InvalidAddress + if onion.is_onion_address(address): + return 'onion' + elif address.startswith('https://'): + return 'https' + elif address.startswith('http://'): + return 'http' + else: + raise e.InvalidAddress + class OONIBClient(object): def __init__(self, address=None, settings={}): self.base_headers = {} @@ -26,7 +38,7 @@ class OONIBClient(object): self.base_address = settings.get('address', address)

if self.backend_type is None: - self._guessBackendType() + self.backend_type = guess_backend_type(self.base_address) self.backend_type = self.backend_type.encode('ascii')

if self.backend_type == 'cloudfront': @@ -39,18 +51,6 @@ class OONIBClient(object): 'front': settings.get('front', '').encode('ascii') }

- def _guessBackendType(self): - if self.base_address is None: - raise e.InvalidAddress - if onion.is_onion_address(self.base_address): - self.backend_type = 'onion' - elif self.base_address.startswith('https://'): - self.backend_type = 'https' - elif self.base_address.startswith('http://'): - self.backend_type = 'http' - else: - raise e.InvalidAddress - def _setupBaseAddress(self): parsed_address = urlparse(self.base_address) if self.backend_type == 'onion': diff --git a/ooni/constants.py b/ooni/constants.py index 882edb4..ef1f3d7 100644 --- a/ooni/constants.py +++ b/ooni/constants.py @@ -1,4 +1,9 @@ CANONICAL_BOUNCER_ONION = 'httpo://nkvphnp3p6agi5qq.onion' +CANONICAL_BOUNCER_HTTPS = 'https://bouncer.ooni.io' +CANONICAL_BOUNCER_CLOUDFRONT = ( + 'XXXX.cloudfront.net', + 'https://a0.awsstatic.com/' +)

MEEK_BRIDGES = [ ("meek 0.0.2.0:2 B9E7141C594AF25699E0079C1F0146F409495296 " diff --git a/ooni/deck.py b/ooni/deck.py index 82f98a1..2aa958d 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*-

from ooni.backend_client import CollectorClient, BouncerClient -from ooni.backend_client import WebConnectivityClient +from ooni.backend_client import WebConnectivityClient, guess_backend_type from ooni.nettest import NetTestLoader from ooni.settings import config from ooni.utils import log, onion +from ooni import constants from ooni import errors as e

from twisted.python.filepath import FilePath @@ -120,7 +121,29 @@ class Deck(InputFile): no_collector=False): self.id = deck_hash self.no_collector = no_collector - self.bouncer = bouncer + + self.preferred_backend = config.advanced.get( + "preferred_backend", "onion" + ) + if self.preferred_backend not in ["onion", "https", "cloudfront"]: + raise e.InvalidPreferredBackend + + if bouncer is None: + bouncer_address = getattr( + constants, "CANONICAL_BOUNCER_{0}".format( + self.preferred_backend.upper() + ) + ) + if self.preferred_backend == "cloudfront": + self.bouncer = self._BouncerClient(settings={ + 'address': bouncer_address[0], + 'front': bouncer_address[1], + 'type': 'cloudfront' + }) + else: + self.bouncer = self._BouncerClient(bouncer_address) + else: + self.bouncer = self._BouncerClient(bouncer)

self.requiresTor = False

@@ -167,19 +190,23 @@ class Deck(InputFile): collector_address ) if test['options'].get('bouncer', None) is not None: - self.bouncer = test['options']['bouncer'] + self.bouncer = self._BouncerClient(test['options']['bouncer']) + if self.bouncer.backend_type is "onion": + self.requiresTor = True self.insert(net_test_loader)

def insert(self, net_test_loader): """ Add a NetTestLoader to this test deck """ + if (net_test_loader.collector is not None + and net_test_loader.collector.backend_type is "onion"): + self.requiresTor = True try: net_test_loader.checkOptions() if net_test_loader.requiresTor: self.requiresTor = True except e.MissingTestHelper: - if not self.bouncer: - raise - self.requiresTor = True + if self.preferred_backend is "onion": + self.requiresTor = True

self.netTestLoaders.append(net_test_loader)

@@ -188,6 +215,7 @@ class Deck(InputFile): """ fetch and verify inputs for all NetTests in the deck """ log.msg("Fetching required net test inputs...") for net_test_loader in self.netTestLoaders: + # XXX figure out if we want to keep this or drop this. yield self.fetchAndVerifyNetTestInput(net_test_loader)

if self.bouncer: @@ -196,44 +224,34 @@ class Deck(InputFile):

def sortAddressesByPriority(self, priority_address, alternate_addresses): - onion_addresses= [] - cloudfront_addresses= [] - https_addresses = [] - plaintext_addresses = [] - - if onion.is_onion_address(priority_address): - priority_address = { - 'address': priority_address, - 'type': 'onion' - } - elif priority_address.startswith('https://'): - priority_address = { - 'address': priority_address, - 'type': 'https' - } - elif priority_address.startswith('http://'): - priority_address = { - 'address': priority_address, - 'type': 'http' - } - else: - raise e.InvalidOONIBCollectorAddress + prioritised_addresses = [] + + backend_type = guess_backend_type(priority_address) + priority_address = { + 'address': priority_address, + 'type': backend_type + } + address_priority = ['onion', 'https', 'cloudfront', 'http'] + address_priority.remove(self.preferred_backend) + address_priority.insert(0, self.preferred_backend)

def filter_by_type(collectors, collector_type): - return filter(lambda x: x['type'] == collector_type, - collectors) - onion_addresses += filter_by_type(alternate_addresses, 'onion') - https_addresses += filter_by_type(alternate_addresses, 'https') - cloudfront_addresses += filter_by_type(alternate_addresses, - 'cloudfront') + return filter(lambda x: x['type'] == collector_type, collectors) + + if (priority_address['type'] != self.preferred_backend): + valid_alternatives = filter_by_type(alternate_addresses, + self.preferred_backend) + if len(valid_alternatives) > 0: + alternate_addresses += [priority_address] + priority_address = valid_alternatives[0] + alternate_addresses.remove(priority_address)

- plaintext_addresses += filter_by_type(alternate_addresses, 'http') + prioritised_addresses += [priority_address] + for address_type in address_priority: + prioritised_addresses += filter_by_type(alternate_addresses, + address_type)

- return ([priority_address] + - onion_addresses + - https_addresses + - cloudfront_addresses + - plaintext_addresses) + return prioritised_addresses

@defer.inlineCallbacks def getReachableCollector(self, collector_address, collector_alternate): @@ -289,9 +307,12 @@ class Deck(InputFile): @defer.inlineCallbacks def getReachableTestHelpersAndCollectors(self, net_tests): for net_test in net_tests: + + primary_address = net_test['collector'] + alternate_addresses = net_test.get('collector-alternate', []) net_test['collector'] = yield self.getReachableCollector( - net_test['collector'], - net_test.get('collector-alternate', []) + primary_address, + alternate_addresses )

for test_helper_name, test_helper_address in net_test['test-helpers'].items(): @@ -307,8 +328,6 @@ class Deck(InputFile):

@defer.inlineCallbacks def lookupCollectorAndTestHelpers(self): - oonibclient = self._BouncerClient(self.bouncer) - required_nettests = []

requires_test_helpers = False @@ -333,7 +352,7 @@ class Deck(InputFile): if not requires_test_helpers and not requires_collector: defer.returnValue(None)

- response = yield oonibclient.lookupTestCollector(required_nettests) + response = yield self.bouncer.lookupTestCollector(required_nettests) try: provided_net_tests = yield self.getReachableTestHelpersAndCollectors(response['net-tests']) except e.NoReachableCollectors: diff --git a/ooni/errors.py b/ooni/errors.py index 8197f34..faa3627 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -316,3 +316,7 @@ class NoReachableCollectors(Exception):

class NoReachableTestHelpers(Exception): pass + + +class InvalidPreferredBackend(Exception): + pass diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 74bf9ac..43654b6 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -45,9 +45,9 @@ class Options(usage.Options): ["collector", "c", None, "Specify the address of the collector for " "test results. In most cases a user will " "prefer to specify a bouncer over this."], - ["bouncer", "b", CANONICAL_BOUNCER_ONION, "Specify the bouncer used to " - "obtain the address of the " - "collector and test helpers."], + ["bouncer", "b", None, "Specify the bouncer used to " + "obtain the address of the " + "collector and test helpers."], ["logfile", "l", None, "Write to this logs to this filename."], ["pcapfile", "O", None, "Write a PCAP of the ooniprobe session to " "this filename."], @@ -388,10 +388,11 @@ def runWithDirector(global_options): log.msg("Not reporting using a collector") global_options['collector'] = None start_tor = False - else: + elif config.advanced.get("preferred_backend", "onion") == "onion": start_tor = True

- if global_options['collector']: + if (global_options['collector'] and + config.advanced.get("preferred_backend", "onion") == "onion"): start_tor |= True

return runTestWithDirector(director=director, diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py index 5d177fa..31cbf94 100644 --- a/ooni/tests/bases.py +++ b/ooni/tests/bases.py @@ -1,3 +1,5 @@ +import os +import shutil from twisted.trial import unittest

from ooni.settings import config @@ -5,11 +7,13 @@ from ooni.settings import config

class ConfigTestCase(unittest.TestCase): def setUp(self): - config.initialize_ooni_home("ooni_home") + self.ooni_home_dir = os.path.abspath("ooni_home") + self.config = config + self.config.initialize_ooni_home("ooni_home") + super(ConfigTestCase, self).setUp()

def skipTest(self, reason): raise unittest.SkipTest(reason)

def tearDown(self): - config.set_paths() - config.read_config_file() + shutil.rmtree("ooni_home") diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py index db7b154..a377fba 100644 --- a/ooni/tests/mocks.py +++ b/ooni/tests/mocks.py @@ -222,6 +222,15 @@ class MockBouncerClient(object): 'version': net_test['version'], 'input-hashes': net_test['input-hashes'], 'collector': 'httpo://thirteenchars123.onion', + 'collector-alternate': [ + {'type': 'https', 'address': 'https://collector.ooni.io%27%7D, + {'type': 'http', 'address': 'http://collector.ooni.io%27%7D, + { + 'type': 'cloudfront', + 'address': 'https://address.cloudfront.net', + 'front': 'https://front.cloudfront.net' + }, + ], 'test-helpers': test_helpers }) return defer.succeed(ret) diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index e4f661f..f299e70 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -6,8 +6,11 @@ from twisted.trial import unittest from hashlib import sha256 from ooni import errors from ooni.deck import InputFile, Deck, nettest_to_path +from ooni.tests.bases import ConfigTestCase from ooni.tests.mocks import MockBouncerClient, MockCollectorClient

+FAKE_BOUNCER_ADDRESS = "httpo://thirteenchars123.onion" + net_test_string = """ from twisted.python import usage from ooni.nettest import NetTestCase @@ -71,7 +74,7 @@ class BaseTestCase(unittest.TestCase): test_file: manipulation/http_invalid_request_line testdeck: null """ - + super(BaseTestCase, self).setUp()

class TestInputFile(BaseTestCase): @@ -112,7 +115,7 @@ class TestInputFile(BaseTestCase): assert input_file.descriptorCached

-class TestDeck(BaseTestCase): +class TestDeck(BaseTestCase, ConfigTestCase): def setUp(self): super(TestDeck, self).setUp() deck_hash = sha256(self.dummy_deck_content).hexdigest() @@ -127,9 +130,10 @@ class TestDeck(BaseTestCase): os.remove(self.deck_file) if self.filename != "": os.remove(self.filename) + super(TestDeck, self).tearDown()

def test_open_deck(self): - deck = Deck(bouncer="httpo://foo.onion", + deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, decks_directory=".") deck.loadDeck(self.deck_file) assert len(deck.netTestLoaders) == 1 @@ -139,7 +143,7 @@ class TestDeck(BaseTestCase): "annotations": {"spam": "ham"}, "collector": "httpo://thirteenchars123.onion" } - deck = Deck(bouncer="httpo://foo.onion", + deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, decks_directory=".") deck.loadDeck(self.deck_file, global_options=global_options) @@ -153,7 +157,7 @@ class TestDeck(BaseTestCase): )

def test_save_deck_descriptor(self): - deck = Deck(bouncer="httpo://foo.onion", + deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, decks_directory=".") deck.loadDeck(self.deck_file) deck.load({'name': 'spam', @@ -169,8 +173,9 @@ class TestDeck(BaseTestCase):

@defer.inlineCallbacks def test_lookup_test_helpers_and_collector(self): - deck = Deck(bouncer="httpo://foo.onion", + deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, decks_directory=".") + deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS) deck._BouncerClient = MockBouncerClient deck._CollectorClient = MockCollectorClient deck.loadDeck(self.deck_file) @@ -211,3 +216,56 @@ class TestDeck(BaseTestCase): self.assertRaises(errors.NetTestNotFound, nettest_to_path, "invalid_test") + + @defer.inlineCallbacks + def test_lookup_test_helpers_and_collector_cloudfront(self): + self.config.advanced.preferred_backend = "cloudfront" + deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, + decks_directory=".") + deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS) + deck._BouncerClient = MockBouncerClient + deck._CollectorClient = MockCollectorClient + deck.loadDeck(self.deck_file) + + self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1) + + yield deck.lookupCollectorAndTestHelpers() + + self.assertEqual( + deck.netTestLoaders[0].collector.settings['address'], + 'https://address.cloudfront.net' + ) + self.assertEqual( + deck.netTestLoaders[0].collector.settings['front'], + 'https://front.cloudfront.net' + ) + + self.assertEqual( + deck.netTestLoaders[0].localOptions['backend'], + '127.0.0.1' + ) + + + @defer.inlineCallbacks + def test_lookup_test_helpers_and_collector_https(self): + self.config.advanced.preferred_backend = "https" + deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, + decks_directory=".") + deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS) + deck._BouncerClient = MockBouncerClient + deck._CollectorClient = MockCollectorClient + deck.loadDeck(self.deck_file) + + self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1) + + yield deck.lookupCollectorAndTestHelpers() + + self.assertEqual( + deck.netTestLoaders[0].collector.settings['address'], + 'https://collector.ooni.io' + ) + + self.assertEqual( + deck.netTestLoaders[0].localOptions['backend'], + '127.0.0.1' + )