[tor-commits] [ooni-probe/master] Add a configuration option that allows to prioritise non-onion backends
art at torproject.org
art at torproject.org
Sun Jul 10 20:22:57 UTC 2016
commit 9d52e351e79521175ce2e91e5b00dbe72d8f1814
Author: Arturo Filastò <arturo at filasto.net>
Date: Thu Jun 16 20:03:57 2016 +0300
Add a configuration option that allows to prioritise non-onion backends
This option allows a user to specify that they wish to use either a https,
cloudfronted or onion backend server.
Write unittests to use the priority address
---
data/ooniprobe.conf.sample | 3 ++
ooni/backend_client.py | 26 +++++------
ooni/constants.py | 5 +++
ooni/deck.py | 109 ++++++++++++++++++++++++++-------------------
ooni/errors.py | 4 ++
ooni/oonicli.py | 11 ++---
ooni/tests/bases.py | 10 +++--
ooni/tests/mocks.py | 9 ++++
ooni/tests/test_deck.py | 70 ++++++++++++++++++++++++++---
9 files changed, 175 insertions(+), 72 deletions(-)
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index be35364..e310ddc 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -48,7 +48,10 @@ advanced:
report_log_file: null
inputs_dir: null
decks_dir: null
+ # If we should support communicating to plaintext backends (via HTTP)
insecure_backend: false
+ # The preferred backend type, can be one of onion, https or cloudfront
+ preferred_backend: onion
tor:
#socks_port: 8801
#control_port: 8802
diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index 0e85dd7..71b774f 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -19,6 +19,18 @@ from ooni.utils.net import BodyReceiver, StringProducer, Downloader
from ooni.utils.socks import TrueHeadersSOCKS5Agent
+def guess_backend_type(address):
+ if address is None:
+ raise e.InvalidAddress
+ if onion.is_onion_address(address):
+ return 'onion'
+ elif address.startswith('https://'):
+ return 'https'
+ elif address.startswith('http://'):
+ return 'http'
+ else:
+ raise e.InvalidAddress
+
class OONIBClient(object):
def __init__(self, address=None, settings={}):
self.base_headers = {}
@@ -26,7 +38,7 @@ class OONIBClient(object):
self.base_address = settings.get('address', address)
if self.backend_type is None:
- self._guessBackendType()
+ self.backend_type = guess_backend_type(self.base_address)
self.backend_type = self.backend_type.encode('ascii')
if self.backend_type == 'cloudfront':
@@ -39,18 +51,6 @@ class OONIBClient(object):
'front': settings.get('front', '').encode('ascii')
}
- def _guessBackendType(self):
- if self.base_address is None:
- raise e.InvalidAddress
- if onion.is_onion_address(self.base_address):
- self.backend_type = 'onion'
- elif self.base_address.startswith('https://'):
- self.backend_type = 'https'
- elif self.base_address.startswith('http://'):
- self.backend_type = 'http'
- else:
- raise e.InvalidAddress
-
def _setupBaseAddress(self):
parsed_address = urlparse(self.base_address)
if self.backend_type == 'onion':
diff --git a/ooni/constants.py b/ooni/constants.py
index 882edb4..ef1f3d7 100644
--- a/ooni/constants.py
+++ b/ooni/constants.py
@@ -1,4 +1,9 @@
CANONICAL_BOUNCER_ONION = 'httpo://nkvphnp3p6agi5qq.onion'
+CANONICAL_BOUNCER_HTTPS = 'https://bouncer.ooni.io'
+CANONICAL_BOUNCER_CLOUDFRONT = (
+ 'XXXX.cloudfront.net',
+ 'https://a0.awsstatic.com/'
+)
MEEK_BRIDGES = [
("meek 0.0.2.0:2 B9E7141C594AF25699E0079C1F0146F409495296 "
diff --git a/ooni/deck.py b/ooni/deck.py
index 82f98a1..2aa958d 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -1,10 +1,11 @@
# -*- coding: utf-8 -*-
from ooni.backend_client import CollectorClient, BouncerClient
-from ooni.backend_client import WebConnectivityClient
+from ooni.backend_client import WebConnectivityClient, guess_backend_type
from ooni.nettest import NetTestLoader
from ooni.settings import config
from ooni.utils import log, onion
+from ooni import constants
from ooni import errors as e
from twisted.python.filepath import FilePath
@@ -120,7 +121,29 @@ class Deck(InputFile):
no_collector=False):
self.id = deck_hash
self.no_collector = no_collector
- self.bouncer = bouncer
+
+ self.preferred_backend = config.advanced.get(
+ "preferred_backend", "onion"
+ )
+ if self.preferred_backend not in ["onion", "https", "cloudfront"]:
+ raise e.InvalidPreferredBackend
+
+ if bouncer is None:
+ bouncer_address = getattr(
+ constants, "CANONICAL_BOUNCER_{0}".format(
+ self.preferred_backend.upper()
+ )
+ )
+ if self.preferred_backend == "cloudfront":
+ self.bouncer = self._BouncerClient(settings={
+ 'address': bouncer_address[0],
+ 'front': bouncer_address[1],
+ 'type': 'cloudfront'
+ })
+ else:
+ self.bouncer = self._BouncerClient(bouncer_address)
+ else:
+ self.bouncer = self._BouncerClient(bouncer)
self.requiresTor = False
@@ -167,19 +190,23 @@ class Deck(InputFile):
collector_address
)
if test['options'].get('bouncer', None) is not None:
- self.bouncer = test['options']['bouncer']
+ self.bouncer = self._BouncerClient(test['options']['bouncer'])
+ if self.bouncer.backend_type is "onion":
+ self.requiresTor = True
self.insert(net_test_loader)
def insert(self, net_test_loader):
""" Add a NetTestLoader to this test deck """
+ if (net_test_loader.collector is not None
+ and net_test_loader.collector.backend_type is "onion"):
+ self.requiresTor = True
try:
net_test_loader.checkOptions()
if net_test_loader.requiresTor:
self.requiresTor = True
except e.MissingTestHelper:
- if not self.bouncer:
- raise
- self.requiresTor = True
+ if self.preferred_backend is "onion":
+ self.requiresTor = True
self.netTestLoaders.append(net_test_loader)
@@ -188,6 +215,7 @@ class Deck(InputFile):
""" fetch and verify inputs for all NetTests in the deck """
log.msg("Fetching required net test inputs...")
for net_test_loader in self.netTestLoaders:
+ # XXX figure out if we want to keep this or drop this.
yield self.fetchAndVerifyNetTestInput(net_test_loader)
if self.bouncer:
@@ -196,44 +224,34 @@ class Deck(InputFile):
def sortAddressesByPriority(self, priority_address, alternate_addresses):
- onion_addresses= []
- cloudfront_addresses= []
- https_addresses = []
- plaintext_addresses = []
-
- if onion.is_onion_address(priority_address):
- priority_address = {
- 'address': priority_address,
- 'type': 'onion'
- }
- elif priority_address.startswith('https://'):
- priority_address = {
- 'address': priority_address,
- 'type': 'https'
- }
- elif priority_address.startswith('http://'):
- priority_address = {
- 'address': priority_address,
- 'type': 'http'
- }
- else:
- raise e.InvalidOONIBCollectorAddress
+ prioritised_addresses = []
+
+ backend_type = guess_backend_type(priority_address)
+ priority_address = {
+ 'address': priority_address,
+ 'type': backend_type
+ }
+ address_priority = ['onion', 'https', 'cloudfront', 'http']
+ address_priority.remove(self.preferred_backend)
+ address_priority.insert(0, self.preferred_backend)
def filter_by_type(collectors, collector_type):
- return filter(lambda x: x['type'] == collector_type,
- collectors)
- onion_addresses += filter_by_type(alternate_addresses, 'onion')
- https_addresses += filter_by_type(alternate_addresses, 'https')
- cloudfront_addresses += filter_by_type(alternate_addresses,
- 'cloudfront')
+ return filter(lambda x: x['type'] == collector_type, collectors)
+
+ if (priority_address['type'] != self.preferred_backend):
+ valid_alternatives = filter_by_type(alternate_addresses,
+ self.preferred_backend)
+ if len(valid_alternatives) > 0:
+ alternate_addresses += [priority_address]
+ priority_address = valid_alternatives[0]
+ alternate_addresses.remove(priority_address)
- plaintext_addresses += filter_by_type(alternate_addresses, 'http')
+ prioritised_addresses += [priority_address]
+ for address_type in address_priority:
+ prioritised_addresses += filter_by_type(alternate_addresses,
+ address_type)
- return ([priority_address] +
- onion_addresses +
- https_addresses +
- cloudfront_addresses +
- plaintext_addresses)
+ return prioritised_addresses
@defer.inlineCallbacks
def getReachableCollector(self, collector_address, collector_alternate):
@@ -289,9 +307,12 @@ class Deck(InputFile):
@defer.inlineCallbacks
def getReachableTestHelpersAndCollectors(self, net_tests):
for net_test in net_tests:
+
+ primary_address = net_test['collector']
+ alternate_addresses = net_test.get('collector-alternate', [])
net_test['collector'] = yield self.getReachableCollector(
- net_test['collector'],
- net_test.get('collector-alternate', [])
+ primary_address,
+ alternate_addresses
)
for test_helper_name, test_helper_address in net_test['test-helpers'].items():
@@ -307,8 +328,6 @@ class Deck(InputFile):
@defer.inlineCallbacks
def lookupCollectorAndTestHelpers(self):
- oonibclient = self._BouncerClient(self.bouncer)
-
required_nettests = []
requires_test_helpers = False
@@ -333,7 +352,7 @@ class Deck(InputFile):
if not requires_test_helpers and not requires_collector:
defer.returnValue(None)
- response = yield oonibclient.lookupTestCollector(required_nettests)
+ response = yield self.bouncer.lookupTestCollector(required_nettests)
try:
provided_net_tests = yield self.getReachableTestHelpersAndCollectors(response['net-tests'])
except e.NoReachableCollectors:
diff --git a/ooni/errors.py b/ooni/errors.py
index 8197f34..faa3627 100644
--- a/ooni/errors.py
+++ b/ooni/errors.py
@@ -316,3 +316,7 @@ class NoReachableCollectors(Exception):
class NoReachableTestHelpers(Exception):
pass
+
+
+class InvalidPreferredBackend(Exception):
+ pass
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 74bf9ac..43654b6 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -45,9 +45,9 @@ class Options(usage.Options):
["collector", "c", None, "Specify the address of the collector for "
"test results. In most cases a user will "
"prefer to specify a bouncer over this."],
- ["bouncer", "b", CANONICAL_BOUNCER_ONION, "Specify the bouncer used to "
- "obtain the address of the "
- "collector and test helpers."],
+ ["bouncer", "b", None, "Specify the bouncer used to "
+ "obtain the address of the "
+ "collector and test helpers."],
["logfile", "l", None, "Write to this logs to this filename."],
["pcapfile", "O", None, "Write a PCAP of the ooniprobe session to "
"this filename."],
@@ -388,10 +388,11 @@ def runWithDirector(global_options):
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
- else:
+ elif config.advanced.get("preferred_backend", "onion") == "onion":
start_tor = True
- if global_options['collector']:
+ if (global_options['collector'] and
+ config.advanced.get("preferred_backend", "onion") == "onion"):
start_tor |= True
return runTestWithDirector(director=director,
diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py
index 5d177fa..31cbf94 100644
--- a/ooni/tests/bases.py
+++ b/ooni/tests/bases.py
@@ -1,3 +1,5 @@
+import os
+import shutil
from twisted.trial import unittest
from ooni.settings import config
@@ -5,11 +7,13 @@ from ooni.settings import config
class ConfigTestCase(unittest.TestCase):
def setUp(self):
- config.initialize_ooni_home("ooni_home")
+ self.ooni_home_dir = os.path.abspath("ooni_home")
+ self.config = config
+ self.config.initialize_ooni_home("ooni_home")
+ super(ConfigTestCase, self).setUp()
def skipTest(self, reason):
raise unittest.SkipTest(reason)
def tearDown(self):
- config.set_paths()
- config.read_config_file()
+ shutil.rmtree("ooni_home")
diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py
index db7b154..a377fba 100644
--- a/ooni/tests/mocks.py
+++ b/ooni/tests/mocks.py
@@ -222,6 +222,15 @@ class MockBouncerClient(object):
'version': net_test['version'],
'input-hashes': net_test['input-hashes'],
'collector': 'httpo://thirteenchars123.onion',
+ 'collector-alternate': [
+ {'type': 'https', 'address': 'https://collector.ooni.io'},
+ {'type': 'http', 'address': 'http://collector.ooni.io'},
+ {
+ 'type': 'cloudfront',
+ 'address': 'https://address.cloudfront.net',
+ 'front': 'https://front.cloudfront.net'
+ },
+ ],
'test-helpers': test_helpers
})
return defer.succeed(ret)
diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py
index e4f661f..f299e70 100644
--- a/ooni/tests/test_deck.py
+++ b/ooni/tests/test_deck.py
@@ -6,8 +6,11 @@ from twisted.trial import unittest
from hashlib import sha256
from ooni import errors
from ooni.deck import InputFile, Deck, nettest_to_path
+from ooni.tests.bases import ConfigTestCase
from ooni.tests.mocks import MockBouncerClient, MockCollectorClient
+FAKE_BOUNCER_ADDRESS = "httpo://thirteenchars123.onion"
+
net_test_string = """
from twisted.python import usage
from ooni.nettest import NetTestCase
@@ -71,7 +74,7 @@ class BaseTestCase(unittest.TestCase):
test_file: manipulation/http_invalid_request_line
testdeck: null
"""
-
+ super(BaseTestCase, self).setUp()
class TestInputFile(BaseTestCase):
@@ -112,7 +115,7 @@ class TestInputFile(BaseTestCase):
assert input_file.descriptorCached
-class TestDeck(BaseTestCase):
+class TestDeck(BaseTestCase, ConfigTestCase):
def setUp(self):
super(TestDeck, self).setUp()
deck_hash = sha256(self.dummy_deck_content).hexdigest()
@@ -127,9 +130,10 @@ class TestDeck(BaseTestCase):
os.remove(self.deck_file)
if self.filename != "":
os.remove(self.filename)
+ super(TestDeck, self).tearDown()
def test_open_deck(self):
- deck = Deck(bouncer="httpo://foo.onion",
+ deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
decks_directory=".")
deck.loadDeck(self.deck_file)
assert len(deck.netTestLoaders) == 1
@@ -139,7 +143,7 @@ class TestDeck(BaseTestCase):
"annotations": {"spam": "ham"},
"collector": "httpo://thirteenchars123.onion"
}
- deck = Deck(bouncer="httpo://foo.onion",
+ deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
decks_directory=".")
deck.loadDeck(self.deck_file,
global_options=global_options)
@@ -153,7 +157,7 @@ class TestDeck(BaseTestCase):
)
def test_save_deck_descriptor(self):
- deck = Deck(bouncer="httpo://foo.onion",
+ deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
decks_directory=".")
deck.loadDeck(self.deck_file)
deck.load({'name': 'spam',
@@ -169,8 +173,9 @@ class TestDeck(BaseTestCase):
@defer.inlineCallbacks
def test_lookup_test_helpers_and_collector(self):
- deck = Deck(bouncer="httpo://foo.onion",
+ deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
decks_directory=".")
+ deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
deck._BouncerClient = MockBouncerClient
deck._CollectorClient = MockCollectorClient
deck.loadDeck(self.deck_file)
@@ -211,3 +216,56 @@ class TestDeck(BaseTestCase):
self.assertRaises(errors.NetTestNotFound,
nettest_to_path,
"invalid_test")
+
+ @defer.inlineCallbacks
+ def test_lookup_test_helpers_and_collector_cloudfront(self):
+ self.config.advanced.preferred_backend = "cloudfront"
+ deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
+ decks_directory=".")
+ deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
+ deck._BouncerClient = MockBouncerClient
+ deck._CollectorClient = MockCollectorClient
+ deck.loadDeck(self.deck_file)
+
+ self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
+
+ yield deck.lookupCollectorAndTestHelpers()
+
+ self.assertEqual(
+ deck.netTestLoaders[0].collector.settings['address'],
+ 'https://address.cloudfront.net'
+ )
+ self.assertEqual(
+ deck.netTestLoaders[0].collector.settings['front'],
+ 'https://front.cloudfront.net'
+ )
+
+ self.assertEqual(
+ deck.netTestLoaders[0].localOptions['backend'],
+ '127.0.0.1'
+ )
+
+
+ @defer.inlineCallbacks
+ def test_lookup_test_helpers_and_collector_https(self):
+ self.config.advanced.preferred_backend = "https"
+ deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
+ decks_directory=".")
+ deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
+ deck._BouncerClient = MockBouncerClient
+ deck._CollectorClient = MockCollectorClient
+ deck.loadDeck(self.deck_file)
+
+ self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
+
+ yield deck.lookupCollectorAndTestHelpers()
+
+ self.assertEqual(
+ deck.netTestLoaders[0].collector.settings['address'],
+ 'https://collector.ooni.io'
+ )
+
+ self.assertEqual(
+ deck.netTestLoaders[0].localOptions['backend'],
+ '127.0.0.1'
+ )
More information about the tor-commits
mailing list