commit 2b238683d01d2d974a3743a226dd30f02371faf0 Author: Arturo Filastò arturo@filasto.net Date: Fri May 13 15:31:58 2016 +0200

Make the test-helper-alternate key work when speaking to backend

* Rename advanced option insecure_collector to insecure_backend

* Refactor isReachable() to only return booleans

* Bugfixing --- data/ooniprobe.conf.sample | 2 +- ooni/backend_client.py | 21 ++++----- ooni/deck.py | 70 +++++++++++++----------------- ooni/errors.py | 11 +---- ooni/nettests/blocking/web_connectivity.py | 1 + ooni/oonicli.py | 4 +- 6 files changed, 45 insertions(+), 64 deletions(-)

diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index edc3f86..72a8e06 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -48,7 +48,7 @@ advanced: report_log_file: null inputs_dir: null decks_dir: null - insecure_collector: false + insecure_backend: false tor: #socks_port: 8801 #control_port: 8802 diff --git a/ooni/backend_client.py b/ooni/backend_client.py index c7de7f0..ae376e4 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -23,7 +23,7 @@ class OONIBClient(object): def __init__(self, address=None, settings={}): self.base_headers = {} self.backend_type = settings.get('type', None) - self.base_address = settings.get('address', address).encode('ascii') + self.base_address = settings.get('address', address)

if self.backend_type is None: self._guessBackendType() @@ -62,6 +62,7 @@ class OONIBClient(object): self.base_address = ("http://%s" % parsed_address.netloc) elif self.backend_type in ('https', 'cloudfront'): self.base_address = ("https://%s" % parsed_address.netloc) + self.base_address = self.base_address.encode('ascii')

def isSupported(self): if self.backend_type in ("https", "cloudfront"): @@ -70,10 +71,10 @@ class OONIBClient(object): "twisted > 14.0.2.") return False elif self.backend_type == "http": - if config.advanced.insecure_collector is not True: + if config.advanced.insecure_backend is not True: log.err("Plaintext backends are not supported. To " "enable at your own risk set " - "advanced->insecure_collector to true") + "advanced->insecure_backend to true") return False elif self.backend_type == "onion": # XXX add an extra check to ensure tor is running @@ -159,7 +160,7 @@ class OONIBClient(object):

class BouncerClient(OONIBClient): def isReachable(self): - pass + return defer.succeed(True)

@defer.inlineCallbacks def lookupTestCollector(self, net_tests): @@ -197,14 +198,14 @@ class CollectorClient(OONIBClient): def cb(_): # We should never be getting an acceptable response for a # request to an invalid path. - raise e.CollectorUnreachable + return False

@d.addErrback def err(failure): failure.trap(Error) if failure.value.status == '404': return True - raise e.CollectorUnreachable + return False

return d

@@ -346,13 +347,13 @@ class WebConnectivityClient(OONIBClient):

@d.addCallback def cb(result): - if result.get("status", None) is not "ok": - raise e.TestHelperUnreachable + if result.get("status", None) != "ok": + return False return True

@d.addErrback def err(_): - raise e.TestHelperUnreachable + return False

return d

@@ -361,4 +362,4 @@ class WebConnectivityClient(OONIBClient): 'http_request': http_request, 'tcp_connect': tcp_connect } - self.queryBackend('POST', '/', query=request) + return self.queryBackend('POST', '/', query=request) diff --git a/ooni/deck.py b/ooni/deck.py index 24c7904..007097e 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -184,11 +184,10 @@ class Deck(InputFile): 'type': 'https' } elif priority_address.startswith('http://'): - if config.advanced.insecure_collector is True: - priority_address = { - 'address': priority_address, - 'type': 'http' - } + priority_address = { + 'address': priority_address, + 'type': 'http' + } else: raise e.InvalidOONIBCollectorAddress

@@ -200,13 +199,13 @@ class Deck(InputFile): cloudfront_addresses += filter_by_type(alternate_addresses, 'cloudfront')

- if config.advanced.insecure_collector is True: - plaintext_addresses += filter_by_type(alternate_addresses, 'http') + plaintext_addresses += filter_by_type(alternate_addresses, 'http')

return ([priority_address] + onion_addresses + https_addresses + - cloudfront_addresses) + cloudfront_addresses + + plaintext_addresses)

@defer.inlineCallbacks def getReachableCollector(self, collector_address, collector_alternate): @@ -214,24 +213,19 @@ class Deck(InputFile): # collectors to plaintext collectors for collector_settings in self.sortAddressesByPriority(collector_address, collector_alternate): - try: - collector = self._CollectorClient(settings=collector_settings) - if not collector.isSupported(): - log.err("Unsupported %s collector %s" % ( - collector_settings['type'], - collector_settings['address'])) - continue - reachable = yield collector.isReachable() - if not reachable: - log.err("Unreachable %s collector %s" % ( - collector_settings['type'], - collector_settings['address'])) - continue - defer.returnValue(collector) - except e.CollectorUnreachable: - log.msg("Could not reach %s collector %s" % ( + collector = self._CollectorClient(settings=collector_settings) + if not collector.isSupported(): + log.err("Unsupported %s collector %s" % ( collector_settings['type'], collector_settings['address'])) + continue + reachable = yield collector.isReachable() + if not reachable: + log.err("Unreachable %s collector %s" % ( + collector_settings['type'], + collector_settings['address'])) + continue + defer.returnValue(collector)

raise e.NoReachableCollectors

@@ -240,32 +234,26 @@ class Deck(InputFile): test_helper_alternate): # For the moment we look for alternate addresses only of # web_connectivity test helpers. - if test_helper_name is 'web_connectivity': + if test_helper_name == 'web-connectivity': for web_connectivity_settings in self.sortAddressesByPriority( test_helper_address, test_helper_alternate): - try: - web_connectivity_test_helper = WebConnectivityClient(web_connectivity_settings) - if not web_connectivity_test_helper.isSupported(): - log.err("Unsupported %s web_connectivity test_helper " - "%s" % ( - web_connectivity_settings['type'], - web_connectivity_settings['address'] - )) - continue - reachable = yield web_connectivity_test_helper.isReachable() - if not reachable: - log.err("Unreachable %s web_connectivity test helper %s" % ( + web_connectivity_test_helper = WebConnectivityClient( + settings=web_connectivity_settings) + if not web_connectivity_test_helper.isSupported(): + log.err("Unsupported %s web_connectivity test_helper " + "%s" % ( web_connectivity_settings['type'], web_connectivity_settings['address'] - )) - continue - defer.returnValue(web_connectivity_settings) - except e.TestHelperUnreachable: + )) + continue + reachable = yield web_connectivity_test_helper.isReachable() + if not reachable: log.err("Unreachable %s web_connectivity test helper %s" % ( web_connectivity_settings['type'], web_connectivity_settings['address'] )) continue + defer.returnValue(web_connectivity_settings) raise e.NoReachableTestHelpers else: defer.returnValue(test_helper_address.encode('ascii')) diff --git a/ooni/errors.py b/ooni/errors.py index e5f26b2..8197f34 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -296,10 +296,9 @@ class LibraryNotInstalledError(Exception): pass

-class InsecureCollector(Exception): +class InsecureBackend(Exception): pass

- class CollectorUnsupported(Exception): pass

@@ -307,10 +306,6 @@ class HTTPSCollectorUnsupported(CollectorUnsupported): pass

-class CollectorUnreachable(Exception): - pass - - class BackendNotSupported(Exception): pass

@@ -319,9 +314,5 @@ class NoReachableCollectors(Exception): pass

-class TestHelperUnreachable(Exception): - pass - - class NoReachableTestHelpers(Exception): pass diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index da25bdc..97cde60 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -82,6 +82,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): 'backend': 'web-connectivity', 'dns-discovery': 'dns-discovery' } + requiredOptions = ['backend', 'dns-discovery'] requiresRoot = False requiresTor = False followRedirects = True diff --git a/ooni/oonicli.py b/ooni/oonicli.py index ca8774f..47c26b2 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -279,10 +279,10 @@ def createDeck(global_options, url=None): except errors.HTTPSCollectorUnsupported: log.err("HTTPS collectors require a twisted version of at least 14.0.2.") sys.exit(6) - except errors.InsecureCollector: + except errors.InsecureBackend: log.err("Attempting to report to an insecure collector.") log.err("To enable reporting to insecure collector set the " - "advanced->insecure_collector option to true in " + "advanced->insecure_backend option to true in " "your ooniprobe.conf file.") sys.exit(7) except Exception as e: