commit cd5992e9bc2877b1105a1584a168136de83148b3 Author: kudrom kudrom@riseup.net Date: Mon Aug 25 17:19:38 2014 +0200

Implement policy aware bouncing support in the backend --- data/bouncer.yaml | 17 +++++++- oonib/bouncer/handlers.py | 102 +++++++++++++++++++++++++++++++++++---------- oonib/errors.py | 10 +++++ oonib/handlers.py | 3 +- 4 files changed, 108 insertions(+), 24 deletions(-)

diff --git a/data/bouncer.yaml b/data/bouncer.yaml index 9ab2dc0..acf2a9a 100644 --- a/data/bouncer.yaml +++ b/data/bouncer.yaml @@ -1,3 +1,18 @@ collector: httpo://ihiderha53f36lsd.onion: - test-helper: {dns: '93.95.227.200:57004', http-return-json-headers: 'http://93.95.227.200', ssl: 'https://93.95.227.20', tcp-echo: '213.138.109.232', traceroute: '93.95.227.200'} + policy: + input: + - {id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1} + - {id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290} + nettest: + - {name: dns_consistency, version: '0.5'} + - {name: http_requests_test, version: 0.2.3} + - {name: tcp_connect, version: '0.1'} + - {name: captivep, version: '0.2'} + - {name: daphn3, version: '0.1'} + - {name: dns_spoof, version: '0.2'} + - {name: http_header_field_manipulation, version: 0.1.3} + - {name: http_invalid_request_line, version: 0.1.4} + - {name: multiprotocol_traceroute_test, version: 0.1.1} + test-helper: {dns: '93.95.227.200:57004', http-return-json-headers: 'http://93.95.227.200', + ssl: 'https://93.95.227.20', tcp-echo: 213.138.109.232, traceroute: 93.95.227.200} diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 81d13f3..c49338e 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -7,24 +7,30 @@ from oonib.config import config

class Bouncer(object): - def __init__(self): - with open(config.main.bouncer_file) as f: - bouncerFile = yaml.safe_load(f) - self.updateKnownHelpers(bouncerFile) - self.updateKnownCollectors(bouncerFile)

- def updateKnownCollectors(self, bouncerFile): + def __init__(self, bouncer_file): + with open(bouncer_file) as f: + self.bouncerFile = yaml.safe_load(f) + self.updateKnownHelpers() + self.updateKnownCollectors() + + def updateKnownCollectors(self): """ Initialize the list of all known collectors """ - self.knownCollectors = [] - for collectorName, helpers in bouncerFile['collector'].items(): - if collectorName not in self.knownCollectors: - self.knownCollectors.append(collectorName) - - def updateKnownHelpers(self, bouncerFile): + self.knownCollectorsWithPolicy = [] + self.knownCollectorsWithoutPolicy = [] + for collectorName, content in self.bouncerFile['collector'].items(): + if content.get('policy') is not None and \ + collectorName not in self.knownCollectorsWithPolicy: + self.knownCollectorsWithPolicy.append(collectorName) + elif content.get('policy') is None and \ + collectorName not in self.knownCollectorsWithoutPolicy: + self.knownCollectorsWithoutPolicy.append(collectorName) + + def updateKnownHelpers(self): self.knownHelpers = {} - for collectorName, helpers in bouncerFile['collector'].items(): + for collectorName, helpers in self.bouncerFile['collector'].items(): for helperName, helperAddress in helpers['test-helper'].items(): if helperName not in self.knownHelpers.keys(): self.knownHelpers[helperName] = [] @@ -106,14 +112,63 @@ class Bouncer(object): response = {'error': 'test-helper-not-found'} return response

- response['default'] = {'collector': - random.choice(self.knownCollectors)} + if len(self.knownCollectorsWithoutPolicy) > 0: + default_collector = random.choice( + self.knownCollectorsWithoutPolicy) + else: + default_collector = None + response['default'] = {'collector': default_collector} return response

+ def collectorAccepting(self, net_test_name, input_hashes, test_helpers): + for collector_address in self.knownCollectorsWithPolicy: + collector = self.bouncerFile['collector'][collector_address] + supported_net_tests = [x['name'] for x in collector['policy']['nettest']] + supported_input_hashes = [x['id'] for x in collector['policy']['input']] + if net_test_name not in supported_net_tests: + continue + if any([input_hash not in supported_input_hashes for input_hash in input_hashes]): + continue + if all([x in collector['test-helper'].keys() for x in test_helpers]): + return collector_address + if len(self.knownCollectorsWithoutPolicy) > 0: + return random.choice(self.knownCollectorsWithoutPolicy) + else: + raise e.CollectorNotFound + + def filterByNetTests(self, requested_nettests): + """ + Here we will return a list containing test helpers and collectors for + the required nettests. + We give favour to the collectors that have a stricter policy and if + those fail we will resort to using the collectors with a more lax + policy. + """ + nettests = [] + for requested_nettest in requested_nettests: + collector = self.collectorAccepting( + requested_nettest['name'], + requested_nettest['input-hashes'], + requested_nettest['test-helpers']) + test_helpers = {} + for test_helper in requested_nettest['test-helpers']: + test_helpers[test_helper] = self.bouncerFile['collector'][collector]['test-helper'][test_helper] + + nettest = { + 'name': requested_nettest['name'], + 'version': requested_nettest['version'], + 'input-hashes': requested_nettest['input-hashes'], + 'test-helpers': test_helpers, + 'collector': collector, + } + nettests.append(nettest) + return {'net-tests': nettests} +

class BouncerQueryHandler(OONIBHandler): + def initialize(self): - self.bouncer = Bouncer() + self.bouncer = Bouncer(config.main.bouncer_file)

def post(self): try: @@ -121,13 +176,16 @@ class BouncerQueryHandler(OONIBHandler): except ValueError: raise e.InvalidRequest

- try: + if 'test-helpers' in query: requested_helpers = query['test-helpers'] - except KeyError: - raise e.TestHelpersKeyMissing + if not isinstance(requested_helpers, list): + raise e.InvalidRequest + response = self.bouncer.filterHelperAddresses(requested_helpers)

- if not isinstance(requested_helpers, list): - raise e.InvalidRequest + elif 'net-tests' in query: + response = self.bouncer.filterByNetTests(query['net-tests']) + + else: + raise e.TestHelpersOrNetTestsKeyMissing

- response = self.bouncer.filterHelperAddresses(requested_helpers) self.write(response) diff --git a/oonib/errors.py b/oonib/errors.py index 3f1167a..e20d471 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -83,6 +83,11 @@ class ReportNotFound(OONIBError): log_message = "report-not-found"

+class CollectorNotFound(OONIBError): + status_code = 404 + log_message = "collector-not-found" + + class NoValidCollector(OONIBError): pass

@@ -92,6 +97,11 @@ class TestHelpersKeyMissing(OONIBError): log_message = "test-helpers-key-missing"

+class TestHelpersOrNetTestsKeyMissing(OONIBError): + status_code = 400 + log_message = "test-helpers-or-net-test-key-missing" + + class TestHelperNotFound(OONIBError): status_code = 404 log_message = "test-helper-not-found" diff --git a/oonib/handlers.py b/oonib/handlers.py index bbf8ade..1417cc5 100644 --- a/oonib/handlers.py +++ b/oonib/handlers.py @@ -7,6 +7,7 @@ from oonib import log

class OONIBHandler(web.RequestHandler): + def write_error(self, status_code, exception=None, **kw): self.set_status(status_code) if hasattr(exception, 'log_message') and exception.log_message is not None: @@ -14,7 +15,7 @@ class OONIBHandler(web.RequestHandler): elif 400 <= status_code < 600: self.write({'error': status_code}) else: - log.error(exception) + log.exception(exception) self.write({'error': 'error'})

def write(self, chunk):