[tor-commits] [oonib/master] Implement policy aware bouncing support in the backend
art at torproject.org
art at torproject.org
Fri Aug 29 09:59:23 UTC 2014
commit fdc0a4460cb3df9ec7ab55669e71f2008e0ee322
Author: kudrom <kudrom at riseup.net>
Date: Mon Aug 25 17:19:38 2014 +0200
Implement policy aware bouncing support in the backend
---
data/bouncer.yaml | 17 +++++++-
oonib/bouncer/handlers.py | 102 +++++++++++++++++++++++++++++++++++----------
oonib/errors.py | 10 +++++
oonib/handlers.py | 3 +-
4 files changed, 108 insertions(+), 24 deletions(-)
diff --git a/data/bouncer.yaml b/data/bouncer.yaml
index 9ab2dc0..acf2a9a 100644
--- a/data/bouncer.yaml
+++ b/data/bouncer.yaml
@@ -1,3 +1,18 @@
collector:
httpo://ihiderha53f36lsd.onion:
- test-helper: {dns: '93.95.227.200:57004', http-return-json-headers: 'http://93.95.227.200', ssl: 'https://93.95.227.20', tcp-echo: '213.138.109.232', traceroute: '93.95.227.200'}
+ policy:
+ input:
+ - {id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1}
+ - {id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290}
+ nettest:
+ - {name: dns_consistency, version: '0.5'}
+ - {name: http_requests_test, version: 0.2.3}
+ - {name: tcp_connect, version: '0.1'}
+ - {name: captivep, version: '0.2'}
+ - {name: daphn3, version: '0.1'}
+ - {name: dns_spoof, version: '0.2'}
+ - {name: http_header_field_manipulation, version: 0.1.3}
+ - {name: http_invalid_request_line, version: 0.1.4}
+ - {name: multiprotocol_traceroute_test, version: 0.1.1}
+ test-helper: {dns: '93.95.227.200:57004', http-return-json-headers: 'http://93.95.227.200',
+ ssl: 'https://93.95.227.20', tcp-echo: 213.138.109.232, traceroute: 93.95.227.200}
diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py
index 81d13f3..c49338e 100644
--- a/oonib/bouncer/handlers.py
+++ b/oonib/bouncer/handlers.py
@@ -7,24 +7,30 @@ from oonib.config import config
class Bouncer(object):
- def __init__(self):
- with open(config.main.bouncer_file) as f:
- bouncerFile = yaml.safe_load(f)
- self.updateKnownHelpers(bouncerFile)
- self.updateKnownCollectors(bouncerFile)
- def updateKnownCollectors(self, bouncerFile):
+ def __init__(self, bouncer_file):
+ with open(bouncer_file) as f:
+ self.bouncerFile = yaml.safe_load(f)
+ self.updateKnownHelpers()
+ self.updateKnownCollectors()
+
+ def updateKnownCollectors(self):
"""
Initialize the list of all known collectors
"""
- self.knownCollectors = []
- for collectorName, helpers in bouncerFile['collector'].items():
- if collectorName not in self.knownCollectors:
- self.knownCollectors.append(collectorName)
-
- def updateKnownHelpers(self, bouncerFile):
+ self.knownCollectorsWithPolicy = []
+ self.knownCollectorsWithoutPolicy = []
+ for collectorName, content in self.bouncerFile['collector'].items():
+ if content.get('policy') is not None and \
+ collectorName not in self.knownCollectorsWithPolicy:
+ self.knownCollectorsWithPolicy.append(collectorName)
+ elif content.get('policy') is None and \
+ collectorName not in self.knownCollectorsWithoutPolicy:
+ self.knownCollectorsWithoutPolicy.append(collectorName)
+
+ def updateKnownHelpers(self):
self.knownHelpers = {}
- for collectorName, helpers in bouncerFile['collector'].items():
+ for collectorName, helpers in self.bouncerFile['collector'].items():
for helperName, helperAddress in helpers['test-helper'].items():
if helperName not in self.knownHelpers.keys():
self.knownHelpers[helperName] = []
@@ -106,14 +112,63 @@ class Bouncer(object):
response = {'error': 'test-helper-not-found'}
return response
- response['default'] = {'collector':
- random.choice(self.knownCollectors)}
+ if len(self.knownCollectorsWithoutPolicy) > 0:
+ default_collector = random.choice(
+ self.knownCollectorsWithoutPolicy)
+ else:
+ default_collector = None
+ response['default'] = {'collector': default_collector}
return response
+ def collectorAccepting(self, net_test_name, input_hashes, test_helpers):
+ for collector_address in self.knownCollectorsWithPolicy:
+ collector = self.bouncerFile['collector'][collector_address]
+ supported_net_tests = [x['name'] for x in collector['policy']['nettest']]
+ supported_input_hashes = [x['id'] for x in collector['policy']['input']]
+ if net_test_name not in supported_net_tests:
+ continue
+ if any([input_hash not in supported_input_hashes for input_hash in input_hashes]):
+ continue
+ if all([x in collector['test-helper'].keys() for x in test_helpers]):
+ return collector_address
+ if len(self.knownCollectorsWithoutPolicy) > 0:
+ return random.choice(self.knownCollectorsWithoutPolicy)
+ else:
+ raise e.CollectorNotFound
+
+ def filterByNetTests(self, requested_nettests):
+ """
+ Here we will return a list containing test helpers and collectors for
+ the required nettests.
+ We give favour to the collectors that have a stricter policy and if
+ those fail we will resort to using the collectors with a more lax
+ policy.
+ """
+ nettests = []
+ for requested_nettest in requested_nettests:
+ collector = self.collectorAccepting(
+ requested_nettest['name'],
+ requested_nettest['input-hashes'],
+ requested_nettest['test-helpers'])
+ test_helpers = {}
+ for test_helper in requested_nettest['test-helpers']:
+ test_helpers[test_helper] = self.bouncerFile['collector'][collector]['test-helper'][test_helper]
+
+ nettest = {
+ 'name': requested_nettest['name'],
+ 'version': requested_nettest['version'],
+ 'input-hashes': requested_nettest['input-hashes'],
+ 'test-helpers': test_helpers,
+ 'collector': collector,
+ }
+ nettests.append(nettest)
+ return {'net-tests': nettests}
+
class BouncerQueryHandler(OONIBHandler):
+
def initialize(self):
- self.bouncer = Bouncer()
+ self.bouncer = Bouncer(config.main.bouncer_file)
def post(self):
try:
@@ -121,13 +176,16 @@ class BouncerQueryHandler(OONIBHandler):
except ValueError:
raise e.InvalidRequest
- try:
+ if 'test-helpers' in query:
requested_helpers = query['test-helpers']
- except KeyError:
- raise e.TestHelpersKeyMissing
+ if not isinstance(requested_helpers, list):
+ raise e.InvalidRequest
+ response = self.bouncer.filterHelperAddresses(requested_helpers)
- if not isinstance(requested_helpers, list):
- raise e.InvalidRequest
+ elif 'net-tests' in query:
+ response = self.bouncer.filterByNetTests(query['net-tests'])
+
+ else:
+ raise e.TestHelpersOrNetTestsKeyMissing
- response = self.bouncer.filterHelperAddresses(requested_helpers)
self.write(response)
diff --git a/oonib/errors.py b/oonib/errors.py
index 3f1167a..e20d471 100644
--- a/oonib/errors.py
+++ b/oonib/errors.py
@@ -83,6 +83,11 @@ class ReportNotFound(OONIBError):
log_message = "report-not-found"
+class CollectorNotFound(OONIBError):
+ status_code = 404
+ log_message = "collector-not-found"
+
+
class NoValidCollector(OONIBError):
pass
@@ -92,6 +97,11 @@ class TestHelpersKeyMissing(OONIBError):
log_message = "test-helpers-key-missing"
+class TestHelpersOrNetTestsKeyMissing(OONIBError):
+ status_code = 400
+ log_message = "test-helpers-or-net-test-key-missing"
+
+
class TestHelperNotFound(OONIBError):
status_code = 404
log_message = "test-helper-not-found"
diff --git a/oonib/handlers.py b/oonib/handlers.py
index bbf8ade..1417cc5 100644
--- a/oonib/handlers.py
+++ b/oonib/handlers.py
@@ -7,6 +7,7 @@ from oonib import log
class OONIBHandler(web.RequestHandler):
+
def write_error(self, status_code, exception=None, **kw):
self.set_status(status_code)
if hasattr(exception, 'log_message') and exception.log_message is not None:
@@ -14,7 +15,7 @@ class OONIBHandler(web.RequestHandler):
elif 400 <= status_code < 600:
self.write({'error': status_code})
else:
- log.error(exception)
+ log.exception(exception)
self.write({'error': 'error'})
def write(self, chunk):
More information about the tor-commits
mailing list