commit cde886b1facaab9a646ccaa7c3f942ca542fd177 Author: Arturo Filastò art@fuffa.org Date: Mon Aug 19 16:45:18 2013 +0200
Return the test helper address of the requested TH
* Refactor bouncer into a separate object --- README.md | 2 +- data/bouncer.yaml | 2 ++ oonib.conf.example | 35 ++++++++++-------- oonib/bouncer/handlers.py | 86 +++++++++++++++++++++++++++++++++++---------- oonib/deck/handlers.py | 2 ++ oonib/errors.py | 18 ++++++++++ oonib/oonibackend.py | 36 +++++++++---------- oonib/report/handlers.py | 24 +++++++++++-- 8 files changed, 150 insertions(+), 55 deletions(-)
diff --git a/README.md b/README.md index 75b60a9..b2d13b9 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ The following iptables commands will map connections on low ports to those bound by oonib:
``` -# Map port 80 to config.helpers.http_return_request.port (default: 57001) +# Map port 80 to config.helpers['http-return-json-headers'].port (default: 57001) iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 57001 # Map port 443 to config.helpers.ssl.port (default: 57006) iptables -t nat -A PREROUTING -p tcp -m tcp --dport 443 -j REDIRECT --to-ports 57006 diff --git a/data/bouncer.yaml b/data/bouncer.yaml index ba54898..7709e40 100644 --- a/data/bouncer.yaml +++ b/data/bouncer.yaml @@ -1,3 +1,5 @@ collector: httpo://nkvphnp3p6agi5qq.onion: test-helper: {dns: 93.95.227.200, http-return-json-headers: 'http://93.95.227.200%27%7D + httpo://nkvphnp3p6agi5q1.onion: + test-helper: {dns: 93.22.227.200, http-return-json-headers: 'http://93.95.227.220%27%7D diff --git a/oonib.conf.example b/oonib.conf.example index 977b5dc..323b2af 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -6,48 +6,53 @@ main: policy_file: data/policy.yaml bouncer_file: data/bouncer.yaml
- logfile: Null - tor_datadir: Null + logfile: null + tor_datadir: null database_uri: 'sqlite://oonib_test_db.db' db_threadpool_size: 10 - tor_binary: Null + tor_binary: null socks_port: 9055 tor2webmode: false pidfile: 'oonib.pid' nodaemon: true - originalname: Null - chroot: Null + originalname: null + chroot: null rundir: . - umask: Null - euid: Null - uid: Null - gid: Null - uuid: Null + umask: null + euid: null + uid: null + gid: null + uuid: null no_save: true - profile: Null + profile: null debug: false stale_time: 3600
tor_hidden_service: true
helpers: - http_return_request: + http-return-json-headers: + address: null port: 57001 server_version: Apache
- tcp_echo: + tcp-echo: + address: null port: 57002
daphn3: - yaml_file: Null - pcap_file: Null + address: null + yaml_file: null + pcap_file: null port: 57003
dns: + address: null udp_port: 57004 tcp_port: 57005
ssl: + address: null private_key: 'private.key' certificate: 'certificate.crt' port: 57006 diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 230f618..44f1c63 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -4,10 +4,72 @@ import yaml from oonib.handlers import OONIBHandler from oonib import config
-class BouncerQueryHandler(OONIBHandler): - def initialize(self): +class Bouncer(object): + def __init__(self): self.knownHelpers = {} self.updateKnownHelpers() + + def updateKnownHelpers(self): + with open(config.main.bouncer_file) as f: + bouncerFile = yaml.safe_load(f) + for collectorName, helpers in bouncerFile['collector'].items(): + for helperName, helperAddress in helpers['test-helper'].items(): + if helperName not in self.knownHelpers.keys(): + self.knownHelpers[helperName] = [] + + self.knownHelpers[helperName].append({ + 'collector-name': collectorName, + 'helper-address': helperAddress + }) + + def getHelperAddresses(self, helper_name): + """ + Returns a dict keyed on the collector address of known test helpers. + example: + { + 'httpo://thirteenchars1.onion': '127.0.0.1', + 'httpo://thirteenchars2.onion': '127.0.0.2', + 'httpo://thirteenchars3.onion': '127.0.0.3' + } + """ + try: + helpers = self.knownHelpers[helper_name] + except KeyError: + raise e.NoHelperFound + + helpers_dict = {} + for helper in helpers: + helpers_dict[helper['collector-name']] = helper['helper-address'] + + return helpers_dict + + def filterHelperAddresses(self, requested_helpers): + """ + Returns a dict of collectors that support all the requested_helpers. + + Example: + requested_helpers = ['a', 'b', 'c'] + will return: + { + 'httpo://thirteenchars1.onion': { + 'a': '127.0.0.1', + 'b': 'http://127.0.0.1', + 'c': '127.0.0.1:590', + } + } + + """ + result = {} + for helper_name in requested_helpers: + for collector, helper_address in self.getHelperAddresses(helper_name).items(): + if collector not in result.keys(): + result[collector] = {} + result[collector][helper_name] = helper_address + return result + +class BouncerQueryHandler(OONIBHandler): + def initialize(self): + self.bouncer = Bouncer()
def updateKnownHelpers(self): with open(config.main.bouncer_file) as f: @@ -26,25 +88,13 @@ class BouncerQueryHandler(OONIBHandler): try: query = json.loads(self.request.body) except ValueError: - self.set_status(400) - self.write(json.dumps({'error': 'invalid-request'})) - return + raise e.InvalidRequest
try: - helpers = query['test-helpers'] + requested_helpers = query['test-helpers'] except KeyError: - self.set_status(400) - self.write(json.dumps({'error': 'test-helpers-key-missing'})) - return + raise e.TestHelpersKeyMissing
response = {} - response['collector'] = {} - for helperName in helpers: - if helperName in self.knownHelpers.keys(): - chosen = random.choice(self.knownHelpers[helperName]) - collectorName, helperAddress = chosen['collector-name'], chosen['helper-address'] - if not collectorName in response['collector'].keys(): - response['collector'][collectorName] = {'test-helper': {}} - response['collector'][collectorName]['test-helper'][helperName] = helperAddress - + response['collector'] = self.bouncer.filterHelperAddresses(requested_helpers) self.write(response) diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index d4b3bbe..558ca6a 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -20,10 +20,12 @@ class DeckDescHandler(OONIBHandler): for k in ['name', 'description', 'version', 'author', 'date']: response[k] = deckDesc[k] self.write(response) + except IOError: log.err("Deck %s missing" % deckID) self.set_status(404) self.write({'error': 'missing-deck'}) + except KeyError: self.set_status(400) log.err("Deck %s missing required keys!" % deckID) diff --git a/oonib/errors.py b/oonib/errors.py index 24a06fc..1dc8539 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -3,6 +3,13 @@ from cyclone.web import HTTPError class OONIBError(HTTPError): pass
+class InvalidRequest(OONIBError): + pass + +class NoHelperFound(OONIBError): + status_code = 404 + log_message = 'no-helper-found' + class InvalidInputHash(OONIBError): status_code = 400 log_message = 'invalid-input-hash' @@ -38,3 +45,14 @@ class InvalidReportHeader(OONIBError): class ReportNotFound(OONIBError): status_code = 404 log_message = "report-not-found" + +class NoValidCollector(OONIBError): + pass + +class TestHelpersKeyMissing(OONIBError): + status_code = 400 + log_message = "test-helpers-key-missing" + +class TestHelperNotFound(OONIBError): + status_code = 400 + log_message = "test-helper-not-found" diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index afd961f..c5f96ee 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -30,45 +30,45 @@ else:
serviceCollection = service.IServiceCollection(application)
-if config.helpers.ssl.port: - print "Starting SSL helper on %s" % config.helpers.ssl.port - ssl_helper = internet.SSLServer(int(config.helpers.ssl.port), +if config.helpers['ssl'].port: + print "Starting SSL helper on %s" % config.helpers['ssl'].port + ssl_helper = internet.SSLServer(int(config.helpers['ssl'].port), http_helpers.HTTPReturnJSONHeadersHelper(), ssl_helpers.SSLContext(config)) ssl_helper.setServiceParent(serviceCollection)
# Start the DNS Server related services -if config.helpers.dns.tcp_port: - print "Starting TCP DNS Helper on %s" % config.helpers.dns.tcp_port - tcp_dns_helper = internet.TCPServer(int(config.helpers.dns.tcp_port), +if config.helpers['dns'].tcp_port: + print "Starting TCP DNS Helper on %s" % config.helpers['dns'].tcp_port + tcp_dns_helper = internet.TCPServer(int(config.helpers['dns'].tcp_port), dns_helpers.DNSTestHelper()) tcp_dns_helper.setServiceParent(serviceCollection)
-if config.helpers.dns.udp_port: - print "Starting UDP DNS Helper on %s" % config.helpers.dns.udp_port +if config.helpers['dns'].udp_port: + print "Starting UDP DNS Helper on %s" % config.helpers['dns'].udp_port udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper()) - udp_dns_helper = internet.UDPServer(int(config.helpers.dns.udp_port), + udp_dns_helper = internet.UDPServer(int(config.helpers['dns'].udp_port), udp_dns_factory) udp_dns_helper.setServiceParent(serviceCollection)
# XXX this needs to be ported # Start the OONI daphn3 backend -if config.helpers.daphn3.port: - print "Starting Daphn3 helper on %s" % config.helpers.daphn3.port - daphn3_helper = internet.TCPServer(int(config.helpers.daphn3.port), +if config.helpers['daphn3'].port: + print "Starting Daphn3 helper on %s" % config.helpers['daphn3'].port + daphn3_helper = internet.TCPServer(int(config.helpers['daphn3'].port), tcp_helpers.Daphn3Server()) daphn3_helper.setServiceParent(serviceCollection)
-if config.helpers.tcp_echo.port: - print "Starting TCP echo helper on %s" % config.helpers.tcp_echo.port - tcp_echo_helper = internet.TCPServer(int(config.helpers.tcp_echo.port), +if config.helpers['tcp-echo'].port: + print "Starting TCP echo helper on %s" % config.helpers['tcp-echo'].port + tcp_echo_helper = internet.TCPServer(int(config.helpers['tcp-echo'].port), tcp_helpers.TCPEchoHelper()) tcp_echo_helper.setServiceParent(serviceCollection)
-if config.helpers.http_return_request.port: - print "Starting HTTP return request helper on %s" % config.helpers.http_return_request.port +if config.helpers['http-return-json-headers'].port: + print "Starting HTTP return request helper on %s" % config.helpers['http-return-json-headers'].port http_return_request_helper = internet.TCPServer( - int(config.helpers.http_return_request.port), + int(config.helpers['http-return-json-headers'].port), http_helpers.HTTPReturnJSONHeadersHelper()) http_return_request_helper.setServiceParent(serviceCollection) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 75ec55d..2441ddf 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -70,6 +70,13 @@ def parseNewReportRequest(request): continue else: raise InvalidRequestField(k) + + try: + test_helper = parsed_request['test_helper'] + if not re.match(regexp, str(test_helper)): + raise InvalidRequestField('test_helper') + except KeyError: + pass
return parsed_request
@@ -127,7 +134,6 @@ class NewReportHandlerFile(OONIBHandler): policy = Policy() policy.validateInputHash(self.inputHash) policy.validateNettest(self.testName) - # XXX add support for version checking too.
def post(self): """ @@ -219,9 +225,21 @@ class NewReportHandlerFile(OONIBHandler): # random nonce report_filename = os.path.join(config.main.report_dir, report_id)
- response = {'backend_version': config.backend_version, - 'report_id': report_id + response = { + 'backend_version': config.backend_version, + 'report_id': report_id } + + try: + requested_helper = report_data['test_helper'] + except KeyError: + pass + + if requested_helper: + try: + response['test_helper_address'] = config.helpers[requested_helper].address + except KeyError: + raise e.TestHelperNotFound
config.reports[report_id] = time.time()