September 2013 - tor-commits - lists.torproject.org

[oonib/master] Add proper error handling in deck & input & nettest & policy
by art＠torproject.org 11 Sep '13

11 Sep '13

commit f2889515ec044dbaaa5b5795ffca00cd5020aff9 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 19 14:20:57 2013 +0200 Add proper error handling in deck & input & nettest & policy * Refactoring of handler related code --- oonib/api.py | 2 -- oonib/bouncer/handlers.py | 4 ---- oonib/deck/handlers.py | 38 +++++++++++++++++++++++++------------- oonib/input/handlers.py | 34 +++++++++++++++++++--------------- oonib/nettest/handlers.py | 20 +++++++++++--------- oonib/policy/handlers.py | 4 ++-- 6 files changed, 57 insertions(+), 45 deletions(-) diff --git a/oonib/api.py b/oonib/api.py index 384b949..622df4d 100644 --- a/oonib/api.py +++ b/oonib/api.py @@ -23,6 +23,4 @@ if config.main.policy_file: if config.main.bouncer_file: ooniBackendAPI += bouncerAPI -print ooniBackendAPI - ooniBackend = web.Application(ooniBackendAPI, debug=True) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index bc81ddb..230f618 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -22,10 +22,6 @@ class BouncerQueryHandler(OONIBHandler): 'helper-address': helperAddress }) - def get(self): - #XXX unused - pass - def post(self): try: query = json.loads(self.request.body) diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index 1b67a3f..d4b3bbe 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -9,30 +9,42 @@ from oonib import config, log class DeckDescHandler(OONIBHandler): def get(self, deckID): + # note: + # we don't have to sanitize deckID, because it's already checked + # against matching a certain pattern in the handler. bn = os.path.basename(deckID) try: - f = open(os.path.join(config.main.deck_dir, bn)) - a = {} - deckDesc = yaml.safe_load(f) - for k in ['name', 'description', 'version', 'author', 'date']: - a[k] = deckDesc[k] - self.write(json.dumps(a)) + with open(os.path.join(config.main.deck_dir, bn)) as f: + response = {} + deckDesc = yaml.safe_load(f) + for k in ['name', 'description', 'version', 'author', 'date']: + response[k] = deckDesc[k] + self.write(response) except IOError: log.err("Deck %s missing" % deckID) + self.set_status(404) + self.write({'error': 'missing-deck'}) except KeyError: + self.set_status(400) log.err("Deck %s missing required keys!" % deckID) + self.write({'error': 'missing-deck-keys'}) class DeckListHandler(OONIBHandler): def get(self): - if not config.main.deck_dir: return + if not config.main.deck_dir: + self.set_status(501) + self.write({'error': 'no-decks-configured'}) + return path = os.path.abspath(config.main.deck_dir) + "/*" decknames = map(os.path.basename, glob.iglob(path)) decknames = filter(lambda y: re.match("[a-z0-9]{40}", y), decknames) deckList = [] for deckname in decknames: - f = open(os.path.join(config.main.deck_dir, deckname)) - d = yaml.safe_load(f) - deckList.append({'id': deckname,'name': d['name'], - 'description': d['description']}) - f.close() - self.write(json.dumps(deckList)) + with open(os.path.join(config.main.deck_dir, deckname)) as f: + d = yaml.safe_load(f) + deckList.append({ + 'id': deckname, + 'name': d['name'], + 'description': d['description'] + }) + self.write(deckList) diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py index 8db1482..ed4c6b2 100644 --- a/oonib/input/handlers.py +++ b/oonib/input/handlers.py @@ -10,16 +10,21 @@ class InputDescHandler(OONIBHandler): def get(self, inputID): bn = os.path.basename(inputID) + ".desc" try: - f = open(os.path.join(config.main.input_dir, bn)) - a = {} - inputDesc = yaml.safe_load(f) - for k in ['name', 'description', 'version', 'author', 'date']: - a[k] = inputDesc[k] - self.write(json.dumps(a)) + with open(os.path.join(config.main.input_dir, bn)) as f: + response = {} + inputDesc = yaml.safe_load(f) + for k in ['name', 'description', 'version', 'author', 'date']: + response[k] = inputDesc[k] + self.write(response) except IOError: log.err("No Input Descriptor found for id %s" % inputID) + self.set_status(404) + self.write({'error': 'missing-input'}) + except Exception, e: log.err("Invalid Input Descriptor found for id %s" % inputID) + self.set_status(500) + self.write({'error': 'invalid-input-descriptor'}) class InputListHandler(OONIBHandler): def get(self): @@ -27,12 +32,11 @@ class InputListHandler(OONIBHandler): inputnames = map(os.path.basename, glob.iglob(path)) inputList = [] for inputname in inputnames: - f = open(os.path.join(config.main.input_dir, inputname)) - d = yaml.safe_load(f) - inputList.append({ - 'id': inputname, - 'name': d['name'], - 'description': d['description'] - }) - f.close() - self.write(json.dumps(inputList)) + with open(os.path.join(config.main.input_dir, inputname)) as f: + d = yaml.safe_load(f) + inputList.append({ + 'id': inputname, + 'name': d['name'], + 'description': d['description'] + }) + self.write(inputList) diff --git a/oonib/nettest/handlers.py b/oonib/nettest/handlers.py index 17d5590..b3ceed4 100644 --- a/oonib/nettest/handlers.py +++ b/oonib/nettest/handlers.py @@ -10,16 +10,18 @@ class NetTestDescHandler(OONIBHandler): def get(self, netTestID): bn = os.path.basename(netTestID) + ".desc" try: - f = open(os.path.join(config.main.nettest_dir, bn)) - a = {} - netTestDesc = yaml.safe_load(f) - for k in ['name', 'description', 'version', 'author', 'date']: - a[k] = netTestDesc[k] - self.write(json.dumps(a)) + with open(os.path.join(config.main.nettest_dir, bn)) as f: + response = {} + netTestDesc = yaml.safe_load(f) + for k in ['name', 'description', 'version', 'author', 'date']: + response[k] = netTestDesc[k] + self.write(response) except IOError: log.err("No NetTest Descriptor found for id %s" % netTestID) + self.set_status(404) + self.write({'error': 'missing-nettest'}) + except Exception, e: log.err("Invalid NetTest Descriptor found for id %s" % netTestID) - - - + self.set_status(500) + self.write({'error': 'invalid-nettest-descriptor'}) diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index f72e746..788e578 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -12,7 +12,7 @@ class NetTestPolicyHandler(OONIBHandler): """ with open(config.main.policy_file) as f: p = yaml.safe_load(f) - self.write(json.dumps(p['nettest'])) + self.write(p['nettest']) class InputPolicyHandler(OONIBHandler): def get(self): @@ -21,4 +21,4 @@ class InputPolicyHandler(OONIBHandler): """ with open(config.main.policy_file) as f: p = yaml.safe_load(f) - self.write(json.dumps(p['input'])) + self.write(p['input'])

1 0

[oonib/master] Add support for serializing lists to json via self.write()
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 1e0d3c0d0b20f92fd901163a4f2b41627f9e931e Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 19 14:03:42 2013 +0200 Add support for serializing lists to json via self.write() --- oonib/handlers.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/oonib/handlers.py b/oonib/handlers.py index 62496ce..515e8c2 100644 --- a/oonib/handlers.py +++ b/oonib/handlers.py @@ -1,7 +1,20 @@ +import types + +from cyclone import escape from cyclone import web class OONIBHandler(web.RequestHandler): - pass + def write(self, chunk): + """ + This is a monkey patch to RequestHandler to allow us to serialize also + json list objects. + """ + if isinstance(chunk, types.ListType): + chunk = escape.json_encode(chunk) + web.RequestHandler.write(self, chunk) + self.set_header("Content-Type", "application/json") + else: + web.RequestHandler.write(self, chunk) class OONIBError(web.HTTPError): pass

1 0

[oonib/master] Perform some changes to the API and fix some bugs.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 8c17eb22d34ccefca1679e2942c8ec150509e6be Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 23 16:46:53 2013 +0200 Perform some changes to the API and fix some bugs. --- oonib/bouncer/handlers.py | 33 +++++++++++++++++++++++++-------- oonib/errors.py | 3 ++- oonib/input/handlers.py | 1 + oonib/report/handlers.py | 5 +++-- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 44f1c63..e669e8e 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -1,6 +1,7 @@ import json import random import yaml +from oonib import errors as e from oonib.handlers import OONIBHandler from oonib import config @@ -51,13 +52,18 @@ class Bouncer(object): requested_helpers = ['a', 'b', 'c'] will return: { - 'httpo://thirteenchars1.onion': { - 'a': '127.0.0.1', - 'b': 'http://127.0.0.1', - 'c': '127.0.0.1:590', - } + 'a': '127.0.0.1', + 'b': 'http://127.0.0.1', + 'c': '127.0.0.1:590', + 'collector': 'httpo://thirteenchars1.onion' } + or + + {} + + if no valid helper was found + """ result = {} for helper_name in requested_helpers: @@ -65,7 +71,19 @@ class Bouncer(object): if collector not in result.keys(): result[collector] = {} result[collector][helper_name] = helper_address - return result + + helper_list = [] + for collector, helpers in result.items(): + if len(helpers) == len(requested_helpers): + valid_helpers = helpers + valid_helpers['collector'] = collector + helper_list.append(valid_helpers) + else: + continue + if len(helper_list) == 0: + return {} + else: + return random.choice(helper_list) class BouncerQueryHandler(OONIBHandler): def initialize(self): @@ -95,6 +113,5 @@ class BouncerQueryHandler(OONIBHandler): except KeyError: raise e.TestHelpersKeyMissing - response = {} - response['collector'] = self.bouncer.filterHelperAddresses(requested_helpers) + response = self.bouncer.filterHelperAddresses(requested_helpers) self.write(response) diff --git a/oonib/errors.py b/oonib/errors.py index 1dc8539..12295aa 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -1,7 +1,8 @@ from cyclone.web import HTTPError class OONIBError(HTTPError): - pass + def __init__(self): + pass class InvalidRequest(OONIBError): pass diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py index bba0af9..d467854 100644 --- a/oonib/input/handlers.py +++ b/oonib/input/handlers.py @@ -23,6 +23,7 @@ class InputDescHandler(OONIBHandler): self.write({'error': 'missing-input'}) except Exception, e: + log.exception(e) log.err("Invalid Input Descriptor found for id %s" % inputID) self.set_status(500) self.write({'error': 'invalid-input-descriptor'}) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 2441ddf..32f653d 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -132,7 +132,8 @@ class NewReportHandlerFile(OONIBHandler): def checkPolicy(self): policy = Policy() - policy.validateInputHash(self.inputHash) + for input_hash in self.inputHashes: + policy.validateInputHash(input_hash) policy.validateNettest(self.testName) def post(self): @@ -194,7 +195,7 @@ class NewReportHandlerFile(OONIBHandler): if config.main.policy_file: try: - self.inputHash = report_data['input_hash'] + self.inputHashes = report_data['input_hashes'] except KeyError: raise e.InputHashNotProvided self.checkPolicy()

1 0

[oonib/master] Move validation logic into policy object
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 912a4d3f1bb479865c3f3890901c2855cd61b266 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 19 15:19:42 2013 +0200 Move validation logic into policy object --- oonib/policy/handlers.py | 20 ++++++++++++++++++++ oonib/report/handlers.py | 6 ++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index 93f9f82..9b92403 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -1,3 +1,4 @@ +from oonib import errors as e from oonib.handlers import OONIBHandler from oonib import config @@ -15,6 +16,25 @@ class Policy(object): self.nettest = list(p['nettest']) self.input = list(p['input']) + def validateInputHash(self, input_hash): + valid = False + for i in self.input: + if input_hash == i['id']: + valid = True + break + if not valid: + raise e.InvalidInputHash + + def validateNettest(self, nettest_name): + # XXX add support for version checking too. + valid = False + for nt in self.nettest: + if nettest_name == nt['name']: + valid = True + break + if not valid: + raise e.InvalidNettestName + class PolicyHandler(OONIBHandler): def initialize(self): self.policy = Policy() diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 6dd5ec4..75ec55d 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -125,10 +125,8 @@ class NewReportHandlerFile(OONIBHandler): def checkPolicy(self): policy = Policy() - if not self.inputHash in policy.input.values(): - raise e.InvalidInputHash - if self.testName not in policy.nettest.keys(): - raise e.InvalidNettestName + policy.validateInputHash(self.inputHash) + policy.validateNettest(self.testName) # XXX add support for version checking too. def post(self):

1 0

[oonib/master] Move policy related things into an appropriate object
by art＠torproject.org 11 Sep '13

11 Sep '13

commit c6c0496a93cec538aaf13e8972d0dda394801567 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 19 14:31:04 2013 +0200 Move policy related things into an appropriate object --- oonib/policy/handlers.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index 788e578..93f9f82 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -5,20 +5,31 @@ import json import os import yaml -class NetTestPolicyHandler(OONIBHandler): +class Policy(object): + nettest = None + input = None + + def __init__(self): + with open(config.main.policy_file) as f: + p = yaml.safe_load(f) + self.nettest = list(p['nettest']) + self.input = list(p['input']) + +class PolicyHandler(OONIBHandler): + def initialize(self): + self.policy = Policy() + +class NetTestPolicyHandler(PolicyHandler): def get(self): """ returns a list of accepted NetTests """ - with open(config.main.policy_file) as f: - p = yaml.safe_load(f) - self.write(p['nettest']) + self.write(self.policy.nettest) -class InputPolicyHandler(OONIBHandler): +class InputPolicyHandler(PolicyHandler): def get(self): """ return list of input ids """ - with open(config.main.policy_file) as f: - p = yaml.safe_load(f) - self.write(p['input']) + self.write(self.policy.input) +

1 0

[oonib/master] Fix input descriptor
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 82933d4c2d8f812dde9790aaa7fcc4c8c2fe186f Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 16:23:50 2013 +0200 Fix input descriptor * Return the id in the /input API --- ...0e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc | 2 +- ...1ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc | 2 +- oonib/input/handlers.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc b/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc index feecc5a..953857d 100644 --- a/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc +++ b/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc @@ -1,2 +1,2 @@ -{id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1, name: Alexa Top 1k domains, description: 'The Alexa list of most visited domains fetched from here: http://www.alexa.com/topsites'} +{id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1, name: Alexa Top 1k domains, description: 'The Alexa list of most visited domains fetched from here: http://www.alexa.com/topsites', version: 1, author: ooni, date: 2013-08-21T210221Z} diff --git a/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc b/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc index 07030c6..3eb5e14 100644 --- a/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc +++ b/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc @@ -1,2 +1,2 @@ -{id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290, name: Alexa Top 1m domains, description: 'The Alexa list of 1 million most visited domains fetched from here: http://www.alexa.com/topsites'} +{id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290, name: Alexa Top 1m domains, description: 'The Alexa list of 1 million most visited domains fetched from here: http://www.alexa.com/topsites', version: 1, author: ooni, date: 2013-08-21T210221Z} diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py index ed4c6b2..bba0af9 100644 --- a/oonib/input/handlers.py +++ b/oonib/input/handlers.py @@ -15,6 +15,7 @@ class InputDescHandler(OONIBHandler): inputDesc = yaml.safe_load(f) for k in ['name', 'description', 'version', 'author', 'date']: response[k] = inputDesc[k] + response['id'] = inputID self.write(response) except IOError: log.err("No Input Descriptor found for id %s" % inputID)

1 0

[oonib/master] Properly write error messages.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 4210908e8853cd81e5945a5c09cc943a8f28bec4 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 13:34:05 2013 +0200 Properly write error messages. Update the /bouncer API to reflect spec changes --- oonib/bouncer/handlers.py | 60 +++++++++++++++++++++++++++++++++------------ oonib/errors.py | 3 ++- oonib/handlers.py | 5 ++++ 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index e669e8e..9e987f0 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -36,7 +36,7 @@ class Bouncer(object): try: helpers = self.knownHelpers[helper_name] except KeyError: - raise e.NoHelperFound + raise e.TestHelperNotFound helpers_dict = {} for helper in helpers: @@ -52,15 +52,26 @@ class Bouncer(object): requested_helpers = ['a', 'b', 'c'] will return: { - 'a': '127.0.0.1', - 'b': 'http://127.0.0.1', - 'c': '127.0.0.1:590', - 'collector': 'httpo://thirteenchars1.onion' + 'a': { + 'address': '127.0.0.1', + 'collector': 'httpo://thirteenchars1.onion' + }, + 'b': { + 'address': '127.0.0.1:8081', + 'collector': 'httpo://thirteenchars1.onion' + }, + 'c': { + 'address': 'http://127.0.0.1', + 'collector': 'httpo://thirteenchars2.onion' + }, + 'default': { + 'collector': 'httpo://thirteenchars1.onion' + } } or - {} + {'error': 'test-helper-not-found'} if no valid helper was found @@ -72,18 +83,37 @@ class Bouncer(object): result[collector] = {} result[collector][helper_name] = helper_address - helper_list = [] + # { + # 'foo.onion': {'some-helper': 'some-address'}, + # 'foo2.onion': {'some-helper': 'some-addres2'} + # } + + response = {} + default_collector = None + max_helper_count = 0 for collector, helpers in result.items(): - if len(helpers) == len(requested_helpers): - valid_helpers = helpers - valid_helpers['collector'] = collector - helper_list.append(valid_helpers) + if len(helpers) > max_helper_count: + default_collector = collector else: continue - if len(helper_list) == 0: - return {} - else: - return random.choice(helper_list) + + response['default'] = {} + response['default']['collector'] = default_collector + + if not len(result[default_collector]) == len(requested_helpers): + found_helpers = set(result[default_collector].items()) + for missing_helper in found_helpers.difference(requested_helpers): + collector, address = random.choice(self.getHelperAddress(missing_helper).items()) + response[missing_helper] = {} + response[missing_helper]['collector'] = collector + response[missing_helper]['address'] = address + + for name, address in result[default_collector].items(): + response[name] = {} + response[name]['address'] = address + response[name]['collector'] = default_collector + + return response class BouncerQueryHandler(OONIBHandler): def initialize(self): diff --git a/oonib/errors.py b/oonib/errors.py index 5c4e4d8..2385a6f 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -55,5 +55,6 @@ class TestHelpersKeyMissing(OONIBError): log_message = "test-helpers-key-missing" class TestHelperNotFound(OONIBError): - status_code = 400 + status_code = 404 log_message = "test-helper-not-found" + diff --git a/oonib/handlers.py b/oonib/handlers.py index 033a69f..2231a2a 100644 --- a/oonib/handlers.py +++ b/oonib/handlers.py @@ -4,6 +4,11 @@ from cyclone import escape from cyclone import web class OONIBHandler(web.RequestHandler): + def write_error(self, status_code, exception=None, **kw): + self.set_status(status_code) + if exception: + self.write({'error': exception.log_message}) + def write(self, chunk): """ This is a monkey patch to RequestHandler to allow us to serialize also

1 0

[oonib/master] Refuse creating new reports if the policy is violated.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 98356357980bf134c3af9b02ba5b01bb570b60a2 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 19 15:00:02 2013 +0200 Refuse creating new reports if the policy is violated. * Move all errors into a dedicated module * Add missing __init__.py for nettest handler --- oonib/errors.py | 40 ++++++++++++++++++++++++++++++++++++++++ oonib/handlers.py | 3 --- oonib/report/handlers.py | 39 ++++++++++++++++++++++++++++++--------- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/oonib/errors.py b/oonib/errors.py new file mode 100644 index 0000000..24a06fc --- /dev/null +++ b/oonib/errors.py @@ -0,0 +1,40 @@ +from cyclone.web import HTTPError + +class OONIBError(HTTPError): + pass + +class InvalidInputHash(OONIBError): + status_code = 400 + log_message = 'invalid-input-hash' + +class InvalidNettestName(OONIBError): + status_code = 400 + log_message = 'invalid-nettest-name' + +class InputHashNotProvided(OONIBError): + status_code = 400 + log_message = 'input-hash-not-provided' + +class InvalidRequestField(OONIBError): + def __init__(self, field_name): + self.status_code = 400 + self.log_message = "invalid-request-field %s" % field_name + +class MissingRequestField(OONIBError): + def __init__(self, field_name): + self.status_code = 400 + self.log_message = "missing-request-field %s" % field_name + +class MissingReportHeaderKey(OONIBError): + def __init__(self, key): + self.status_code = 406 + self.log_message = "missing-report-header-key %s" % key + +class InvalidReportHeader(OONIBError): + def __init__(self, key): + self.status_code = 406 + self.log_message = "invalid-report-header %s" % key + +class ReportNotFound(OONIBError): + status_code = 404 + log_message = "report-not-found" diff --git a/oonib/handlers.py b/oonib/handlers.py index 515e8c2..033a69f 100644 --- a/oonib/handlers.py +++ b/oonib/handlers.py @@ -15,6 +15,3 @@ class OONIBHandler(web.RequestHandler): self.set_header("Content-Type", "application/json") else: web.RequestHandler.write(self, chunk) - -class OONIBError(web.HTTPError): - pass diff --git a/oonib/nettest/__init__.py b/oonib/nettest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 05949e6..6dd5ec4 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -3,7 +3,9 @@ import string import time import yaml +from oonib import errors as e from oonib.handlers import OONIBHandler +from oonib.policy.handlers import Policy from datetime import datetime from oonib import randomStr, otime, config, log @@ -121,6 +123,14 @@ class NewReportHandlerFile(OONIBHandler): Responsible for creating and updating reports by writing to flat file. """ + def checkPolicy(self): + policy = Policy() + if not self.inputHash in policy.input.values(): + raise e.InvalidInputHash + if self.testName not in policy.nettest.keys(): + raise e.InvalidNettestName + # XXX add support for version checking too. + def post(self): """ Creates a new report with the input @@ -164,16 +174,27 @@ class NewReportHandlerFile(OONIBHandler): try: report_data = parseNewReportRequest(self.request.body) except InvalidRequestField, e: - raise OONIBError(400, "Invalid Request Field %s" % e) + raise e.InvalidRequestField(e) except MissingField, e: - raise OONIBError(400, "Missing Request Field %s" % e) + raise e.MissingRequestField(e) + + log.debug("Parsed this data %s" % report_data) - print "Parsed this data %s" % report_data software_name = report_data['software_name'] software_version = report_data['software_version'] - test_name = report_data['test_name'] - test_version = report_data['test_version'] + probe_asn = report_data['probe_asn'] + + self.testName = report_data['test_name'] + self.testVersion = report_data['test_version'] + + if config.main.policy_file: + try: + self.inputHash = report_data['input_hash'] + except KeyError: + raise e.InputHashNotProvided + self.checkPolicy() + content = yaml.safe_load(report_data['content']) content['backend_version'] = config.backend_version @@ -181,10 +202,10 @@ class NewReportHandlerFile(OONIBHandler): report_header = validate_report_header(content) except MissingReportHeaderKey, key: - raise OONIBError(406, "Missing report header key %s" % key) + raise e.MissingReportHeaderKey(key) except InvalidReportHeader, key: - raise OONIBError(406, "Invalid report header %s" % key) + raise e.InvalidReportHeaderKey(key) report_header = yaml.dump(report_header) content = "---\n" + report_header + '...\n' @@ -245,7 +266,7 @@ class NewReportHandlerFile(OONIBHandler): fdesc.setNonBlocking(fd.fileno()) fdesc.writeToFD(fd.fileno(), data) except IOError as e: - OONIBError(404, "Report not found") + e.OONIBError(404, "Report not found") class ReportNotFound(Exception): pass @@ -281,7 +302,7 @@ class CloseReportHandlerFile(OONIBHandler): try: close_report(report_id) except ReportNotFound: - OONIBError(404, "Report not found") + e.ReportNotFound class PCAPReportHandler(OONIBHandler): def get(self):

1 0

[oonib/master] Return the test helper address of the requested TH
by art＠torproject.org 11 Sep '13

11 Sep '13

commit cde886b1facaab9a646ccaa7c3f942ca542fd177 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 19 16:45:18 2013 +0200 Return the test helper address of the requested TH * Refactor bouncer into a separate object --- README.md | 2 +- data/bouncer.yaml | 2 ++ oonib.conf.example | 35 ++++++++++-------- oonib/bouncer/handlers.py | 86 +++++++++++++++++++++++++++++++++++---------- oonib/deck/handlers.py | 2 ++ oonib/errors.py | 18 ++++++++++ oonib/oonibackend.py | 36 +++++++++---------- oonib/report/handlers.py | 24 +++++++++++-- 8 files changed, 150 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 75b60a9..b2d13b9 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ The following iptables commands will map connections on low ports to those bound by oonib: ``` -# Map port 80 to config.helpers.http_return_request.port (default: 57001) +# Map port 80 to config.helpers['http-return-json-headers'].port (default: 57001) iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 57001 # Map port 443 to config.helpers.ssl.port (default: 57006) iptables -t nat -A PREROUTING -p tcp -m tcp --dport 443 -j REDIRECT --to-ports 57006 diff --git a/data/bouncer.yaml b/data/bouncer.yaml index ba54898..7709e40 100644 --- a/data/bouncer.yaml +++ b/data/bouncer.yaml @@ -1,3 +1,5 @@ collector: httpo://nkvphnp3p6agi5qq.onion: test-helper: {dns: 93.95.227.200, http-return-json-headers: 'http://93.95.227.200'} + httpo://nkvphnp3p6agi5q1.onion: + test-helper: {dns: 93.22.227.200, http-return-json-headers: 'http://93.95.227.220'} diff --git a/oonib.conf.example b/oonib.conf.example index 977b5dc..323b2af 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -6,48 +6,53 @@ main: policy_file: data/policy.yaml bouncer_file: data/bouncer.yaml - logfile: Null - tor_datadir: Null + logfile: null + tor_datadir: null database_uri: 'sqlite://oonib_test_db.db' db_threadpool_size: 10 - tor_binary: Null + tor_binary: null socks_port: 9055 tor2webmode: false pidfile: 'oonib.pid' nodaemon: true - originalname: Null - chroot: Null + originalname: null + chroot: null rundir: . - umask: Null - euid: Null - uid: Null - gid: Null - uuid: Null + umask: null + euid: null + uid: null + gid: null + uuid: null no_save: true - profile: Null + profile: null debug: false stale_time: 3600 tor_hidden_service: true helpers: - http_return_request: + http-return-json-headers: + address: null port: 57001 server_version: Apache - tcp_echo: + tcp-echo: + address: null port: 57002 daphn3: - yaml_file: Null - pcap_file: Null + address: null + yaml_file: null + pcap_file: null port: 57003 dns: + address: null udp_port: 57004 tcp_port: 57005 ssl: + address: null private_key: 'private.key' certificate: 'certificate.crt' port: 57006 diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 230f618..44f1c63 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -4,10 +4,72 @@ import yaml from oonib.handlers import OONIBHandler from oonib import config -class BouncerQueryHandler(OONIBHandler): - def initialize(self): +class Bouncer(object): + def __init__(self): self.knownHelpers = {} self.updateKnownHelpers() + + def updateKnownHelpers(self): + with open(config.main.bouncer_file) as f: + bouncerFile = yaml.safe_load(f) + for collectorName, helpers in bouncerFile['collector'].items(): + for helperName, helperAddress in helpers['test-helper'].items(): + if helperName not in self.knownHelpers.keys(): + self.knownHelpers[helperName] = [] + + self.knownHelpers[helperName].append({ + 'collector-name': collectorName, + 'helper-address': helperAddress + }) + + def getHelperAddresses(self, helper_name): + """ + Returns a dict keyed on the collector address of known test helpers. + example: + { + 'httpo://thirteenchars1.onion': '127.0.0.1', + 'httpo://thirteenchars2.onion': '127.0.0.2', + 'httpo://thirteenchars3.onion': '127.0.0.3' + } + """ + try: + helpers = self.knownHelpers[helper_name] + except KeyError: + raise e.NoHelperFound + + helpers_dict = {} + for helper in helpers: + helpers_dict[helper['collector-name']] = helper['helper-address'] + + return helpers_dict + + def filterHelperAddresses(self, requested_helpers): + """ + Returns a dict of collectors that support all the requested_helpers. + + Example: + requested_helpers = ['a', 'b', 'c'] + will return: + { + 'httpo://thirteenchars1.onion': { + 'a': '127.0.0.1', + 'b': 'http://127.0.0.1', + 'c': '127.0.0.1:590', + } + } + + """ + result = {} + for helper_name in requested_helpers: + for collector, helper_address in self.getHelperAddresses(helper_name).items(): + if collector not in result.keys(): + result[collector] = {} + result[collector][helper_name] = helper_address + return result + +class BouncerQueryHandler(OONIBHandler): + def initialize(self): + self.bouncer = Bouncer() def updateKnownHelpers(self): with open(config.main.bouncer_file) as f: @@ -26,25 +88,13 @@ class BouncerQueryHandler(OONIBHandler): try: query = json.loads(self.request.body) except ValueError: - self.set_status(400) - self.write(json.dumps({'error': 'invalid-request'})) - return + raise e.InvalidRequest try: - helpers = query['test-helpers'] + requested_helpers = query['test-helpers'] except KeyError: - self.set_status(400) - self.write(json.dumps({'error': 'test-helpers-key-missing'})) - return + raise e.TestHelpersKeyMissing response = {} - response['collector'] = {} - for helperName in helpers: - if helperName in self.knownHelpers.keys(): - chosen = random.choice(self.knownHelpers[helperName]) - collectorName, helperAddress = chosen['collector-name'], chosen['helper-address'] - if not collectorName in response['collector'].keys(): - response['collector'][collectorName] = {'test-helper': {}} - response['collector'][collectorName]['test-helper'][helperName] = helperAddress - + response['collector'] = self.bouncer.filterHelperAddresses(requested_helpers) self.write(response) diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index d4b3bbe..558ca6a 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -20,10 +20,12 @@ class DeckDescHandler(OONIBHandler): for k in ['name', 'description', 'version', 'author', 'date']: response[k] = deckDesc[k] self.write(response) + except IOError: log.err("Deck %s missing" % deckID) self.set_status(404) self.write({'error': 'missing-deck'}) + except KeyError: self.set_status(400) log.err("Deck %s missing required keys!" % deckID) diff --git a/oonib/errors.py b/oonib/errors.py index 24a06fc..1dc8539 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -3,6 +3,13 @@ from cyclone.web import HTTPError class OONIBError(HTTPError): pass +class InvalidRequest(OONIBError): + pass + +class NoHelperFound(OONIBError): + status_code = 404 + log_message = 'no-helper-found' + class InvalidInputHash(OONIBError): status_code = 400 log_message = 'invalid-input-hash' @@ -38,3 +45,14 @@ class InvalidReportHeader(OONIBError): class ReportNotFound(OONIBError): status_code = 404 log_message = "report-not-found" + +class NoValidCollector(OONIBError): + pass + +class TestHelpersKeyMissing(OONIBError): + status_code = 400 + log_message = "test-helpers-key-missing" + +class TestHelperNotFound(OONIBError): + status_code = 400 + log_message = "test-helper-not-found" diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index afd961f..c5f96ee 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -30,45 +30,45 @@ else: serviceCollection = service.IServiceCollection(application) -if config.helpers.ssl.port: - print "Starting SSL helper on %s" % config.helpers.ssl.port - ssl_helper = internet.SSLServer(int(config.helpers.ssl.port), +if config.helpers['ssl'].port: + print "Starting SSL helper on %s" % config.helpers['ssl'].port + ssl_helper = internet.SSLServer(int(config.helpers['ssl'].port), http_helpers.HTTPReturnJSONHeadersHelper(), ssl_helpers.SSLContext(config)) ssl_helper.setServiceParent(serviceCollection) # Start the DNS Server related services -if config.helpers.dns.tcp_port: - print "Starting TCP DNS Helper on %s" % config.helpers.dns.tcp_port - tcp_dns_helper = internet.TCPServer(int(config.helpers.dns.tcp_port), +if config.helpers['dns'].tcp_port: + print "Starting TCP DNS Helper on %s" % config.helpers['dns'].tcp_port + tcp_dns_helper = internet.TCPServer(int(config.helpers['dns'].tcp_port), dns_helpers.DNSTestHelper()) tcp_dns_helper.setServiceParent(serviceCollection) -if config.helpers.dns.udp_port: - print "Starting UDP DNS Helper on %s" % config.helpers.dns.udp_port +if config.helpers['dns'].udp_port: + print "Starting UDP DNS Helper on %s" % config.helpers['dns'].udp_port udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper()) - udp_dns_helper = internet.UDPServer(int(config.helpers.dns.udp_port), + udp_dns_helper = internet.UDPServer(int(config.helpers['dns'].udp_port), udp_dns_factory) udp_dns_helper.setServiceParent(serviceCollection) # XXX this needs to be ported # Start the OONI daphn3 backend -if config.helpers.daphn3.port: - print "Starting Daphn3 helper on %s" % config.helpers.daphn3.port - daphn3_helper = internet.TCPServer(int(config.helpers.daphn3.port), +if config.helpers['daphn3'].port: + print "Starting Daphn3 helper on %s" % config.helpers['daphn3'].port + daphn3_helper = internet.TCPServer(int(config.helpers['daphn3'].port), tcp_helpers.Daphn3Server()) daphn3_helper.setServiceParent(serviceCollection) -if config.helpers.tcp_echo.port: - print "Starting TCP echo helper on %s" % config.helpers.tcp_echo.port - tcp_echo_helper = internet.TCPServer(int(config.helpers.tcp_echo.port), +if config.helpers['tcp-echo'].port: + print "Starting TCP echo helper on %s" % config.helpers['tcp-echo'].port + tcp_echo_helper = internet.TCPServer(int(config.helpers['tcp-echo'].port), tcp_helpers.TCPEchoHelper()) tcp_echo_helper.setServiceParent(serviceCollection) -if config.helpers.http_return_request.port: - print "Starting HTTP return request helper on %s" % config.helpers.http_return_request.port +if config.helpers['http-return-json-headers'].port: + print "Starting HTTP return request helper on %s" % config.helpers['http-return-json-headers'].port http_return_request_helper = internet.TCPServer( - int(config.helpers.http_return_request.port), + int(config.helpers['http-return-json-headers'].port), http_helpers.HTTPReturnJSONHeadersHelper()) http_return_request_helper.setServiceParent(serviceCollection) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 75ec55d..2441ddf 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -70,6 +70,13 @@ def parseNewReportRequest(request): continue else: raise InvalidRequestField(k) + + try: + test_helper = parsed_request['test_helper'] + if not re.match(regexp, str(test_helper)): + raise InvalidRequestField('test_helper') + except KeyError: + pass return parsed_request @@ -127,7 +134,6 @@ class NewReportHandlerFile(OONIBHandler): policy = Policy() policy.validateInputHash(self.inputHash) policy.validateNettest(self.testName) - # XXX add support for version checking too. def post(self): """ @@ -219,9 +225,21 @@ class NewReportHandlerFile(OONIBHandler): # random nonce report_filename = os.path.join(config.main.report_dir, report_id) - response = {'backend_version': config.backend_version, - 'report_id': report_id + response = { + 'backend_version': config.backend_version, + 'report_id': report_id } + + try: + requested_helper = report_data['test_helper'] + except KeyError: + pass + + if requested_helper: + try: + response['test_helper_address'] = config.helpers[requested_helper].address + except KeyError: + raise e.TestHelperNotFound config.reports[report_id] = time.time()

1 0

[oonib/master] Add imports and fix referenced-before-assignment
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 06870f856c864deca0f6bb28b5f2bfc0c870940e Author: aagbsn <aagbsn(a)extc.org> Date: Sat Aug 24 15:00:20 2013 +0200 Add imports and fix referenced-before-assignment Adds required imports and fixes a bug where no test-helper is requested but requested_helper is referenced without being assigned. --- oonib/report/handlers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 32f653d..fd008dd 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -1,4 +1,7 @@ +import json +import os import random +import re import string import time import yaml @@ -231,10 +234,11 @@ class NewReportHandlerFile(OONIBHandler): 'report_id': report_id } + try: requested_helper = report_data['test_helper'] except KeyError: - pass + requested_helper = None if requested_helper: try:

1 0