tor-commits September 2013

tor-commits@lists.torproject.org

17 participants
1101 discussions

[oonib/master] More through e2e testing revealing some other bugs..
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 6b6dd406123169404907e8d556d9ab3ad9af29fa Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 17:07:59 2013 +0200 More through e2e testing revealing some other bugs.. * Kill'em all.. --- oonib/deck/api.py | 2 +- oonib/deck/handlers.py | 15 +++++++-------- oonib/errors.py | 17 ++++++++++++++++- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/oonib/deck/api.py b/oonib/deck/api.py index 6fbc7f4..ffccb0a 100644 --- a/oonib/deck/api.py +++ b/oonib/deck/api.py @@ -5,6 +5,6 @@ from oonib import config deckAPI = [ (r"/deck", handlers.DeckListHandler), (r"/deck/([a-z0-9]{64})$", handlers.DeckDescHandler), - (r"/deck/([a-z0-9]{64})/yaml$", web.StaticFileHandler, {"path": + (r"/deck/([a-z0-9]{64})/file$", web.StaticFileHandler, {"path": config.main.deck_dir}), ] diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index 558ca6a..c8aed69 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -4,6 +4,7 @@ import os import re import yaml +from oonib import errors as e from oonib.handlers import OONIBHandler from oonib import config, log @@ -12,7 +13,7 @@ class DeckDescHandler(OONIBHandler): # note: # we don't have to sanitize deckID, because it's already checked # against matching a certain pattern in the handler. - bn = os.path.basename(deckID) + bn = os.path.basename(deckID + '.desc') try: with open(os.path.join(config.main.deck_dir, bn)) as f: response = {} @@ -23,23 +24,21 @@ class DeckDescHandler(OONIBHandler): except IOError: log.err("Deck %s missing" % deckID) - self.set_status(404) - self.write({'error': 'missing-deck'}) + raise e.MissingDeck except KeyError: - self.set_status(400) log.err("Deck %s missing required keys!" % deckID) - self.write({'error': 'missing-deck-keys'}) + raise e.MissingDeckKeys class DeckListHandler(OONIBHandler): def get(self): if not config.main.deck_dir: self.set_status(501) - self.write({'error': 'no-decks-configured'}) - return + raise e.NoDecksConfigured + path = os.path.abspath(config.main.deck_dir) + "/*" decknames = map(os.path.basename, glob.iglob(path)) - decknames = filter(lambda y: re.match("[a-z0-9]{40}", y), decknames) + decknames = filter(lambda y: re.match("[a-z0-9]{64}.desc", y), decknames) deckList = [] for deckname in decknames: with open(os.path.join(config.main.deck_dir, deckname)) as f: diff --git a/oonib/errors.py b/oonib/errors.py index 2385a6f..8d6015e 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -1,11 +1,14 @@ from cyclone.web import HTTPError class OONIBError(HTTPError): + status_code = 500 + log_message = 'oonib-error' def __init__(self): pass class InvalidRequest(OONIBError): - pass + status_code = 400 + log_message = 'invalid-request' class NoHelperFound(OONIBError): status_code = 404 @@ -38,6 +41,18 @@ class MissingReportHeaderKey(OONIBError): self.status_code = 406 self.log_message = "missing-report-header-key %s" % key +class MissingDeckKeys(OONIBError): + status_code = 400 + log_message = "missing-deck-keys" + +class MissingDeck(OONIBError): + status_code = 400 + log_message = "missing-deck" + +class NoDecksConfigured(OONIBError): + status_code = 501 + log_message = "no-decks-configured" + class InvalidReportHeader(OONIBError): def __init__(self, key): self.status_code = 406

1 0

[oonib/master] Use correct status-code per oonib.md spec
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 02aac483bcfb8a2ca867449d436c0d7ab2378d2f Author: aagbsn <aagbsn(a)extc.org> Date: Sat Aug 24 15:20:28 2013 +0200 Use correct status-code per oonib.md spec --- oonib/errors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oonib/errors.py b/oonib/errors.py index 12295aa..5c4e4d8 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -12,15 +12,15 @@ class NoHelperFound(OONIBError): log_message = 'no-helper-found' class InvalidInputHash(OONIBError): - status_code = 400 + status_code = 406 log_message = 'invalid-input-hash' class InvalidNettestName(OONIBError): - status_code = 400 + status_code = 406 log_message = 'invalid-nettest-name' class InputHashNotProvided(OONIBError): - status_code = 400 + status_code = 406 log_message = 'input-hash-not-provided' class InvalidRequestField(OONIBError):

1 0

[oonib/master] Update data/policy.yaml to use the NetTest name.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 49e5967408c61145c7d75c67b35c8b7a145dfa48 Author: aagbsn <aagbsn(a)extc.org> Date: Sat Aug 24 15:02:09 2013 +0200 Update data/policy.yaml to use the NetTest name. Some of the NetTest name strings do not match the convention used by ooni-probe. Note that we normalize the NetTestClass.name field. See : ooni-probe/ooni/nettest.py:195 and 392 --- data/policy.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/data/policy.yaml b/data/policy.yaml index e0d6c08..fcc1df4 100644 --- a/data/policy.yaml +++ b/data/policy.yaml @@ -2,14 +2,13 @@ input: - {id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1} - {id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290} nettest: -- {name: dnsconsistency, version: 0.5} -- {name: http_requests, version: 0.2.3} -- {name: tcpconnect, version: 0.1} -- {name: captiveportal, version: 0.2} -- {name: daphne, version: 0.1} -- {name: dnsspoof, version: 0.2} -- {name: http_body, version: 0.0.1} +- {name: dns_consistency, version: 0.5} +- {name: http_requests_test, version: 0.2.3} +- {name: tcp_connect, version: 0.1} +- {name: captivep, version: 0.2} +- {name: daphne3, version: 0.1} +- {name: dns_spoof, version: 0.2} - {name: http_header_field_manipulation, version: 0.1.3} - {name: http_host, version: 0.2.3} - {name: http_invalid_request_line, version: 0.1.4} -- {name: traceroute, version: 0.1.1} +- {name: multi_protocol_traceroute_test, version: 0.1.1}

1 0

[oonib/master] Bump to version 1.0.0-rc1
by art＠torproject.org 11 Sep '13

11 Sep '13

commit f13cd5b4c7ebf2a3b1ba14e1a93d7d689e0e299c Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 15:13:15 2013 +0200 Bump to version 1.0.0-rc1 --- oonib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oonib/__init__.py b/oonib/__init__.py index 54d3006..e01b7b3 100644 --- a/oonib/__init__.py +++ b/oonib/__init__.py @@ -3,7 +3,7 @@ In here we shall keep track of all variables and objects that should be instantiated only once and be common to pieces of GLBackend code. """ -__version__ = '0.9.1' +__version__ = '1.0.0-rc1' __all__ = ['Storage', 'randomStr']

1 0

[oonib/master] Add imports and fix referenced-before-assignment
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 06870f856c864deca0f6bb28b5f2bfc0c870940e Author: aagbsn <aagbsn(a)extc.org> Date: Sat Aug 24 15:00:20 2013 +0200 Add imports and fix referenced-before-assignment Adds required imports and fixes a bug where no test-helper is requested but requested_helper is referenced without being assigned. --- oonib/report/handlers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 32f653d..fd008dd 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -1,4 +1,7 @@ +import json +import os import random +import re import string import time import yaml @@ -231,10 +234,11 @@ class NewReportHandlerFile(OONIBHandler): 'report_id': report_id } + try: requested_helper = report_data['test_helper'] except KeyError: - pass + requested_helper = None if requested_helper: try:

1 0

[oonib/master] Return the test helper address of the requested TH
by art＠torproject.org 11 Sep '13

11 Sep '13

commit cde886b1facaab9a646ccaa7c3f942ca542fd177 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 19 16:45:18 2013 +0200 Return the test helper address of the requested TH * Refactor bouncer into a separate object --- README.md | 2 +- data/bouncer.yaml | 2 ++ oonib.conf.example | 35 ++++++++++-------- oonib/bouncer/handlers.py | 86 +++++++++++++++++++++++++++++++++++---------- oonib/deck/handlers.py | 2 ++ oonib/errors.py | 18 ++++++++++ oonib/oonibackend.py | 36 +++++++++---------- oonib/report/handlers.py | 24 +++++++++++-- 8 files changed, 150 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 75b60a9..b2d13b9 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ The following iptables commands will map connections on low ports to those bound by oonib: ``` -# Map port 80 to config.helpers.http_return_request.port (default: 57001) +# Map port 80 to config.helpers['http-return-json-headers'].port (default: 57001) iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 57001 # Map port 443 to config.helpers.ssl.port (default: 57006) iptables -t nat -A PREROUTING -p tcp -m tcp --dport 443 -j REDIRECT --to-ports 57006 diff --git a/data/bouncer.yaml b/data/bouncer.yaml index ba54898..7709e40 100644 --- a/data/bouncer.yaml +++ b/data/bouncer.yaml @@ -1,3 +1,5 @@ collector: httpo://nkvphnp3p6agi5qq.onion: test-helper: {dns: 93.95.227.200, http-return-json-headers: 'http://93.95.227.200'} + httpo://nkvphnp3p6agi5q1.onion: + test-helper: {dns: 93.22.227.200, http-return-json-headers: 'http://93.95.227.220'} diff --git a/oonib.conf.example b/oonib.conf.example index 977b5dc..323b2af 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -6,48 +6,53 @@ main: policy_file: data/policy.yaml bouncer_file: data/bouncer.yaml - logfile: Null - tor_datadir: Null + logfile: null + tor_datadir: null database_uri: 'sqlite://oonib_test_db.db' db_threadpool_size: 10 - tor_binary: Null + tor_binary: null socks_port: 9055 tor2webmode: false pidfile: 'oonib.pid' nodaemon: true - originalname: Null - chroot: Null + originalname: null + chroot: null rundir: . - umask: Null - euid: Null - uid: Null - gid: Null - uuid: Null + umask: null + euid: null + uid: null + gid: null + uuid: null no_save: true - profile: Null + profile: null debug: false stale_time: 3600 tor_hidden_service: true helpers: - http_return_request: + http-return-json-headers: + address: null port: 57001 server_version: Apache - tcp_echo: + tcp-echo: + address: null port: 57002 daphn3: - yaml_file: Null - pcap_file: Null + address: null + yaml_file: null + pcap_file: null port: 57003 dns: + address: null udp_port: 57004 tcp_port: 57005 ssl: + address: null private_key: 'private.key' certificate: 'certificate.crt' port: 57006 diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 230f618..44f1c63 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -4,10 +4,72 @@ import yaml from oonib.handlers import OONIBHandler from oonib import config -class BouncerQueryHandler(OONIBHandler): - def initialize(self): +class Bouncer(object): + def __init__(self): self.knownHelpers = {} self.updateKnownHelpers() + + def updateKnownHelpers(self): + with open(config.main.bouncer_file) as f: + bouncerFile = yaml.safe_load(f) + for collectorName, helpers in bouncerFile['collector'].items(): + for helperName, helperAddress in helpers['test-helper'].items(): + if helperName not in self.knownHelpers.keys(): + self.knownHelpers[helperName] = [] + + self.knownHelpers[helperName].append({ + 'collector-name': collectorName, + 'helper-address': helperAddress + }) + + def getHelperAddresses(self, helper_name): + """ + Returns a dict keyed on the collector address of known test helpers. + example: + { + 'httpo://thirteenchars1.onion': '127.0.0.1', + 'httpo://thirteenchars2.onion': '127.0.0.2', + 'httpo://thirteenchars3.onion': '127.0.0.3' + } + """ + try: + helpers = self.knownHelpers[helper_name] + except KeyError: + raise e.NoHelperFound + + helpers_dict = {} + for helper in helpers: + helpers_dict[helper['collector-name']] = helper['helper-address'] + + return helpers_dict + + def filterHelperAddresses(self, requested_helpers): + """ + Returns a dict of collectors that support all the requested_helpers. + + Example: + requested_helpers = ['a', 'b', 'c'] + will return: + { + 'httpo://thirteenchars1.onion': { + 'a': '127.0.0.1', + 'b': 'http://127.0.0.1', + 'c': '127.0.0.1:590', + } + } + + """ + result = {} + for helper_name in requested_helpers: + for collector, helper_address in self.getHelperAddresses(helper_name).items(): + if collector not in result.keys(): + result[collector] = {} + result[collector][helper_name] = helper_address + return result + +class BouncerQueryHandler(OONIBHandler): + def initialize(self): + self.bouncer = Bouncer() def updateKnownHelpers(self): with open(config.main.bouncer_file) as f: @@ -26,25 +88,13 @@ class BouncerQueryHandler(OONIBHandler): try: query = json.loads(self.request.body) except ValueError: - self.set_status(400) - self.write(json.dumps({'error': 'invalid-request'})) - return + raise e.InvalidRequest try: - helpers = query['test-helpers'] + requested_helpers = query['test-helpers'] except KeyError: - self.set_status(400) - self.write(json.dumps({'error': 'test-helpers-key-missing'})) - return + raise e.TestHelpersKeyMissing response = {} - response['collector'] = {} - for helperName in helpers: - if helperName in self.knownHelpers.keys(): - chosen = random.choice(self.knownHelpers[helperName]) - collectorName, helperAddress = chosen['collector-name'], chosen['helper-address'] - if not collectorName in response['collector'].keys(): - response['collector'][collectorName] = {'test-helper': {}} - response['collector'][collectorName]['test-helper'][helperName] = helperAddress - + response['collector'] = self.bouncer.filterHelperAddresses(requested_helpers) self.write(response) diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index d4b3bbe..558ca6a 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -20,10 +20,12 @@ class DeckDescHandler(OONIBHandler): for k in ['name', 'description', 'version', 'author', 'date']: response[k] = deckDesc[k] self.write(response) + except IOError: log.err("Deck %s missing" % deckID) self.set_status(404) self.write({'error': 'missing-deck'}) + except KeyError: self.set_status(400) log.err("Deck %s missing required keys!" % deckID) diff --git a/oonib/errors.py b/oonib/errors.py index 24a06fc..1dc8539 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -3,6 +3,13 @@ from cyclone.web import HTTPError class OONIBError(HTTPError): pass +class InvalidRequest(OONIBError): + pass + +class NoHelperFound(OONIBError): + status_code = 404 + log_message = 'no-helper-found' + class InvalidInputHash(OONIBError): status_code = 400 log_message = 'invalid-input-hash' @@ -38,3 +45,14 @@ class InvalidReportHeader(OONIBError): class ReportNotFound(OONIBError): status_code = 404 log_message = "report-not-found" + +class NoValidCollector(OONIBError): + pass + +class TestHelpersKeyMissing(OONIBError): + status_code = 400 + log_message = "test-helpers-key-missing" + +class TestHelperNotFound(OONIBError): + status_code = 400 + log_message = "test-helper-not-found" diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index afd961f..c5f96ee 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -30,45 +30,45 @@ else: serviceCollection = service.IServiceCollection(application) -if config.helpers.ssl.port: - print "Starting SSL helper on %s" % config.helpers.ssl.port - ssl_helper = internet.SSLServer(int(config.helpers.ssl.port), +if config.helpers['ssl'].port: + print "Starting SSL helper on %s" % config.helpers['ssl'].port + ssl_helper = internet.SSLServer(int(config.helpers['ssl'].port), http_helpers.HTTPReturnJSONHeadersHelper(), ssl_helpers.SSLContext(config)) ssl_helper.setServiceParent(serviceCollection) # Start the DNS Server related services -if config.helpers.dns.tcp_port: - print "Starting TCP DNS Helper on %s" % config.helpers.dns.tcp_port - tcp_dns_helper = internet.TCPServer(int(config.helpers.dns.tcp_port), +if config.helpers['dns'].tcp_port: + print "Starting TCP DNS Helper on %s" % config.helpers['dns'].tcp_port + tcp_dns_helper = internet.TCPServer(int(config.helpers['dns'].tcp_port), dns_helpers.DNSTestHelper()) tcp_dns_helper.setServiceParent(serviceCollection) -if config.helpers.dns.udp_port: - print "Starting UDP DNS Helper on %s" % config.helpers.dns.udp_port +if config.helpers['dns'].udp_port: + print "Starting UDP DNS Helper on %s" % config.helpers['dns'].udp_port udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper()) - udp_dns_helper = internet.UDPServer(int(config.helpers.dns.udp_port), + udp_dns_helper = internet.UDPServer(int(config.helpers['dns'].udp_port), udp_dns_factory) udp_dns_helper.setServiceParent(serviceCollection) # XXX this needs to be ported # Start the OONI daphn3 backend -if config.helpers.daphn3.port: - print "Starting Daphn3 helper on %s" % config.helpers.daphn3.port - daphn3_helper = internet.TCPServer(int(config.helpers.daphn3.port), +if config.helpers['daphn3'].port: + print "Starting Daphn3 helper on %s" % config.helpers['daphn3'].port + daphn3_helper = internet.TCPServer(int(config.helpers['daphn3'].port), tcp_helpers.Daphn3Server()) daphn3_helper.setServiceParent(serviceCollection) -if config.helpers.tcp_echo.port: - print "Starting TCP echo helper on %s" % config.helpers.tcp_echo.port - tcp_echo_helper = internet.TCPServer(int(config.helpers.tcp_echo.port), +if config.helpers['tcp-echo'].port: + print "Starting TCP echo helper on %s" % config.helpers['tcp-echo'].port + tcp_echo_helper = internet.TCPServer(int(config.helpers['tcp-echo'].port), tcp_helpers.TCPEchoHelper()) tcp_echo_helper.setServiceParent(serviceCollection) -if config.helpers.http_return_request.port: - print "Starting HTTP return request helper on %s" % config.helpers.http_return_request.port +if config.helpers['http-return-json-headers'].port: + print "Starting HTTP return request helper on %s" % config.helpers['http-return-json-headers'].port http_return_request_helper = internet.TCPServer( - int(config.helpers.http_return_request.port), + int(config.helpers['http-return-json-headers'].port), http_helpers.HTTPReturnJSONHeadersHelper()) http_return_request_helper.setServiceParent(serviceCollection) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 75ec55d..2441ddf 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -70,6 +70,13 @@ def parseNewReportRequest(request): continue else: raise InvalidRequestField(k) + + try: + test_helper = parsed_request['test_helper'] + if not re.match(regexp, str(test_helper)): + raise InvalidRequestField('test_helper') + except KeyError: + pass return parsed_request @@ -127,7 +134,6 @@ class NewReportHandlerFile(OONIBHandler): policy = Policy() policy.validateInputHash(self.inputHash) policy.validateNettest(self.testName) - # XXX add support for version checking too. def post(self): """ @@ -219,9 +225,21 @@ class NewReportHandlerFile(OONIBHandler): # random nonce report_filename = os.path.join(config.main.report_dir, report_id) - response = {'backend_version': config.backend_version, - 'report_id': report_id + response = { + 'backend_version': config.backend_version, + 'report_id': report_id } + + try: + requested_helper = report_data['test_helper'] + except KeyError: + pass + + if requested_helper: + try: + response['test_helper_address'] = config.helpers[requested_helper].address + except KeyError: + raise e.TestHelperNotFound config.reports[report_id] = time.time()

1 0

[oonib/master] Refuse creating new reports if the policy is violated.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 98356357980bf134c3af9b02ba5b01bb570b60a2 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 19 15:00:02 2013 +0200 Refuse creating new reports if the policy is violated. * Move all errors into a dedicated module * Add missing __init__.py for nettest handler --- oonib/errors.py | 40 ++++++++++++++++++++++++++++++++++++++++ oonib/handlers.py | 3 --- oonib/report/handlers.py | 39 ++++++++++++++++++++++++++++++--------- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/oonib/errors.py b/oonib/errors.py new file mode 100644 index 0000000..24a06fc --- /dev/null +++ b/oonib/errors.py @@ -0,0 +1,40 @@ +from cyclone.web import HTTPError + +class OONIBError(HTTPError): + pass + +class InvalidInputHash(OONIBError): + status_code = 400 + log_message = 'invalid-input-hash' + +class InvalidNettestName(OONIBError): + status_code = 400 + log_message = 'invalid-nettest-name' + +class InputHashNotProvided(OONIBError): + status_code = 400 + log_message = 'input-hash-not-provided' + +class InvalidRequestField(OONIBError): + def __init__(self, field_name): + self.status_code = 400 + self.log_message = "invalid-request-field %s" % field_name + +class MissingRequestField(OONIBError): + def __init__(self, field_name): + self.status_code = 400 + self.log_message = "missing-request-field %s" % field_name + +class MissingReportHeaderKey(OONIBError): + def __init__(self, key): + self.status_code = 406 + self.log_message = "missing-report-header-key %s" % key + +class InvalidReportHeader(OONIBError): + def __init__(self, key): + self.status_code = 406 + self.log_message = "invalid-report-header %s" % key + +class ReportNotFound(OONIBError): + status_code = 404 + log_message = "report-not-found" diff --git a/oonib/handlers.py b/oonib/handlers.py index 515e8c2..033a69f 100644 --- a/oonib/handlers.py +++ b/oonib/handlers.py @@ -15,6 +15,3 @@ class OONIBHandler(web.RequestHandler): self.set_header("Content-Type", "application/json") else: web.RequestHandler.write(self, chunk) - -class OONIBError(web.HTTPError): - pass diff --git a/oonib/nettest/__init__.py b/oonib/nettest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 05949e6..6dd5ec4 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -3,7 +3,9 @@ import string import time import yaml +from oonib import errors as e from oonib.handlers import OONIBHandler +from oonib.policy.handlers import Policy from datetime import datetime from oonib import randomStr, otime, config, log @@ -121,6 +123,14 @@ class NewReportHandlerFile(OONIBHandler): Responsible for creating and updating reports by writing to flat file. """ + def checkPolicy(self): + policy = Policy() + if not self.inputHash in policy.input.values(): + raise e.InvalidInputHash + if self.testName not in policy.nettest.keys(): + raise e.InvalidNettestName + # XXX add support for version checking too. + def post(self): """ Creates a new report with the input @@ -164,16 +174,27 @@ class NewReportHandlerFile(OONIBHandler): try: report_data = parseNewReportRequest(self.request.body) except InvalidRequestField, e: - raise OONIBError(400, "Invalid Request Field %s" % e) + raise e.InvalidRequestField(e) except MissingField, e: - raise OONIBError(400, "Missing Request Field %s" % e) + raise e.MissingRequestField(e) + + log.debug("Parsed this data %s" % report_data) - print "Parsed this data %s" % report_data software_name = report_data['software_name'] software_version = report_data['software_version'] - test_name = report_data['test_name'] - test_version = report_data['test_version'] + probe_asn = report_data['probe_asn'] + + self.testName = report_data['test_name'] + self.testVersion = report_data['test_version'] + + if config.main.policy_file: + try: + self.inputHash = report_data['input_hash'] + except KeyError: + raise e.InputHashNotProvided + self.checkPolicy() + content = yaml.safe_load(report_data['content']) content['backend_version'] = config.backend_version @@ -181,10 +202,10 @@ class NewReportHandlerFile(OONIBHandler): report_header = validate_report_header(content) except MissingReportHeaderKey, key: - raise OONIBError(406, "Missing report header key %s" % key) + raise e.MissingReportHeaderKey(key) except InvalidReportHeader, key: - raise OONIBError(406, "Invalid report header %s" % key) + raise e.InvalidReportHeaderKey(key) report_header = yaml.dump(report_header) content = "---\n" + report_header + '...\n' @@ -245,7 +266,7 @@ class NewReportHandlerFile(OONIBHandler): fdesc.setNonBlocking(fd.fileno()) fdesc.writeToFD(fd.fileno(), data) except IOError as e: - OONIBError(404, "Report not found") + e.OONIBError(404, "Report not found") class ReportNotFound(Exception): pass @@ -281,7 +302,7 @@ class CloseReportHandlerFile(OONIBHandler): try: close_report(report_id) except ReportNotFound: - OONIBError(404, "Report not found") + e.ReportNotFound class PCAPReportHandler(OONIBHandler): def get(self):

1 0

[oonib/master] Properly write error messages.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 4210908e8853cd81e5945a5c09cc943a8f28bec4 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 13:34:05 2013 +0200 Properly write error messages. Update the /bouncer API to reflect spec changes --- oonib/bouncer/handlers.py | 60 +++++++++++++++++++++++++++++++++------------ oonib/errors.py | 3 ++- oonib/handlers.py | 5 ++++ 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index e669e8e..9e987f0 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -36,7 +36,7 @@ class Bouncer(object): try: helpers = self.knownHelpers[helper_name] except KeyError: - raise e.NoHelperFound + raise e.TestHelperNotFound helpers_dict = {} for helper in helpers: @@ -52,15 +52,26 @@ class Bouncer(object): requested_helpers = ['a', 'b', 'c'] will return: { - 'a': '127.0.0.1', - 'b': 'http://127.0.0.1', - 'c': '127.0.0.1:590', - 'collector': 'httpo://thirteenchars1.onion' + 'a': { + 'address': '127.0.0.1', + 'collector': 'httpo://thirteenchars1.onion' + }, + 'b': { + 'address': '127.0.0.1:8081', + 'collector': 'httpo://thirteenchars1.onion' + }, + 'c': { + 'address': 'http://127.0.0.1', + 'collector': 'httpo://thirteenchars2.onion' + }, + 'default': { + 'collector': 'httpo://thirteenchars1.onion' + } } or - {} + {'error': 'test-helper-not-found'} if no valid helper was found @@ -72,18 +83,37 @@ class Bouncer(object): result[collector] = {} result[collector][helper_name] = helper_address - helper_list = [] + # { + # 'foo.onion': {'some-helper': 'some-address'}, + # 'foo2.onion': {'some-helper': 'some-addres2'} + # } + + response = {} + default_collector = None + max_helper_count = 0 for collector, helpers in result.items(): - if len(helpers) == len(requested_helpers): - valid_helpers = helpers - valid_helpers['collector'] = collector - helper_list.append(valid_helpers) + if len(helpers) > max_helper_count: + default_collector = collector else: continue - if len(helper_list) == 0: - return {} - else: - return random.choice(helper_list) + + response['default'] = {} + response['default']['collector'] = default_collector + + if not len(result[default_collector]) == len(requested_helpers): + found_helpers = set(result[default_collector].items()) + for missing_helper in found_helpers.difference(requested_helpers): + collector, address = random.choice(self.getHelperAddress(missing_helper).items()) + response[missing_helper] = {} + response[missing_helper]['collector'] = collector + response[missing_helper]['address'] = address + + for name, address in result[default_collector].items(): + response[name] = {} + response[name]['address'] = address + response[name]['collector'] = default_collector + + return response class BouncerQueryHandler(OONIBHandler): def initialize(self): diff --git a/oonib/errors.py b/oonib/errors.py index 5c4e4d8..2385a6f 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -55,5 +55,6 @@ class TestHelpersKeyMissing(OONIBError): log_message = "test-helpers-key-missing" class TestHelperNotFound(OONIBError): - status_code = 400 + status_code = 404 log_message = "test-helper-not-found" + diff --git a/oonib/handlers.py b/oonib/handlers.py index 033a69f..2231a2a 100644 --- a/oonib/handlers.py +++ b/oonib/handlers.py @@ -4,6 +4,11 @@ from cyclone import escape from cyclone import web class OONIBHandler(web.RequestHandler): + def write_error(self, status_code, exception=None, **kw): + self.set_status(status_code) + if exception: + self.write({'error': exception.log_message}) + def write(self, chunk): """ This is a monkey patch to RequestHandler to allow us to serialize also

1 0

[oonib/master] Fix input descriptor
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 82933d4c2d8f812dde9790aaa7fcc4c8c2fe186f Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 16:23:50 2013 +0200 Fix input descriptor * Return the id in the /input API --- ...0e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc | 2 +- ...1ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc | 2 +- oonib/input/handlers.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc b/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc index feecc5a..953857d 100644 --- a/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc +++ b/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc @@ -1,2 +1,2 @@ -{id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1, name: Alexa Top 1k domains, description: 'The Alexa list of most visited domains fetched from here: http://www.alexa.com/topsites'} +{id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1, name: Alexa Top 1k domains, description: 'The Alexa list of most visited domains fetched from here: http://www.alexa.com/topsites', version: 1, author: ooni, date: 2013-08-21T210221Z} diff --git a/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc b/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc index 07030c6..3eb5e14 100644 --- a/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc +++ b/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc @@ -1,2 +1,2 @@ -{id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290, name: Alexa Top 1m domains, description: 'The Alexa list of 1 million most visited domains fetched from here: http://www.alexa.com/topsites'} +{id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290, name: Alexa Top 1m domains, description: 'The Alexa list of 1 million most visited domains fetched from here: http://www.alexa.com/topsites', version: 1, author: ooni, date: 2013-08-21T210221Z} diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py index ed4c6b2..bba0af9 100644 --- a/oonib/input/handlers.py +++ b/oonib/input/handlers.py @@ -15,6 +15,7 @@ class InputDescHandler(OONIBHandler): inputDesc = yaml.safe_load(f) for k in ['name', 'description', 'version', 'author', 'date']: response[k] = inputDesc[k] + response['id'] = inputID self.write(response) except IOError: log.err("No Input Descriptor found for id %s" % inputID)

1 0

[oonib/master] Move policy related things into an appropriate object
by art＠torproject.org 11 Sep '13

11 Sep '13

commit c6c0496a93cec538aaf13e8972d0dda394801567 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 19 14:31:04 2013 +0200 Move policy related things into an appropriate object --- oonib/policy/handlers.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index 788e578..93f9f82 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -5,20 +5,31 @@ import json import os import yaml -class NetTestPolicyHandler(OONIBHandler): +class Policy(object): + nettest = None + input = None + + def __init__(self): + with open(config.main.policy_file) as f: + p = yaml.safe_load(f) + self.nettest = list(p['nettest']) + self.input = list(p['input']) + +class PolicyHandler(OONIBHandler): + def initialize(self): + self.policy = Policy() + +class NetTestPolicyHandler(PolicyHandler): def get(self): """ returns a list of accepted NetTests """ - with open(config.main.policy_file) as f: - p = yaml.safe_load(f) - self.write(p['nettest']) + self.write(self.policy.nettest) -class InputPolicyHandler(OONIBHandler): +class InputPolicyHandler(PolicyHandler): def get(self): """ return list of input ids """ - with open(config.main.policy_file) as f: - p = yaml.safe_load(f) - self.write(p['input']) + self.write(self.policy.input) +

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits September 2013