September 2013 - tor-commits - lists.torproject.org

[oonib/master] Add imports and fix referenced-before-assignment
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 06870f856c864deca0f6bb28b5f2bfc0c870940e Author: aagbsn <aagbsn(a)extc.org> Date: Sat Aug 24 15:00:20 2013 +0200 Add imports and fix referenced-before-assignment Adds required imports and fixes a bug where no test-helper is requested but requested_helper is referenced without being assigned. --- oonib/report/handlers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 32f653d..fd008dd 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -1,4 +1,7 @@ +import json +import os import random +import re import string import time import yaml @@ -231,10 +234,11 @@ class NewReportHandlerFile(OONIBHandler): 'report_id': report_id } + try: requested_helper = report_data['test_helper'] except KeyError: - pass + requested_helper = None if requested_helper: try:

1 0

[oonib/master] Bump to version 1.0.0-rc1
by art＠torproject.org 11 Sep '13

11 Sep '13

commit f13cd5b4c7ebf2a3b1ba14e1a93d7d689e0e299c Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 15:13:15 2013 +0200 Bump to version 1.0.0-rc1 --- oonib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oonib/__init__.py b/oonib/__init__.py index 54d3006..e01b7b3 100644 --- a/oonib/__init__.py +++ b/oonib/__init__.py @@ -3,7 +3,7 @@ In here we shall keep track of all variables and objects that should be instantiated only once and be common to pieces of GLBackend code. """ -__version__ = '0.9.1' +__version__ = '1.0.0-rc1' __all__ = ['Storage', 'randomStr']

1 0

[oonib/master] Update data/policy.yaml to use the NetTest name.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 49e5967408c61145c7d75c67b35c8b7a145dfa48 Author: aagbsn <aagbsn(a)extc.org> Date: Sat Aug 24 15:02:09 2013 +0200 Update data/policy.yaml to use the NetTest name. Some of the NetTest name strings do not match the convention used by ooni-probe. Note that we normalize the NetTestClass.name field. See : ooni-probe/ooni/nettest.py:195 and 392 --- data/policy.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/data/policy.yaml b/data/policy.yaml index e0d6c08..fcc1df4 100644 --- a/data/policy.yaml +++ b/data/policy.yaml @@ -2,14 +2,13 @@ input: - {id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1} - {id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290} nettest: -- {name: dnsconsistency, version: 0.5} -- {name: http_requests, version: 0.2.3} -- {name: tcpconnect, version: 0.1} -- {name: captiveportal, version: 0.2} -- {name: daphne, version: 0.1} -- {name: dnsspoof, version: 0.2} -- {name: http_body, version: 0.0.1} +- {name: dns_consistency, version: 0.5} +- {name: http_requests_test, version: 0.2.3} +- {name: tcp_connect, version: 0.1} +- {name: captivep, version: 0.2} +- {name: daphne3, version: 0.1} +- {name: dns_spoof, version: 0.2} - {name: http_header_field_manipulation, version: 0.1.3} - {name: http_host, version: 0.2.3} - {name: http_invalid_request_line, version: 0.1.4} -- {name: traceroute, version: 0.1.1} +- {name: multi_protocol_traceroute_test, version: 0.1.1}

1 0

[oonib/master] Use correct status-code per oonib.md spec
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 02aac483bcfb8a2ca867449d436c0d7ab2378d2f Author: aagbsn <aagbsn(a)extc.org> Date: Sat Aug 24 15:20:28 2013 +0200 Use correct status-code per oonib.md spec --- oonib/errors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oonib/errors.py b/oonib/errors.py index 12295aa..5c4e4d8 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -12,15 +12,15 @@ class NoHelperFound(OONIBError): log_message = 'no-helper-found' class InvalidInputHash(OONIBError): - status_code = 400 + status_code = 406 log_message = 'invalid-input-hash' class InvalidNettestName(OONIBError): - status_code = 400 + status_code = 406 log_message = 'invalid-nettest-name' class InputHashNotProvided(OONIBError): - status_code = 400 + status_code = 406 log_message = 'input-hash-not-provided' class InvalidRequestField(OONIBError):

1 0

[oonib/master] More through e2e testing revealing some other bugs..
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 6b6dd406123169404907e8d556d9ab3ad9af29fa Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 17:07:59 2013 +0200 More through e2e testing revealing some other bugs.. * Kill'em all.. --- oonib/deck/api.py | 2 +- oonib/deck/handlers.py | 15 +++++++-------- oonib/errors.py | 17 ++++++++++++++++- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/oonib/deck/api.py b/oonib/deck/api.py index 6fbc7f4..ffccb0a 100644 --- a/oonib/deck/api.py +++ b/oonib/deck/api.py @@ -5,6 +5,6 @@ from oonib import config deckAPI = [ (r"/deck", handlers.DeckListHandler), (r"/deck/([a-z0-9]{64})$", handlers.DeckDescHandler), - (r"/deck/([a-z0-9]{64})/yaml$", web.StaticFileHandler, {"path": + (r"/deck/([a-z0-9]{64})/file$", web.StaticFileHandler, {"path": config.main.deck_dir}), ] diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index 558ca6a..c8aed69 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -4,6 +4,7 @@ import os import re import yaml +from oonib import errors as e from oonib.handlers import OONIBHandler from oonib import config, log @@ -12,7 +13,7 @@ class DeckDescHandler(OONIBHandler): # note: # we don't have to sanitize deckID, because it's already checked # against matching a certain pattern in the handler. - bn = os.path.basename(deckID) + bn = os.path.basename(deckID + '.desc') try: with open(os.path.join(config.main.deck_dir, bn)) as f: response = {} @@ -23,23 +24,21 @@ class DeckDescHandler(OONIBHandler): except IOError: log.err("Deck %s missing" % deckID) - self.set_status(404) - self.write({'error': 'missing-deck'}) + raise e.MissingDeck except KeyError: - self.set_status(400) log.err("Deck %s missing required keys!" % deckID) - self.write({'error': 'missing-deck-keys'}) + raise e.MissingDeckKeys class DeckListHandler(OONIBHandler): def get(self): if not config.main.deck_dir: self.set_status(501) - self.write({'error': 'no-decks-configured'}) - return + raise e.NoDecksConfigured + path = os.path.abspath(config.main.deck_dir) + "/*" decknames = map(os.path.basename, glob.iglob(path)) - decknames = filter(lambda y: re.match("[a-z0-9]{40}", y), decknames) + decknames = filter(lambda y: re.match("[a-z0-9]{64}.desc", y), decknames) deckList = [] for deckname in decknames: with open(os.path.join(config.main.deck_dir, deckname)) as f: diff --git a/oonib/errors.py b/oonib/errors.py index 2385a6f..8d6015e 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -1,11 +1,14 @@ from cyclone.web import HTTPError class OONIBError(HTTPError): + status_code = 500 + log_message = 'oonib-error' def __init__(self): pass class InvalidRequest(OONIBError): - pass + status_code = 400 + log_message = 'invalid-request' class NoHelperFound(OONIBError): status_code = 404 @@ -38,6 +41,18 @@ class MissingReportHeaderKey(OONIBError): self.status_code = 406 self.log_message = "missing-report-header-key %s" % key +class MissingDeckKeys(OONIBError): + status_code = 400 + log_message = "missing-deck-keys" + +class MissingDeck(OONIBError): + status_code = 400 + log_message = "missing-deck" + +class NoDecksConfigured(OONIBError): + status_code = 501 + log_message = "no-decks-configured" + class InvalidReportHeader(OONIBError): def __init__(self, key): self.status_code = 406

1 0

[oonib/master] Start 2 separate Tor Hidden Services for the bouncer and the collector
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 54889673061b7420ed67c2c42dd66f43ac36d851 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 15:12:56 2013 +0200 Start 2 separate Tor Hidden Services for the bouncer and the collector --- oonib/api.py | 3 ++- oonib/runner.py | 35 +++++++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/oonib/api.py b/oonib/api.py index 622df4d..f2c48be 100644 --- a/oonib/api.py +++ b/oonib/api.py @@ -8,6 +8,7 @@ from oonib.bouncer.api import bouncerAPI from oonib import config +ooniBouncer = None ooniBackendAPI = [] ooniBackendAPI += reportAPI @@ -21,6 +22,6 @@ if config.main.policy_file: ooniBackendAPI += policyAPI if config.main.bouncer_file: - ooniBackendAPI += bouncerAPI + ooniBouncer = web.Application(bouncerAPI, debug=True) ooniBackend = web.Application(ooniBackendAPI, debug=True) diff --git a/oonib/runner.py b/oonib/runner.py index 294f414..d03aa33 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -21,7 +21,7 @@ from txtorcon import TCPHiddenServiceEndpoint, TorConfig from txtorcon import launch_tor from oonib.report.api import reportAPI -from oonib.api import ooniBackend +from oonib.api import ooniBackend, ooniBouncer from oonib import oonibackend from oonib import config @@ -52,10 +52,29 @@ def setupCollector(tor_process_protocol, datadir): #XXX: set up the various API endpoints, if configured and enabled #XXX: also set up a separate keyed hidden service for collectors to push their status to, if the bouncer is enabled hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, - data_dir=datadir) - hidden_service = hs_endpoint.listen(ooniBackend) - hidden_service.addCallback(setup_complete) - hidden_service.addErrback(txSetupFailed) + data_dir=os.path.join(datadir, 'collector')) + d = hs_endpoint.listen(ooniBackend) + + d.addCallback(setup_complete) + d.addErrback(txSetupFailed) + + return tor_process_protocol + +def setupBouncer(tor_process_protocol, datadir): + def setup_complete(port): + #XXX: drop some other noise about what API are available on this machine + print("Exposed bouncer Tor hidden service on httpo://%s" + % port.onion_uri) + + torconfig = TorConfig(tor_process_protocol.tor_protocol) + public_port = 80 + + hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, + data_dir=os.path.join(datadir, 'bouncer')) + + d = hs_endpoint.listen(ooniBouncer) + d.addCallback(setup_complete) + d.addErrback(txSetupFailed) def startTor(): def updates(prog, tag, summary): @@ -87,6 +106,8 @@ def startTor(): else: d = launch_tor(torconfig, reactor, progress_updates=updates) d.addCallback(setupCollector, datadir) + if ooniBouncer: + d.addCallback(setupBouncer, datadir) d.addErrback(txSetupFailed) if platformType == "win32": @@ -110,7 +131,9 @@ else: if config.main.tor_hidden_service: startTor() else: - reactor.listenTCP(8888, ooniBackend, interface="127.0.0.1") + if ooniBouncer: + reactor.listenTCP(8888, ooniBouncer, interface="127.0.0.1") + reactor.listenTCP(8889, ooniBackend, interface="127.0.0.1") self.startReactor(None, self.oldstdout, self.oldstderr) self.removePID(self.config['pidfile']) if os.path.exists(tempfile.gettempdir()):

1 0

[oonib/master] Remove all nettest API related code.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit f9181caa8d5805cec51b7ce37277f32553c726fa Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 15:03:07 2013 +0200 Remove all nettest API related code. --- oonib/nettest/api.py | 10 ---------- oonib/nettest/handlers.py | 27 --------------------------- 2 files changed, 37 deletions(-) diff --git a/oonib/nettest/__init__.py b/oonib/nettest/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/oonib/nettest/api.py b/oonib/nettest/api.py deleted file mode 100644 index 758b31d..0000000 --- a/oonib/nettest/api.py +++ /dev/null @@ -1,10 +0,0 @@ -from cyclone import web -from oonib.nettest import handlers -from oonib import config - -nettestAPI = [ - (r"/nettest", handlers.NetTestListHandler), - (r"/nettest/([a-f0-9]{64})", handlers.NetTestDescHandler), - (r"/nettest/([a-f0-9]{64})/py$", web.StaticFileHandler, {"path": - config.main.nettest_dir}), -] diff --git a/oonib/nettest/handlers.py b/oonib/nettest/handlers.py deleted file mode 100644 index b3ceed4..0000000 --- a/oonib/nettest/handlers.py +++ /dev/null @@ -1,27 +0,0 @@ -import glob -import json -import os -import yaml - -from oonib.handlers import OONIBHandler -from oonib import config, log - -class NetTestDescHandler(OONIBHandler): - def get(self, netTestID): - bn = os.path.basename(netTestID) + ".desc" - try: - with open(os.path.join(config.main.nettest_dir, bn)) as f: - response = {} - netTestDesc = yaml.safe_load(f) - for k in ['name', 'description', 'version', 'author', 'date']: - response[k] = netTestDesc[k] - self.write(response) - except IOError: - log.err("No NetTest Descriptor found for id %s" % netTestID) - self.set_status(404) - self.write({'error': 'missing-nettest'}) - - except Exception, e: - log.err("Invalid NetTest Descriptor found for id %s" % netTestID) - self.set_status(500) - self.write({'error': 'invalid-nettest-descriptor'})

1 0

[oonib/master] Simplify filterHelperAddresses
by art＠torproject.org 11 Sep '13

11 Sep '13

commit c4f3a34ae559419cfc8a6b9e95ab1b60ab480ee9 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Sep 9 14:47:19 2013 +0200 Simplify filterHelperAddresses --- oonib/bouncer/handlers.py | 67 +++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 9e987f0..710eb6c 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -9,6 +9,18 @@ class Bouncer(object): def __init__(self): self.knownHelpers = {} self.updateKnownHelpers() + self.updateKnownCollectors() + + def updateKnownCollectors(self): + """ + Returns the list of all known collectors + """ + self.knownCollectors = [] + with open(config.main.bouncer_file) as f: + bouncerFile = yaml.safe_load(f) + for collectorName, helpers in bouncerFile['collector'].items(): + if collectorName not in self.knownCollectors: + self.knownCollectors.append(collectorName) def updateKnownHelpers(self): with open(config.main.bouncer_file) as f: @@ -76,43 +88,26 @@ class Bouncer(object): if no valid helper was found """ - result = {} - for helper_name in requested_helpers: - for collector, helper_address in self.getHelperAddresses(helper_name).items(): - if collector not in result.keys(): - result[collector] = {} - result[collector][helper_name] = helper_address - - # { - # 'foo.onion': {'some-helper': 'some-address'}, - # 'foo2.onion': {'some-helper': 'some-addres2'} - # } - response = {} - default_collector = None - max_helper_count = 0 - for collector, helpers in result.items(): - if len(helpers) > max_helper_count: - default_collector = collector - else: - continue - - response['default'] = {} - response['default']['collector'] = default_collector - - if not len(result[default_collector]) == len(requested_helpers): - found_helpers = set(result[default_collector].items()) - for missing_helper in found_helpers.difference(requested_helpers): - collector, address = random.choice(self.getHelperAddress(missing_helper).items()) - response[missing_helper] = {} - response[missing_helper]['collector'] = collector - response[missing_helper]['address'] = address - - for name, address in result[default_collector].items(): - response[name] = {} - response[name]['address'] = address - response[name]['collector'] = default_collector - + for helper_name in requested_helpers: + try: + # If we can, try to pick the same collector. + choices = self.getHelperAddresses(helper_name) + for item in response.values(): + if item['collector'] in choices.keys(): + choice = item + continue + # Or default to a random selection + else: + c,h = random.choice(choices.items()) + choice = {'collector': c, 'address': h} + response[helper_name] = choice + + except e.TestHelperNotFound: + response = {'error': 'test-helper-not-found'} + return response + + response['default'] = {'collector': random.choice(self.knownCollectors)} return response class BouncerQueryHandler(OONIBHandler):

1 0

[oonib/master] Should be break; not continue
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 9d61639c0f2783c37bb1037d55d7670bc43e7d01 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Sep 9 14:58:55 2013 +0200 Should be break; not continue --- oonib/bouncer/handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 710eb6c..5033fad 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -96,7 +96,7 @@ class Bouncer(object): for item in response.values(): if item['collector'] in choices.keys(): choice = item - continue + break # Or default to a random selection else: c,h = random.choice(choices.items())

1 0

[oonib/master] OONIB release candidate 2 version 1.0.0
by art＠torproject.org 11 Sep '13

11 Sep '13

commit fb493733f237ae637d17a754938d40e7f05f1fb9 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 17:15:12 2013 +0200 OONIB release candidate 2 version 1.0.0 --- oonib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oonib/__init__.py b/oonib/__init__.py index e01b7b3..4473fd8 100644 --- a/oonib/__init__.py +++ b/oonib/__init__.py @@ -3,7 +3,7 @@ In here we shall keep track of all variables and objects that should be instantiated only once and be common to pieces of GLBackend code. """ -__version__ = '1.0.0-rc1' +__version__ = '1.0.0-rc2' __all__ = ['Storage', 'randomStr']

1 0