[tor-commits] [oonib/master] Return the test helper address of the requested TH
art at torproject.org
art at torproject.org
Wed Sep 11 09:13:52 UTC 2013
commit cde886b1facaab9a646ccaa7c3f942ca542fd177
Author: Arturo Filastò <art at fuffa.org>
Date: Mon Aug 19 16:45:18 2013 +0200
Return the test helper address of the requested TH
* Refactor bouncer into a separate object
---
README.md | 2 +-
data/bouncer.yaml | 2 ++
oonib.conf.example | 35 ++++++++++--------
oonib/bouncer/handlers.py | 86 +++++++++++++++++++++++++++++++++++----------
oonib/deck/handlers.py | 2 ++
oonib/errors.py | 18 ++++++++++
oonib/oonibackend.py | 36 +++++++++----------
oonib/report/handlers.py | 24 +++++++++++--
8 files changed, 150 insertions(+), 55 deletions(-)
diff --git a/README.md b/README.md
index 75b60a9..b2d13b9 100644
--- a/README.md
+++ b/README.md
@@ -109,7 +109,7 @@ The following iptables commands will map connections on low ports to those
bound by oonib:
```
-# Map port 80 to config.helpers.http_return_request.port (default: 57001)
+# Map port 80 to config.helpers['http-return-json-headers'].port (default: 57001)
iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 57001
# Map port 443 to config.helpers.ssl.port (default: 57006)
iptables -t nat -A PREROUTING -p tcp -m tcp --dport 443 -j REDIRECT --to-ports 57006
diff --git a/data/bouncer.yaml b/data/bouncer.yaml
index ba54898..7709e40 100644
--- a/data/bouncer.yaml
+++ b/data/bouncer.yaml
@@ -1,3 +1,5 @@
collector:
httpo://nkvphnp3p6agi5qq.onion:
test-helper: {dns: 93.95.227.200, http-return-json-headers: 'http://93.95.227.200'}
+ httpo://nkvphnp3p6agi5q1.onion:
+ test-helper: {dns: 93.22.227.200, http-return-json-headers: 'http://93.95.227.220'}
diff --git a/oonib.conf.example b/oonib.conf.example
index 977b5dc..323b2af 100644
--- a/oonib.conf.example
+++ b/oonib.conf.example
@@ -6,48 +6,53 @@ main:
policy_file: data/policy.yaml
bouncer_file: data/bouncer.yaml
- logfile: Null
- tor_datadir: Null
+ logfile: null
+ tor_datadir: null
database_uri: 'sqlite://oonib_test_db.db'
db_threadpool_size: 10
- tor_binary: Null
+ tor_binary: null
socks_port: 9055
tor2webmode: false
pidfile: 'oonib.pid'
nodaemon: true
- originalname: Null
- chroot: Null
+ originalname: null
+ chroot: null
rundir: .
- umask: Null
- euid: Null
- uid: Null
- gid: Null
- uuid: Null
+ umask: null
+ euid: null
+ uid: null
+ gid: null
+ uuid: null
no_save: true
- profile: Null
+ profile: null
debug: false
stale_time: 3600
tor_hidden_service: true
helpers:
- http_return_request:
+ http-return-json-headers:
+ address: null
port: 57001
server_version: Apache
- tcp_echo:
+ tcp-echo:
+ address: null
port: 57002
daphn3:
- yaml_file: Null
- pcap_file: Null
+ address: null
+ yaml_file: null
+ pcap_file: null
port: 57003
dns:
+ address: null
udp_port: 57004
tcp_port: 57005
ssl:
+ address: null
private_key: 'private.key'
certificate: 'certificate.crt'
port: 57006
diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py
index 230f618..44f1c63 100644
--- a/oonib/bouncer/handlers.py
+++ b/oonib/bouncer/handlers.py
@@ -4,10 +4,72 @@ import yaml
from oonib.handlers import OONIBHandler
from oonib import config
-class BouncerQueryHandler(OONIBHandler):
- def initialize(self):
+class Bouncer(object):
+ def __init__(self):
self.knownHelpers = {}
self.updateKnownHelpers()
+
+ def updateKnownHelpers(self):
+ with open(config.main.bouncer_file) as f:
+ bouncerFile = yaml.safe_load(f)
+ for collectorName, helpers in bouncerFile['collector'].items():
+ for helperName, helperAddress in helpers['test-helper'].items():
+ if helperName not in self.knownHelpers.keys():
+ self.knownHelpers[helperName] = []
+
+ self.knownHelpers[helperName].append({
+ 'collector-name': collectorName,
+ 'helper-address': helperAddress
+ })
+
+ def getHelperAddresses(self, helper_name):
+ """
+ Returns a dict keyed on the collector address of known test helpers.
+ example:
+ {
+ 'httpo://thirteenchars1.onion': '127.0.0.1',
+ 'httpo://thirteenchars2.onion': '127.0.0.2',
+ 'httpo://thirteenchars3.onion': '127.0.0.3'
+ }
+ """
+ try:
+ helpers = self.knownHelpers[helper_name]
+ except KeyError:
+ raise e.NoHelperFound
+
+ helpers_dict = {}
+ for helper in helpers:
+ helpers_dict[helper['collector-name']] = helper['helper-address']
+
+ return helpers_dict
+
+ def filterHelperAddresses(self, requested_helpers):
+ """
+ Returns a dict of collectors that support all the requested_helpers.
+
+ Example:
+ requested_helpers = ['a', 'b', 'c']
+ will return:
+ {
+ 'httpo://thirteenchars1.onion': {
+ 'a': '127.0.0.1',
+ 'b': 'http://127.0.0.1',
+ 'c': '127.0.0.1:590',
+ }
+ }
+
+ """
+ result = {}
+ for helper_name in requested_helpers:
+ for collector, helper_address in self.getHelperAddresses(helper_name).items():
+ if collector not in result.keys():
+ result[collector] = {}
+ result[collector][helper_name] = helper_address
+ return result
+
+class BouncerQueryHandler(OONIBHandler):
+ def initialize(self):
+ self.bouncer = Bouncer()
def updateKnownHelpers(self):
with open(config.main.bouncer_file) as f:
@@ -26,25 +88,13 @@ class BouncerQueryHandler(OONIBHandler):
try:
query = json.loads(self.request.body)
except ValueError:
- self.set_status(400)
- self.write(json.dumps({'error': 'invalid-request'}))
- return
+ raise e.InvalidRequest
try:
- helpers = query['test-helpers']
+ requested_helpers = query['test-helpers']
except KeyError:
- self.set_status(400)
- self.write(json.dumps({'error': 'test-helpers-key-missing'}))
- return
+ raise e.TestHelpersKeyMissing
response = {}
- response['collector'] = {}
- for helperName in helpers:
- if helperName in self.knownHelpers.keys():
- chosen = random.choice(self.knownHelpers[helperName])
- collectorName, helperAddress = chosen['collector-name'], chosen['helper-address']
- if not collectorName in response['collector'].keys():
- response['collector'][collectorName] = {'test-helper': {}}
- response['collector'][collectorName]['test-helper'][helperName] = helperAddress
-
+ response['collector'] = self.bouncer.filterHelperAddresses(requested_helpers)
self.write(response)
diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py
index d4b3bbe..558ca6a 100644
--- a/oonib/deck/handlers.py
+++ b/oonib/deck/handlers.py
@@ -20,10 +20,12 @@ class DeckDescHandler(OONIBHandler):
for k in ['name', 'description', 'version', 'author', 'date']:
response[k] = deckDesc[k]
self.write(response)
+
except IOError:
log.err("Deck %s missing" % deckID)
self.set_status(404)
self.write({'error': 'missing-deck'})
+
except KeyError:
self.set_status(400)
log.err("Deck %s missing required keys!" % deckID)
diff --git a/oonib/errors.py b/oonib/errors.py
index 24a06fc..1dc8539 100644
--- a/oonib/errors.py
+++ b/oonib/errors.py
@@ -3,6 +3,13 @@ from cyclone.web import HTTPError
class OONIBError(HTTPError):
pass
+class InvalidRequest(OONIBError):
+ pass
+
+class NoHelperFound(OONIBError):
+ status_code = 404
+ log_message = 'no-helper-found'
+
class InvalidInputHash(OONIBError):
status_code = 400
log_message = 'invalid-input-hash'
@@ -38,3 +45,14 @@ class InvalidReportHeader(OONIBError):
class ReportNotFound(OONIBError):
status_code = 404
log_message = "report-not-found"
+
+class NoValidCollector(OONIBError):
+ pass
+
+class TestHelpersKeyMissing(OONIBError):
+ status_code = 400
+ log_message = "test-helpers-key-missing"
+
+class TestHelperNotFound(OONIBError):
+ status_code = 400
+ log_message = "test-helper-not-found"
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index afd961f..c5f96ee 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -30,45 +30,45 @@ else:
serviceCollection = service.IServiceCollection(application)
-if config.helpers.ssl.port:
- print "Starting SSL helper on %s" % config.helpers.ssl.port
- ssl_helper = internet.SSLServer(int(config.helpers.ssl.port),
+if config.helpers['ssl'].port:
+ print "Starting SSL helper on %s" % config.helpers['ssl'].port
+ ssl_helper = internet.SSLServer(int(config.helpers['ssl'].port),
http_helpers.HTTPReturnJSONHeadersHelper(),
ssl_helpers.SSLContext(config))
ssl_helper.setServiceParent(serviceCollection)
# Start the DNS Server related services
-if config.helpers.dns.tcp_port:
- print "Starting TCP DNS Helper on %s" % config.helpers.dns.tcp_port
- tcp_dns_helper = internet.TCPServer(int(config.helpers.dns.tcp_port),
+if config.helpers['dns'].tcp_port:
+ print "Starting TCP DNS Helper on %s" % config.helpers['dns'].tcp_port
+ tcp_dns_helper = internet.TCPServer(int(config.helpers['dns'].tcp_port),
dns_helpers.DNSTestHelper())
tcp_dns_helper.setServiceParent(serviceCollection)
-if config.helpers.dns.udp_port:
- print "Starting UDP DNS Helper on %s" % config.helpers.dns.udp_port
+if config.helpers['dns'].udp_port:
+ print "Starting UDP DNS Helper on %s" % config.helpers['dns'].udp_port
udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper())
- udp_dns_helper = internet.UDPServer(int(config.helpers.dns.udp_port),
+ udp_dns_helper = internet.UDPServer(int(config.helpers['dns'].udp_port),
udp_dns_factory)
udp_dns_helper.setServiceParent(serviceCollection)
# XXX this needs to be ported
# Start the OONI daphn3 backend
-if config.helpers.daphn3.port:
- print "Starting Daphn3 helper on %s" % config.helpers.daphn3.port
- daphn3_helper = internet.TCPServer(int(config.helpers.daphn3.port),
+if config.helpers['daphn3'].port:
+ print "Starting Daphn3 helper on %s" % config.helpers['daphn3'].port
+ daphn3_helper = internet.TCPServer(int(config.helpers['daphn3'].port),
tcp_helpers.Daphn3Server())
daphn3_helper.setServiceParent(serviceCollection)
-if config.helpers.tcp_echo.port:
- print "Starting TCP echo helper on %s" % config.helpers.tcp_echo.port
- tcp_echo_helper = internet.TCPServer(int(config.helpers.tcp_echo.port),
+if config.helpers['tcp-echo'].port:
+ print "Starting TCP echo helper on %s" % config.helpers['tcp-echo'].port
+ tcp_echo_helper = internet.TCPServer(int(config.helpers['tcp-echo'].port),
tcp_helpers.TCPEchoHelper())
tcp_echo_helper.setServiceParent(serviceCollection)
-if config.helpers.http_return_request.port:
- print "Starting HTTP return request helper on %s" % config.helpers.http_return_request.port
+if config.helpers['http-return-json-headers'].port:
+ print "Starting HTTP return request helper on %s" % config.helpers['http-return-json-headers'].port
http_return_request_helper = internet.TCPServer(
- int(config.helpers.http_return_request.port),
+ int(config.helpers['http-return-json-headers'].port),
http_helpers.HTTPReturnJSONHeadersHelper())
http_return_request_helper.setServiceParent(serviceCollection)
diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py
index 75ec55d..2441ddf 100644
--- a/oonib/report/handlers.py
+++ b/oonib/report/handlers.py
@@ -70,6 +70,13 @@ def parseNewReportRequest(request):
continue
else:
raise InvalidRequestField(k)
+
+ try:
+ test_helper = parsed_request['test_helper']
+ if not re.match(regexp, str(test_helper)):
+ raise InvalidRequestField('test_helper')
+ except KeyError:
+ pass
return parsed_request
@@ -127,7 +134,6 @@ class NewReportHandlerFile(OONIBHandler):
policy = Policy()
policy.validateInputHash(self.inputHash)
policy.validateNettest(self.testName)
- # XXX add support for version checking too.
def post(self):
"""
@@ -219,9 +225,21 @@ class NewReportHandlerFile(OONIBHandler):
# random nonce
report_filename = os.path.join(config.main.report_dir, report_id)
- response = {'backend_version': config.backend_version,
- 'report_id': report_id
+ response = {
+ 'backend_version': config.backend_version,
+ 'report_id': report_id
}
+
+ try:
+ requested_helper = report_data['test_helper']
+ except KeyError:
+ pass
+
+ if requested_helper:
+ try:
+ response['test_helper_address'] = config.helpers[requested_helper].address
+ except KeyError:
+ raise e.TestHelperNotFound
config.reports[report_id] = time.time()
More information about the tor-commits
mailing list