- tor-commits - lists.torproject.org

[oonib/master] maintain old behaviour
by art＠torproject.org 30 May '16

30 May '16

commit 5fa24c3364c4349995d3bb1ca900dc54e546f5c4 Author: Joe Landers <joe(a)joelanders.net> Date: Fri May 6 13:11:33 2016 +0200 maintain old behaviour --- oonib/oonibackend.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 0b8c4ec..1e77d6b 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -154,15 +154,20 @@ if config.main.tor_hidden_service: torconfig = TorConfig() configTor(torconfig) -if config.main.bouncer_endpoints: - for endpoint_config in config.main.bouncer_endpoints: - print "Starting bouncer with config %s" % endpoint_config - endpoint = getEndpoint(endpoint_config) - createService(endpoint, 'bouncer', endpoint_config) - -if config.main.bouncer_endpoints: - for endpoint_config in config.main.collector_endpoints: - print "Starting collector with config %s" % endpoint_config - endpoint = getEndpoint(endpoint_config) - createService(endpoint, 'collector', endpoint_config) +# this is to ensure same behaviour with an old config file +if config.main.bouncer_endpoints is None: + config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': 'bouncer'} ] + +if config.main.collector_endpoints is None: + config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 'collector'} ] + +for endpoint_config in config.main.bouncer_endpoints: + print "Starting bouncer with config %s" % endpoint_config + endpoint = getEndpoint(endpoint_config) + createService(endpoint, 'bouncer', endpoint_config) + +for endpoint_config in config.main.collector_endpoints: + print "Starting collector with config %s" % endpoint_config + endpoint = getEndpoint(endpoint_config) + createService(endpoint, 'collector', endpoint_config)

1 0

[oonib/master] easily swappable {TCP, TLS, onion} endpoints
by art＠torproject.org 30 May '16

30 May '16

commit 8ef4dc35b303d0b1e7d3e5ee0cc8a0f3e2eac240 Author: Joe Landers <joe(a)joelanders.net> Date: Wed May 4 22:45:37 2016 +0200 easily swappable {TCP, TLS, onion} endpoints --- gen-ssl-key-cert.sh | 13 ++++++ oonib.conf.example | 13 +++++- oonib/onion.py | 10 +---- oonib/oonibackend.py | 125 ++++++++++++++++++++++++++++----------------------- 4 files changed, 93 insertions(+), 68 deletions(-) diff --git a/gen-ssl-key-cert.sh b/gen-ssl-key-cert.sh new file mode 100755 index 0000000..2a50766 --- /dev/null +++ b/gen-ssl-key-cert.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +if [ ! -d private ] +then + mkdir private +fi + +openssl req -x509 -newkey rsa:2048 \ + -keyout private/ssl-key.pem -out private/ssl-cert.pem \ + -days 400 -nodes -subj '/CN=selfie' + +cat private/ssl-key.pem private/ssh-cert.pem > private/ssl-key-and-cert.pem diff --git a/oonib.conf.example b/oonib.conf.example index 7821dc1..0a5f4a5 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -7,7 +7,6 @@ main: bouncer_file: data/bouncer.yaml logfile: null - tor_datadir: null database_uri: 'sqlite://oonib_test_db.db' db_threadpool_size: 10 tor_binary: null @@ -28,7 +27,17 @@ main: debug: false stale_time: 3600 - tor_hidden_service: true + tor_hidden_service: false + tor_datadir: test_datadir + + bouncer_endpoints: + - {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"} + - {type: tcp, port: 10080} + - {type: onion, hsdir: bouncer} + + collector_endpoints: + - {type: tls, port: 11443, cert: "private/ssl-key-and-cert.pem"} + report_file_template: '{iso8601_timestamp}-{test_name}-{report_id}-{probe_asn}-{probe_cc}-probe-0.2.0.{ext}' helpers: http-return-json-headers: diff --git a/oonib/onion.py b/oonib/onion.py index c06dc6b..262e672 100644 --- a/oonib/onion.py +++ b/oonib/onion.py @@ -53,7 +53,7 @@ def txSetupFailed(failure): log.err("Setup failed") log.exception(failure) -def startTor(torconfig): +def configTor(torconfig): def updates(prog, tag, summary): print("%d%%: %s" % (prog, summary)) @@ -89,11 +89,3 @@ def startTor(torconfig): config.main.socks_port = socks_port torconfig.save() - - if config.main.tor_binary is not None: - d = launch_tor(torconfig, reactor, - tor_binary=config.main.tor_binary, - progress_updates=updates) - else: - d = launch_tor(torconfig, reactor, progress_updates=updates) - return d diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index a392b82..1c6a23a 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -11,14 +11,14 @@ from distutils.version import LooseVersion from oonib.api import ooniBackend, ooniBouncer from oonib.config import config -from oonib.onion import startTor +from oonib.onion import configTor from oonib.testhelpers import dns_helpers, ssl_helpers from oonib.testhelpers import http_helpers, tcp_helpers import os from twisted.application import internet, service -from twisted.internet import reactor +from twisted.internet import reactor, endpoints, defer, ssl, protocol from twisted.names import dns from txtorcon import TCPHiddenServiceEndpoint, TorConfig @@ -102,60 +102,71 @@ if config.helpers['http-return-json-headers'].port: multiService.addService(http_return_request_helper) http_return_request_helper.startService() -# add the tor collector service here +def getHSEndpoint(endpoint_config): + hsdir = os.path.join(torconfig.DataDirectory, endpoint_config['hsdir']) + if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'): + return TCPHiddenServiceEndpoint.global_tor(reactor, + 80, + hidden_service_dir=hsdir) + else: + return TCPHiddenServiceEndpoint.global_tor(reactor, + 80, + data_dir=hsdir) + +def getTCPEndpoint(endpoint_config): + return endpoints.TCP4ServerEndpoint(reactor, endpoint_config['port']) + +def getTLSEndpoint(endpoint_config): + with open(endpoint_config['cert'], 'r') as f: + cert_data = f.read() + certificate = ssl.PrivateCertificate.loadPEM(cert_data) + print certificate.inspect() + return endpoints.SSL4ServerEndpoint(reactor, + endpoint_config['port'], + certificate.options()) + +def getEndpoint(endpoint_config): + if endpoint_config['type'] == 'onion': + return getHSEndpoint(endpoint_config) + elif endpoint_config['type'] == 'tcp': + return getTCPEndpoint(endpoint_config) + elif endpoint_config['type'] == 'tls': + return getTLSEndpoint(endpoint_config) + else: + raise Exception("unknown endpoint type") + +class Echo(protocol.Protocol): + def dataReceived(self, data): + self.transport.write('server echoes ' + data) + +def createService(endpoint, role, endpoint_config): + if role == 'bouncer': + factory = ooniBouncer + elif role == 'collector': + factory = ooniBackend + else: + raise Exception("unknown service type") + + service = internet.StreamServerEndpointService( + endpoint, factory + ) + service.setName("-".join([endpoint_config['type'], role])) + multiService.addService(service) + service.startService() + if config.main.tor_hidden_service: torconfig = TorConfig() - d = startTor(torconfig) - - def getHSEndpoint(data_dir): - if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'): - return TCPHiddenServiceEndpoint(reactor, - torconfig, - 80, - hidden_service_dir=data_dir) - else: - return TCPHiddenServiceEndpoint(reactor, - torconfig, - 80, - data_dir=data_dir) - - def printOnionEndpoint(endpointService): - print ("Exposed %s Tor hidden service on httpo://%s" % - (endpointService.name, endpointService.endpoint.onion_uri)) - - def addCollector(torControlProtocol): - data_dir = os.path.join(torconfig.DataDirectory, 'collector') - collector_service = internet.StreamServerEndpointService( - getHSEndpoint(data_dir), ooniBackend - ) - collector_service.setName('collector') - multiService.addService(collector_service) - collector_service.startService() - return collector_service - - d.addCallback(addCollector) - d.addCallback(printOnionEndpoint) - - if ooniBouncer: - def addBouncer(torControlProtocol): - data_dir = os.path.join(torconfig.DataDirectory, 'bouncer') - bouncer_service = internet.StreamServerEndpointService( - getHSEndpoint(data_dir), ooniBouncer - ) - bouncer_service.setName('bouncer') - multiService.addService(bouncer_service) - bouncer_service.startService() - return bouncer_service - - d.addCallback(addBouncer) - d.addCallback(printOnionEndpoint) -else: - if ooniBouncer: - bouncer_service = internet.TCPServer(8888, ooniBouncer, - interface="127.0.0.1") - multiService.addService(bouncer_service) - bouncer_service.startService() - collector_service = internet.TCPServer(8889, ooniBackend, - interface="127.0.0.1") - multiService.addService(collector_service) - collector_service.startService() + configTor(torconfig) + +if config.main.bouncer_endpoints: + for endpoint_config in config.main.bouncer_endpoints: + print "Starting bouncer with config %s" % endpoint_config + endpoint = getEndpoint(endpoint_config) + createService(endpoint, 'bouncer', endpoint_config) + +if config.main.bouncer_endpoints: + for endpoint_config in config.main.collector_endpoints: + print "Starting collector with config %s" % endpoint_config + endpoint = getEndpoint(endpoint_config) + createService(endpoint, 'collector', endpoint_config) +

1 0

[oonib/master] remove another print from debugging
by art＠torproject.org 30 May '16

30 May '16

commit 2fa28488fc6dea793da147f9f1bd7a87961ec7a7 Author: Joe Landers <joe(a)joelanders.net> Date: Fri May 6 13:21:31 2016 +0200 remove another print from debugging --- oonib/oonibackend.py | 1 - 1 file changed, 1 deletion(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 1e77d6b..9c9a0fa 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -120,7 +120,6 @@ def getTLSEndpoint(endpoint_config): with open(endpoint_config['cert'], 'r') as f: cert_data = f.read() certificate = ssl.PrivateCertificate.loadPEM(cert_data) - print certificate.inspect() return endpoints.SSL4ServerEndpoint(reactor, endpoint_config['port'], certificate.options())

1 0

[oonib/master] Merge branch 'feature/https_endpoint' of github.com:TheTorProject/ooni-backend into feature/https_endpoint
by art＠torproject.org 30 May '16

30 May '16

commit 4141c6a1485a32a7c0e43161201c77150b665a3c Merge: dda3f4d 35ccea7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri May 13 12:52:28 2016 +0200 Merge branch 'feature/https_endpoint' of github.com:TheTorProject/ooni-backend into feature/https_endpoint * 'feature/https_endpoint' of github.com:TheTorProject/ooni-backend: only assume HS endpoint when tor_hidden_service: true explain + bail out when inconsistent config oonib/oonibackend.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)

1 0

[oonib/master] explain + bail out when inconsistent config
by art＠torproject.org 30 May '16

30 May '16

commit 3bcce2080b6e986c992c9cad9dc4b8e25bebaa10 Author: Joe Landers <joe(a)joelanders.net> Date: Fri May 13 09:32:17 2016 +0200 explain + bail out when inconsistent config --- oonib/oonibackend.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 9c9a0fa..3802566 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -103,6 +103,8 @@ if config.helpers['http-return-json-headers'].port: http_return_request_helper.startService() def getHSEndpoint(endpoint_config): + if torconfig is None: + raise Exception("you probably need to set tor_hidden_service: true") hsdir = os.path.join(torconfig.DataDirectory, endpoint_config['hsdir']) if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'): return TCPHiddenServiceEndpoint.global_tor(reactor, @@ -149,6 +151,7 @@ def createService(endpoint, role, endpoint_config): multiService.addService(service) service.startService() +torconfig = None if config.main.tor_hidden_service: torconfig = TorConfig() configTor(torconfig)

1 0

[oonib/master] Remove unused imports
by art＠torproject.org 30 May '16

30 May '16

commit 0bd12a10549d214fff93c4b11e14b5d62bafc0ea Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri May 13 13:18:05 2016 +0200 Remove unused imports --- oonib/oonibackend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index eaf235f..5a994e8 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -18,7 +18,7 @@ from oonib.testhelpers import http_helpers, tcp_helpers import os from twisted.application import internet, service -from twisted.internet import reactor, endpoints, defer, ssl, protocol +from twisted.internet import reactor, endpoints, ssl from twisted.names import dns from txtorcon import TCPHiddenServiceEndpoint, TorConfig

1 0

[oonib/master] Move gen-ssl-key-cert to scripts directory
by art＠torproject.org 30 May '16

30 May '16

commit dda3f4d2412ee4f2e2c681f562cf07297dbe9de9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed May 11 16:52:38 2016 +0200 Move gen-ssl-key-cert to scripts directory --- gen-ssl-key-cert.sh | 13 ------------- scripts/gen-ssl-key-cert.sh | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/gen-ssl-key-cert.sh b/gen-ssl-key-cert.sh deleted file mode 100755 index 2a50766..0000000 --- a/gen-ssl-key-cert.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -set -eu - -if [ ! -d private ] -then - mkdir private -fi - -openssl req -x509 -newkey rsa:2048 \ - -keyout private/ssl-key.pem -out private/ssl-cert.pem \ - -days 400 -nodes -subj '/CN=selfie' - -cat private/ssl-key.pem private/ssh-cert.pem > private/ssl-key-and-cert.pem diff --git a/scripts/gen-ssl-key-cert.sh b/scripts/gen-ssl-key-cert.sh new file mode 100755 index 0000000..2a50766 --- /dev/null +++ b/scripts/gen-ssl-key-cert.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +if [ ! -d private ] +then + mkdir private +fi + +openssl req -x509 -newkey rsa:2048 \ + -keyout private/ssl-key.pem -out private/ssl-cert.pem \ + -days 400 -nodes -subj '/CN=selfie' + +cat private/ssl-key.pem private/ssh-cert.pem > private/ssl-key-and-cert.pem

1 0

[oonib/master] Merge branch 'feature/https_endpoint'
by art＠torproject.org 30 May '16

30 May '16

commit 612eb5a1c7a6ea1f2f83d3eb7a9f5ffb1d68c64f Merge: 94d3452 90ca955 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri May 13 13:21:28 2016 +0200 Merge branch 'feature/https_endpoint' * feature/https_endpoint: Ensure bouncer_endpoints and collector_endpoints are by default empty lists Remove unused imports Fix typo in gen-ssl-key-cert script only assume HS endpoint when tor_hidden_service: true explain + bail out when inconsistent config Move gen-ssl-key-cert to scripts directory remove another print from debugging maintain old behaviour remove echo server i used for debugging easily swappable {TCP, TLS, onion} endpoints oonib.conf.example | 13 ++++- oonib/onion.py | 10 +--- oonib/oonibackend.py | 130 +++++++++++++++++++++++++------------------- scripts/gen-ssl-key-cert.sh | 13 +++++ 4 files changed, 98 insertions(+), 68 deletions(-)

1 0

[oonib/master] Ensure bouncer_endpoints and collector_endpoints are by default empty lists
by art＠torproject.org 30 May '16

30 May '16

commit 90ca955e970ef30c5963acce77c5f736372fc440 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri May 13 13:20:28 2016 +0200 Ensure bouncer_endpoints and collector_endpoints are by default empty lists --- oonib/oonibackend.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 5a994e8..337b797 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -163,6 +163,9 @@ if config.main.bouncer_endpoints is None and config.main.tor_hidden_service: if config.main.collector_endpoints is None and config.main.tor_hidden_service: config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 'collector'} ] +config.main.bouncer_endpoints = config.main.get('bouncer_endpoints', []) +config.main.collector_endpoints = config.main.get('collector_endpoints', []) + for endpoint_config in config.main.bouncer_endpoints: print "Starting bouncer with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) @@ -172,4 +175,3 @@ for endpoint_config in config.main.collector_endpoints: print "Starting collector with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) createService(endpoint, 'collector', endpoint_config) -

1 0

[oonib/master] Fix typo in gen-ssl-key-cert script
by art＠torproject.org 30 May '16

30 May '16

commit d7f9a4c731acd9979376fa73438d4e8afae75446 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri May 13 12:57:38 2016 +0200 Fix typo in gen-ssl-key-cert script --- scripts/gen-ssl-key-cert.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen-ssl-key-cert.sh b/scripts/gen-ssl-key-cert.sh index 2a50766..da85b58 100755 --- a/scripts/gen-ssl-key-cert.sh +++ b/scripts/gen-ssl-key-cert.sh @@ -10,4 +10,4 @@ openssl req -x509 -newkey rsa:2048 \ -keyout private/ssl-key.pem -out private/ssl-cert.pem \ -days 400 -nodes -subj '/CN=selfie' -cat private/ssl-key.pem private/ssh-cert.pem > private/ssl-key-and-cert.pem +cat private/ssl-key.pem private/ssl-cert.pem > private/ssl-key-and-cert.pem

1 0