[tor-commits] [oonib/master] Refactor oonib's usage of twisted application API

commit 4d76c781a67b77d0c4115b011ee6257cf933c4ce Author: aagbsn <aagbsn@extc.org> Date: Wed Aug 27 14:13:48 2014 +0000 Refactor oonib's usage of twisted application API --- bin/oonib | 16 +++--- oonib/onion.py | 96 ++++++++++++++++++++++++++++++++++ oonib/oonibackend.py | 73 +++++++++++++++++++++----- oonib/runner.py | 140 -------------------------------------------------- 4 files changed, 166 insertions(+), 159 deletions(-) diff --git a/bin/oonib b/bin/oonib index 7fb1c35..fcb0c72 100755 --- a/bin/oonib +++ b/bin/oonib @@ -12,6 +12,7 @@ sys.path.insert(0, os.path.abspath(os.getcwd())) from oonib import errors as e from oonib.config import config +from oonib.log import LoggerFactory try: config.load() @@ -36,9 +37,6 @@ except e.InvalidDeckDirectory, path: print "Invalid deck directory: %s" % path sys.exit(6) -from oonib import runner -from oonib.oonibackend import application - if config.main.chroot: sys.argv.append('--chroot') sys.argv.append(config.chroot) @@ -46,6 +44,12 @@ if config.main.chroot: if not config.main.nodaemon: sys.argv.append('-y') -def runApp(config): - runner.OBaseRunner(config).run() -runApp(config.main) +from oonib.oonibackend import application + +from twisted.scripts._twistd_unix import UnixApplicationRunner +class OBaseRunner(UnixApplicationRunner): + temporary_data_dir = None + def createOrGetApplication(self): + return application +OBaseRunner.loggerFactory = LoggerFactory +OBaseRunner(config.main).run() diff --git a/oonib/onion.py b/oonib/onion.py new file mode 100644 index 0000000..1950a41 --- /dev/null +++ b/oonib/onion.py @@ -0,0 +1,96 @@ +import tempfile +from oonib import log +from oonib.config import config +from twisted.internet import reactor, endpoints +import os + +from random import randint +import socket + +from txtorcon import TCPHiddenServiceEndpoint, TorConfig +from txtorcon import launch_tor + +from txtorcon import __version__ as txtorcon_version +if tuple(map(int, txtorcon_version.split('.'))) < (0, 9, 0): + """ + Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint + listens on all interfaces by default. + """ + def create_listener(self, proto): + self._update_onion(self.hiddenservice.dir) + self.tcp_endpoint = endpoints.TCP4ServerEndpoint(self.reactor, + self.listen_port, + interface='127.0.0.1') + d = self.tcp_endpoint.listen(self.protocolfactory) + d.addCallback(self._add_attributes).addErrback(self._retry_local_port) + return d + TCPHiddenServiceEndpoint._create_listener = create_listener + +def randomFreePort(addr="127.0.0.1"): + """ + Args: + + addr (str): the IP address to attempt to bind to. + + Returns an int representing the free port number at the moment of calling + + Note: there is no guarantee that some other application will attempt to + bind to this port once this function has been called. + """ + free = False + while not free: + port = randint(1024, 65535) + s = socket.socket() + try: + s.bind((addr, port)) + free = True + except: + pass + s.close() + return port + +def txSetupFailed(failure): + log.err("Setup failed") + log.exception(failure) + +def startTor(torconfig): + def updates(prog, tag, summary): + print("%d%%: %s" % (prog, summary)) + + if config.main.socks_port: + torconfig.SocksPort = config.main.socks_port + if config.main.control_port: + torconfig.ControlPort = config.main.control_port + if config.main.tor2webmode: + torconfig.Tor2webMode = 1 + torconfig.CircuitBuildTimeout = 60 + if config.main.tor_datadir is None: + temporary_data_dir = tempfile.mkdtemp() + log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") + log.warn("Using %s" % temporary_data_dir) + torconfig.DataDirectory = temporary_data_dir + else: + if os.path.exists(config.main.tor_datadir): + torconfig.DataDirectory = os.path.abspath(config.main.tor_datadir) + else: + raise Exception + torconfig.save() + if not hasattr(torconfig, 'ControlPort'): + control_port = int(randomFreePort()) + torconfig.ControlPort = control_port + config.main.control_port = control_port + + if not hasattr(torconfig, 'SocksPort'): + socks_port = int(randomFreePort()) + torconfig.SocksPort = socks_port + config.main.socks_port = socks_port + + torconfig.save() + + if config.main.tor_binary is not None: + d = launch_tor(torconfig, reactor, + tor_binary=config.main.tor_binary, + progress_updates=updates) + else: + d = launch_tor(torconfig, reactor, progress_updates=updates) + return d diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 2a9fa7d..ca51f45 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -7,13 +7,19 @@ # In here we start all the test helpers that are required by ooniprobe and # start the report collector -from twisted.application import internet, service -from twisted.names import dns - +from oonib.api import ooniBackend, ooniBouncer +from oonib.config import config +from oonib.onion import startTor from oonib.testhelpers import dns_helpers, ssl_helpers from oonib.testhelpers import http_helpers, tcp_helpers -from oonib.config import config +import os + +from twisted.application import internet, service +from twisted.internet import reactor +from twisted.names import dns + +from txtorcon import TCPHiddenServiceEndpoint, TorConfig if config.main.uid and config.main.gid: application = service.Application('oonibackend', uid=config.main.uid, @@ -21,28 +27,28 @@ if config.main.uid and config.main.gid: else: application = service.Application('oonibackend') -serviceCollection = service.IServiceCollection(application) +multiService = service.MultiService() if config.helpers['ssl'].port: print "Starting SSL helper on %s" % config.helpers['ssl'].port ssl_helper = internet.SSLServer(int(config.helpers['ssl'].port), http_helpers.HTTPReturnJSONHeadersHelper(), ssl_helpers.SSLContext(config)) - ssl_helper.setServiceParent(serviceCollection) + multiService.addService(ssl_helper) # Start the DNS Server related services if config.helpers['dns'].tcp_port: print "Starting TCP DNS Helper on %s" % config.helpers['dns'].tcp_port tcp_dns_helper = internet.TCPServer(int(config.helpers['dns'].tcp_port), dns_helpers.DNSTestHelper()) - tcp_dns_helper.setServiceParent(serviceCollection) + multiService.addService(tcp_dns_helper) if config.helpers['dns'].udp_port: print "Starting UDP DNS Helper on %s" % config.helpers['dns'].udp_port udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper()) udp_dns_helper = internet.UDPServer(int(config.helpers['dns'].udp_port), udp_dns_factory) - udp_dns_helper.setServiceParent(serviceCollection) + multiService.addService(udp_dns_helper) if config.helpers['dns_discovery'].udp_port: print ("Starting UDP DNS Discovery Helper on %s" % @@ -51,14 +57,14 @@ if config.helpers['dns_discovery'].udp_port: dns.DNSDatagramProtocol( dns_helpers.DNSResolverDiscovery() )) - udp_dns_discovery.setServiceParent(serviceCollection) + multiService.addService(udp_dns_discovery) if config.helpers['dns_discovery'].tcp_port: print ("Starting TCP DNS Discovery Helper on %s" % config.helpers['dns_discovery'].tcp_port) tcp_dns_discovery = internet.TCPServer(int(config.helpers['dns_discovery'].tcp_port), dns_helpers.DNSResolverDiscovery()) - tcp_dns_discovery.setServiceParent(serviceCollection) + multiService.addService(tcp_dns_discovery) # XXX this needs to be ported @@ -67,18 +73,59 @@ if config.helpers['daphn3'].port: print "Starting Daphn3 helper on %s" % config.helpers['daphn3'].port daphn3_helper = internet.TCPServer(int(config.helpers['daphn3'].port), tcp_helpers.Daphn3Server()) - daphn3_helper.setServiceParent(serviceCollection) + multiService.addService(daphn3_helper) if config.helpers['tcp-echo'].port: print "Starting TCP echo helper on %s" % config.helpers['tcp-echo'].port tcp_echo_helper = internet.TCPServer(int(config.helpers['tcp-echo'].port), tcp_helpers.TCPEchoHelper()) - tcp_echo_helper.setServiceParent(serviceCollection) + multiService.addService(tcp_echo_helper) if config.helpers['http-return-json-headers'].port: print "Starting HTTP return request helper on %s" % config.helpers['http-return-json-headers'].port http_return_request_helper = internet.TCPServer( int(config.helpers['http-return-json-headers'].port), http_helpers.HTTPReturnJSONHeadersHelper()) - http_return_request_helper.setServiceParent(serviceCollection) +multiService.addService(http_return_request_helper) + +# add the tor collector service here +if config.main.tor_hidden_service: + torconfig = TorConfig() + d = startTor(torconfig) + + def addCollector(torControlProtocol): + data_dir = os.path.join(torconfig.DataDirectory, 'collector') + collector_service = internet.StreamServerEndpointService( + TCPHiddenServiceEndpoint(reactor, + torconfig, 80, + hidden_service_dir=data_dir), + ooniBackend) + multiService.addService(collector_service) + collector_service.startService() + return torControlProtocol + + d.addCallback(addCollector) + + if ooniBouncer: + def addBouncer(torControlProtocol): + data_dir = os.path.join(torconfig.DataDirectory, 'bouncer') + + bouncer_service = internet.StreamServerEndpointService( + TCPHiddenServiceEndpoint(reactor, + torconfig, 80, + hidden_service_dir=data_dir), + ooniBouncer) + multiService.addService(bouncer_service) + bouncer_service.startService() + return torControlProtocol + + d.addCallback(addBouncer) +else: + if ooniBouncer: + bouncer_service = internet.TCPServer(8888, ooniBouncer, interface="127.0.0.1") + multiService.addService(bouncer_service) + bouncer_service.startService() + collector_service = internet.TCPServer(8889, ooniBackend, interface="127.0.0.1") + multiService.addService(collector_service) + collector_service.startService() diff --git a/oonib/runner.py b/oonib/runner.py deleted file mode 100644 index 47ddc35..0000000 --- a/oonib/runner.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -In here we define a runner for the oonib backend system. -""" - -from __future__ import print_function - -from distutils.version import LooseVersion -import tempfile -import os - -from shutil import rmtree - -from twisted.internet import reactor, endpoints -from twisted.python.runtime import platformType - -from txtorcon import TCPHiddenServiceEndpoint, TorConfig -from txtorcon import launch_tor - -from oonib.api import ooniBackend, ooniBouncer -from oonib.config import config - -from oonib import oonibackend -from oonib import log - -from txtorcon import __version__ as txtorcon_version -if tuple(map(int, txtorcon_version.split('.'))) < (0, 9, 0): - """ - Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint - listens on all interfaces by default. - """ - def create_listener(self, proto): - self._update_onion(self.hiddenservice.dir) - self.tcp_endpoint = endpoints.TCP4ServerEndpoint(self.reactor, - self.listen_port, - interface='127.0.0.1') - d = self.tcp_endpoint.listen(self.protocolfactory) - d.addCallback(self._add_attributes).addErrback(self._retry_local_port) - return d - TCPHiddenServiceEndpoint._create_listener = create_listener - -class OBaseRunner(object): - pass - -if platformType == "win32": - from twisted.scripts._twistw import WindowsApplicationRunner - - OBaseRunner = WindowsApplicationRunner - # XXX Currently we don't support windows for starting a Tor Hidden Service - log.warn( - "Apologies! We don't support starting a Tor Hidden Service on Windows.") - -else: - from twisted.scripts._twistd_unix import UnixApplicationRunner - class OBaseRunner(UnixApplicationRunner): - temporary_data_dir = None - - def txSetupFailed(self, failure): - log.err("Setup failed") - log.exception(failure) - - def setupHSEndpoint(self, tor_process_protocol, torconfig, endpoint): - endpointName = endpoint.settings['name'] - - def setup_complete(port): - if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'): - onion_uri = port.address.onion_uri - else: - onion_uri = port.onion_uri - print("Exposed %s Tor hidden service " - "on httpo://%s" % (endpointName, onion_uri)) - - public_port = 80 - data_dir = os.path.join(torconfig.DataDirectory, endpointName) - if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'): - hs_endpoint = TCPHiddenServiceEndpoint(reactor, - torconfig, - public_port, - hidden_service_dir=data_dir) - else: - hs_endpoint = TCPHiddenServiceEndpoint(reactor, - torconfig, - public_port, - data_dir=data_dir) - d = hs_endpoint.listen(endpoint) - d.addCallback(setup_complete) - d.addErrback(self.txSetupFailed) - return d - - def startTor(self, torconfig): - def updates(prog, tag, summary): - print("%d%%: %s" % (prog, summary)) - - torconfig.SocksPort = config.main.socks_port - if config.main.tor2webmode: - torconfig.Tor2webMode = 1 - torconfig.CircuitBuildTimeout = 60 - if config.main.tor_datadir is None: - self.temporary_data_dir = tempfile.mkdtemp() - log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") - log.warn("Using %s" % self.temporary_data_dir) - torconfig.DataDirectory = self.temporary_data_dir - else: - torconfig.DataDirectory = config.main.tor_datadir - torconfig.save() - if config.main.tor_binary is not None: - d = launch_tor(torconfig, reactor, - tor_binary=config.main.tor_binary, - progress_updates=updates) - else: - d = launch_tor(torconfig, reactor, progress_updates=updates) - return d - - def postApplication(self): - """After the application is created, start the application and run - the reactor. After the reactor stops, clean up PID files and such. - """ - self.startApplication(self.application) - # This is our addition. The rest is taken from - # twisted/scripts/_twistd_unix.py 12.2.0 - if config.main.tor_hidden_service: - torconfig = TorConfig() - d = self.startTor(torconfig) - d.addCallback(self.setupHSEndpoint, torconfig, ooniBackend) - if ooniBouncer: - d.addCallback(self.setupHSEndpoint, torconfig, ooniBouncer) - else: - if ooniBouncer: - reactor.listenTCP(8888, ooniBouncer, interface="127.0.0.1") - reactor.listenTCP(8889, ooniBackend, interface="127.0.0.1") - self.startReactor(None, self.oldstdout, self.oldstderr) - self.removePID(self.config['pidfile']) - if self.temporary_data_dir: - log.msg("Removing temporary directory: %s" - % self.temporary_data_dir) - rmtree(self.temporary_data_dir, onerror=log.err) - - def createOrGetApplication(self): - return oonibackend.application - -OBaseRunner.loggerFactory = log.LoggerFactory