commit 4d76c781a67b77d0c4115b011ee6257cf933c4ce
Author: aagbsn <aagbsn(a)extc.org>
Date: Wed Aug 27 14:13:48 2014 +0000
Refactor oonib's usage of twisted application API
---
bin/oonib | 16 +++---
oonib/onion.py | 96 ++++++++++++++++++++++++++++++++++
oonib/oonibackend.py | 73 +++++++++++++++++++++-----
oonib/runner.py | 140 --------------------------------------------------
4 files changed, 166 insertions(+), 159 deletions(-)
diff --git a/bin/oonib b/bin/oonib
index 7fb1c35..fcb0c72 100755
--- a/bin/oonib
+++ b/bin/oonib
@@ -12,6 +12,7 @@ sys.path.insert(0, os.path.abspath(os.getcwd()))
from oonib import errors as e
from oonib.config import config
+from oonib.log import LoggerFactory
try:
config.load()
@@ -36,9 +37,6 @@ except e.InvalidDeckDirectory, path:
print "Invalid deck directory: %s" % path
sys.exit(6)
-from oonib import runner
-from oonib.oonibackend import application
-
if config.main.chroot:
sys.argv.append('--chroot')
sys.argv.append(config.chroot)
@@ -46,6 +44,12 @@ if config.main.chroot:
if not config.main.nodaemon:
sys.argv.append('-y')
-def runApp(config):
- runner.OBaseRunner(config).run()
-runApp(config.main)
+from oonib.oonibackend import application
+
+from twisted.scripts._twistd_unix import UnixApplicationRunner
+class OBaseRunner(UnixApplicationRunner):
+ temporary_data_dir = None
+ def createOrGetApplication(self):
+ return application
+OBaseRunner.loggerFactory = LoggerFactory
+OBaseRunner(config.main).run()
diff --git a/oonib/onion.py b/oonib/onion.py
new file mode 100644
index 0000000..1950a41
--- /dev/null
+++ b/oonib/onion.py
@@ -0,0 +1,96 @@
+import tempfile
+from oonib import log
+from oonib.config import config
+from twisted.internet import reactor, endpoints
+import os
+
+from random import randint
+import socket
+
+from txtorcon import TCPHiddenServiceEndpoint, TorConfig
+from txtorcon import launch_tor
+
+from txtorcon import __version__ as txtorcon_version
+if tuple(map(int, txtorcon_version.split('.'))) < (0, 9, 0):
+ """
+ Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint
+ listens on all interfaces by default.
+ """
+ def create_listener(self, proto):
+ self._update_onion(self.hiddenservice.dir)
+ self.tcp_endpoint = endpoints.TCP4ServerEndpoint(self.reactor,
+ self.listen_port,
+ interface='127.0.0.1')
+ d = self.tcp_endpoint.listen(self.protocolfactory)
+ d.addCallback(self._add_attributes).addErrback(self._retry_local_port)
+ return d
+ TCPHiddenServiceEndpoint._create_listener = create_listener
+
+def randomFreePort(addr="127.0.0.1"):
+ """
+ Args:
+
+ addr (str): the IP address to attempt to bind to.
+
+ Returns an int representing the free port number at the moment of calling
+
+ Note: there is no guarantee that some other application will attempt to
+ bind to this port once this function has been called.
+ """
+ free = False
+ while not free:
+ port = randint(1024, 65535)
+ s = socket.socket()
+ try:
+ s.bind((addr, port))
+ free = True
+ except:
+ pass
+ s.close()
+ return port
+
+def txSetupFailed(failure):
+ log.err("Setup failed")
+ log.exception(failure)
+
+def startTor(torconfig):
+ def updates(prog, tag, summary):
+ print("%d%%: %s" % (prog, summary))
+
+ if config.main.socks_port:
+ torconfig.SocksPort = config.main.socks_port
+ if config.main.control_port:
+ torconfig.ControlPort = config.main.control_port
+ if config.main.tor2webmode:
+ torconfig.Tor2webMode = 1
+ torconfig.CircuitBuildTimeout = 60
+ if config.main.tor_datadir is None:
+ temporary_data_dir = tempfile.mkdtemp()
+ log.warn("Option 'tor_datadir' in oonib.conf is unspecified!")
+ log.warn("Using %s" % temporary_data_dir)
+ torconfig.DataDirectory = temporary_data_dir
+ else:
+ if os.path.exists(config.main.tor_datadir):
+ torconfig.DataDirectory = os.path.abspath(config.main.tor_datadir)
+ else:
+ raise Exception
+ torconfig.save()
+ if not hasattr(torconfig, 'ControlPort'):
+ control_port = int(randomFreePort())
+ torconfig.ControlPort = control_port
+ config.main.control_port = control_port
+
+ if not hasattr(torconfig, 'SocksPort'):
+ socks_port = int(randomFreePort())
+ torconfig.SocksPort = socks_port
+ config.main.socks_port = socks_port
+
+ torconfig.save()
+
+ if config.main.tor_binary is not None:
+ d = launch_tor(torconfig, reactor,
+ tor_binary=config.main.tor_binary,
+ progress_updates=updates)
+ else:
+ d = launch_tor(torconfig, reactor, progress_updates=updates)
+ return d
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 2a9fa7d..ca51f45 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -7,13 +7,19 @@
# In here we start all the test helpers that are required by ooniprobe and
# start the report collector
-from twisted.application import internet, service
-from twisted.names import dns
-
+from oonib.api import ooniBackend, ooniBouncer
+from oonib.config import config
+from oonib.onion import startTor
from oonib.testhelpers import dns_helpers, ssl_helpers
from oonib.testhelpers import http_helpers, tcp_helpers
-from oonib.config import config
+import os
+
+from twisted.application import internet, service
+from twisted.internet import reactor
+from twisted.names import dns
+
+from txtorcon import TCPHiddenServiceEndpoint, TorConfig
if config.main.uid and config.main.gid:
application = service.Application('oonibackend', uid=config.main.uid,
@@ -21,28 +27,28 @@ if config.main.uid and config.main.gid:
else:
application = service.Application('oonibackend')
-serviceCollection = service.IServiceCollection(application)
+multiService = service.MultiService()
if config.helpers['ssl'].port:
print "Starting SSL helper on %s" % config.helpers['ssl'].port
ssl_helper = internet.SSLServer(int(config.helpers['ssl'].port),
http_helpers.HTTPReturnJSONHeadersHelper(),
ssl_helpers.SSLContext(config))
- ssl_helper.setServiceParent(serviceCollection)
+ multiService.addService(ssl_helper)
# Start the DNS Server related services
if config.helpers['dns'].tcp_port:
print "Starting TCP DNS Helper on %s" % config.helpers['dns'].tcp_port
tcp_dns_helper = internet.TCPServer(int(config.helpers['dns'].tcp_port),
dns_helpers.DNSTestHelper())
- tcp_dns_helper.setServiceParent(serviceCollection)
+ multiService.addService(tcp_dns_helper)
if config.helpers['dns'].udp_port:
print "Starting UDP DNS Helper on %s" % config.helpers['dns'].udp_port
udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper())
udp_dns_helper = internet.UDPServer(int(config.helpers['dns'].udp_port),
udp_dns_factory)
- udp_dns_helper.setServiceParent(serviceCollection)
+ multiService.addService(udp_dns_helper)
if config.helpers['dns_discovery'].udp_port:
print ("Starting UDP DNS Discovery Helper on %s" %
@@ -51,14 +57,14 @@ if config.helpers['dns_discovery'].udp_port:
dns.DNSDatagramProtocol(
dns_helpers.DNSResolverDiscovery()
))
- udp_dns_discovery.setServiceParent(serviceCollection)
+ multiService.addService(udp_dns_discovery)
if config.helpers['dns_discovery'].tcp_port:
print ("Starting TCP DNS Discovery Helper on %s" %
config.helpers['dns_discovery'].tcp_port)
tcp_dns_discovery = internet.TCPServer(int(config.helpers['dns_discovery'].tcp_port),
dns_helpers.DNSResolverDiscovery())
- tcp_dns_discovery.setServiceParent(serviceCollection)
+ multiService.addService(tcp_dns_discovery)
# XXX this needs to be ported
@@ -67,18 +73,59 @@ if config.helpers['daphn3'].port:
print "Starting Daphn3 helper on %s" % config.helpers['daphn3'].port
daphn3_helper = internet.TCPServer(int(config.helpers['daphn3'].port),
tcp_helpers.Daphn3Server())
- daphn3_helper.setServiceParent(serviceCollection)
+ multiService.addService(daphn3_helper)
if config.helpers['tcp-echo'].port:
print "Starting TCP echo helper on %s" % config.helpers['tcp-echo'].port
tcp_echo_helper = internet.TCPServer(int(config.helpers['tcp-echo'].port),
tcp_helpers.TCPEchoHelper())
- tcp_echo_helper.setServiceParent(serviceCollection)
+ multiService.addService(tcp_echo_helper)
if config.helpers['http-return-json-headers'].port:
print "Starting HTTP return request helper on %s" % config.helpers['http-return-json-headers'].port
http_return_request_helper = internet.TCPServer(
int(config.helpers['http-return-json-headers'].port),
http_helpers.HTTPReturnJSONHeadersHelper())
- http_return_request_helper.setServiceParent(serviceCollection)
+multiService.addService(http_return_request_helper)
+
+# add the tor collector service here
+if config.main.tor_hidden_service:
+ torconfig = TorConfig()
+ d = startTor(torconfig)
+
+ def addCollector(torControlProtocol):
+ data_dir = os.path.join(torconfig.DataDirectory, 'collector')
+ collector_service = internet.StreamServerEndpointService(
+ TCPHiddenServiceEndpoint(reactor,
+ torconfig, 80,
+ hidden_service_dir=data_dir),
+ ooniBackend)
+ multiService.addService(collector_service)
+ collector_service.startService()
+ return torControlProtocol
+
+ d.addCallback(addCollector)
+
+ if ooniBouncer:
+ def addBouncer(torControlProtocol):
+ data_dir = os.path.join(torconfig.DataDirectory, 'bouncer')
+
+ bouncer_service = internet.StreamServerEndpointService(
+ TCPHiddenServiceEndpoint(reactor,
+ torconfig, 80,
+ hidden_service_dir=data_dir),
+ ooniBouncer)
+ multiService.addService(bouncer_service)
+ bouncer_service.startService()
+ return torControlProtocol
+
+ d.addCallback(addBouncer)
+else:
+ if ooniBouncer:
+ bouncer_service = internet.TCPServer(8888, ooniBouncer, interface="127.0.0.1")
+ multiService.addService(bouncer_service)
+ bouncer_service.startService()
+ collector_service = internet.TCPServer(8889, ooniBackend, interface="127.0.0.1")
+ multiService.addService(collector_service)
+ collector_service.startService()
diff --git a/oonib/runner.py b/oonib/runner.py
deleted file mode 100644
index 47ddc35..0000000
--- a/oonib/runner.py
+++ /dev/null
@@ -1,140 +0,0 @@
-"""
-In here we define a runner for the oonib backend system.
-"""
-
-from __future__ import print_function
-
-from distutils.version import LooseVersion
-import tempfile
-import os
-
-from shutil import rmtree
-
-from twisted.internet import reactor, endpoints
-from twisted.python.runtime import platformType
-
-from txtorcon import TCPHiddenServiceEndpoint, TorConfig
-from txtorcon import launch_tor
-
-from oonib.api import ooniBackend, ooniBouncer
-from oonib.config import config
-
-from oonib import oonibackend
-from oonib import log
-
-from txtorcon import __version__ as txtorcon_version
-if tuple(map(int, txtorcon_version.split('.'))) < (0, 9, 0):
- """
- Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint
- listens on all interfaces by default.
- """
- def create_listener(self, proto):
- self._update_onion(self.hiddenservice.dir)
- self.tcp_endpoint = endpoints.TCP4ServerEndpoint(self.reactor,
- self.listen_port,
- interface='127.0.0.1')
- d = self.tcp_endpoint.listen(self.protocolfactory)
- d.addCallback(self._add_attributes).addErrback(self._retry_local_port)
- return d
- TCPHiddenServiceEndpoint._create_listener = create_listener
-
-class OBaseRunner(object):
- pass
-
-if platformType == "win32":
- from twisted.scripts._twistw import WindowsApplicationRunner
-
- OBaseRunner = WindowsApplicationRunner
- # XXX Currently we don't support windows for starting a Tor Hidden Service
- log.warn(
- "Apologies! We don't support starting a Tor Hidden Service on Windows.")
-
-else:
- from twisted.scripts._twistd_unix import UnixApplicationRunner
- class OBaseRunner(UnixApplicationRunner):
- temporary_data_dir = None
-
- def txSetupFailed(self, failure):
- log.err("Setup failed")
- log.exception(failure)
-
- def setupHSEndpoint(self, tor_process_protocol, torconfig, endpoint):
- endpointName = endpoint.settings['name']
-
- def setup_complete(port):
- if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'):
- onion_uri = port.address.onion_uri
- else:
- onion_uri = port.onion_uri
- print("Exposed %s Tor hidden service "
- "on httpo://%s" % (endpointName, onion_uri))
-
- public_port = 80
- data_dir = os.path.join(torconfig.DataDirectory, endpointName)
- if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'):
- hs_endpoint = TCPHiddenServiceEndpoint(reactor,
- torconfig,
- public_port,
- hidden_service_dir=data_dir)
- else:
- hs_endpoint = TCPHiddenServiceEndpoint(reactor,
- torconfig,
- public_port,
- data_dir=data_dir)
- d = hs_endpoint.listen(endpoint)
- d.addCallback(setup_complete)
- d.addErrback(self.txSetupFailed)
- return d
-
- def startTor(self, torconfig):
- def updates(prog, tag, summary):
- print("%d%%: %s" % (prog, summary))
-
- torconfig.SocksPort = config.main.socks_port
- if config.main.tor2webmode:
- torconfig.Tor2webMode = 1
- torconfig.CircuitBuildTimeout = 60
- if config.main.tor_datadir is None:
- self.temporary_data_dir = tempfile.mkdtemp()
- log.warn("Option 'tor_datadir' in oonib.conf is unspecified!")
- log.warn("Using %s" % self.temporary_data_dir)
- torconfig.DataDirectory = self.temporary_data_dir
- else:
- torconfig.DataDirectory = config.main.tor_datadir
- torconfig.save()
- if config.main.tor_binary is not None:
- d = launch_tor(torconfig, reactor,
- tor_binary=config.main.tor_binary,
- progress_updates=updates)
- else:
- d = launch_tor(torconfig, reactor, progress_updates=updates)
- return d
-
- def postApplication(self):
- """After the application is created, start the application and run
- the reactor. After the reactor stops, clean up PID files and such.
- """
- self.startApplication(self.application)
- # This is our addition. The rest is taken from
- # twisted/scripts/_twistd_unix.py 12.2.0
- if config.main.tor_hidden_service:
- torconfig = TorConfig()
- d = self.startTor(torconfig)
- d.addCallback(self.setupHSEndpoint, torconfig, ooniBackend)
- if ooniBouncer:
- d.addCallback(self.setupHSEndpoint, torconfig, ooniBouncer)
- else:
- if ooniBouncer:
- reactor.listenTCP(8888, ooniBouncer, interface="127.0.0.1")
- reactor.listenTCP(8889, ooniBackend, interface="127.0.0.1")
- self.startReactor(None, self.oldstdout, self.oldstderr)
- self.removePID(self.config['pidfile'])
- if self.temporary_data_dir:
- log.msg("Removing temporary directory: %s"
- % self.temporary_data_dir)
- rmtree(self.temporary_data_dir, onerror=log.err)
-
- def createOrGetApplication(self):
- return oonibackend.application
-
-OBaseRunner.loggerFactory = log.LoggerFactory
