- tor-commits - lists.torproject.org

[oonib/master] keep old config behavior with new endpoint code
by art＠torproject.org 30 May '16

30 May '16

commit 575f0aa5a44d1f851fb3d4fbb1bcf1fbf36a4f2d Author: Joe Landers <joe(a)joelanders.net> Date: Sun May 15 18:25:41 2016 +0200 keep old config behavior with new endpoint code --- oonib/onion.py | 36 ++++++++++++++++++++++++++++++++---- oonib/oonibackend.py | 42 +++++++++++++++++++----------------------- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/oonib/onion.py b/oonib/onion.py index 262e672..8abefe5 100644 --- a/oonib/onion.py +++ b/oonib/onion.py @@ -1,7 +1,7 @@ import tempfile from oonib import log from oonib.config import config -from twisted.internet import reactor, endpoints +from twisted.internet import reactor, endpoints, defer import os from random import randint @@ -53,9 +53,8 @@ def txSetupFailed(failure): log.err("Setup failed") log.exception(failure) -def configTor(torconfig): - def updates(prog, tag, summary): - print("%d%%: %s" % (prog, summary)) +def _configTor(): + torconfig = TorConfig() if config.main.socks_port: torconfig.SocksPort = config.main.socks_port @@ -89,3 +88,32 @@ def configTor(torconfig): config.main.socks_port = socks_port torconfig.save() + return torconfig + +# get_global_tor is a near-rip of that from txtorcon (so you can have some +# confidence in the logic of it), but we use our own _configTor() while +# the txtorcon function hardcodes some default values we don't want. +_global_tor_config = None +_global_tor_lock = defer.DeferredLock() +# we need the lock because we (potentially) yield several times while +# "creating" the TorConfig instance + +(a)defer.inlineCallbacks +def get_global_tor(reactor): + global _global_tor_config + global _global_tor_lock + yield _global_tor_lock.acquire() + + try: + if _global_tor_config is None: + _global_tor_config = config = _configTor() + + # start Tor launching + def updates(prog, tag, summary): + print("%d%%: %s" % (prog, summary)) + yield launch_tor(config, reactor, progress_updates=updates) + yield config.post_bootstrap + + defer.returnValue(_global_tor_config) + finally: + _global_tor_lock.release() diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 337b797..18bacf0 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -11,14 +11,14 @@ from distutils.version import LooseVersion from oonib.api import ooniBackend, ooniBouncer from oonib.config import config -from oonib.onion import configTor +from oonib.onion import get_global_tor from oonib.testhelpers import dns_helpers, ssl_helpers from oonib.testhelpers import http_helpers, tcp_helpers import os from twisted.application import internet, service -from twisted.internet import reactor, endpoints, ssl +from twisted.internet import reactor, endpoints, ssl, defer from twisted.names import dns from txtorcon import TCPHiddenServiceEndpoint, TorConfig @@ -103,15 +103,17 @@ if config.helpers['http-return-json-headers'].port: http_return_request_helper.startService() def getHSEndpoint(endpoint_config): - if torconfig is None: - raise Exception("you probably need to set tor_hidden_service: true") - hsdir = os.path.join(torconfig.DataDirectory, endpoint_config['hsdir']) + hsdir = endpoint_config['hsdir'] + hsdir = os.path.expanduser(hsdir) + hsdir = os.path.realpath(hsdir) if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'): - return TCPHiddenServiceEndpoint.global_tor(reactor, + return TCPHiddenServiceEndpoint(reactor, + get_global_tor(reactor), 80, hidden_service_dir=hsdir) else: - return TCPHiddenServiceEndpoint.global_tor(reactor, + return TCPHiddenServiceEndpoint(reactor, + get_global_tor(reactor), 80, data_dir=hsdir) @@ -151,27 +153,21 @@ def createService(endpoint, role, endpoint_config): multiService.addService(service) service.startService() -torconfig = None -if config.main.tor_hidden_service: - torconfig = TorConfig() - configTor(torconfig) - # this is to ensure same behaviour with an old config file -if config.main.bouncer_endpoints is None and config.main.tor_hidden_service: - config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': 'bouncer'} ] - -if config.main.collector_endpoints is None and config.main.tor_hidden_service: - config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': 'collector'} ] - -config.main.bouncer_endpoints = config.main.get('bouncer_endpoints', []) -config.main.collector_endpoints = config.main.get('collector_endpoints', []) - -for endpoint_config in config.main.bouncer_endpoints: +if config.main.tor_hidden_service and \ + config.main.bouncer_endpoints is None and \ + config.main.collector_endpoints is None: + bouncer_hsdir = os.path.join(config.main.tor_datadir, 'bouncer') + collector_hsdir = os.path.join(config.main.tor_datadir, 'collector') + config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': bouncer_hsdir} ] + config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': collector_hsdir} ] + +for endpoint_config in config.main.get('bouncer_endpoints'): print "Starting bouncer with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) createService(endpoint, 'bouncer', endpoint_config) -for endpoint_config in config.main.collector_endpoints: +for endpoint_config in config.main.get('collector_endpoints'): print "Starting collector with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) createService(endpoint, 'collector', endpoint_config)

1 0

[oonib/master] pass tor_binary config to txtorcon (#79)
by art＠torproject.org 30 May '16

30 May '16

commit 0b6477c7af96663d2b0d853db66e413cdd599b30 Author: Joe Landers <joe(a)joelanders.net> Date: Mon May 23 17:51:52 2016 +0200 pass tor_binary config to txtorcon (#79) --- oonib/onion.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/oonib/onion.py b/oonib/onion.py index 8abefe5..0a7a155 100644 --- a/oonib/onion.py +++ b/oonib/onion.py @@ -106,13 +106,15 @@ def get_global_tor(reactor): try: if _global_tor_config is None: - _global_tor_config = config = _configTor() + _global_tor_config = _configTor() # start Tor launching def updates(prog, tag, summary): print("%d%%: %s" % (prog, summary)) - yield launch_tor(config, reactor, progress_updates=updates) - yield config.post_bootstrap + yield launch_tor(_global_tor_config, reactor, + progress_updates=updates, + tor_binary=config.main.tor_binary) + yield _global_tor_config.post_bootstrap defer.returnValue(_global_tor_config) finally:

1 0

[oonib/master] Fix privilege shedding and daemonisation of oonibackend (#80)
by art＠torproject.org 30 May '16

30 May '16

commit 5940999a49af50ff49aeabb1103543889c598312 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 30 15:24:07 2016 +0200 Fix privilege shedding and daemonisation of oonibackend (#80) * Fix privilege shedding and daemonisation of oonibackend * Rewrite to use twistd * Launch tor only after we have forked * Set the UID of when starting tor This fixes #65 * Add gitignore files and remove empty.txt files * Drop support for super old versions of txtorcon * Remove now unnecessary hacks to maintain backward compatibility * Fix permissions on temporary tor data directory --- bin/oonib | 38 ++----- data/archive/.gitignore | 2 + data/archive/empty.txt | 2 - data/decks/.gitignore | 3 + data/inputs/.gitignore | 3 + data/reports/.gitignore | 2 + data/tor/.gitignore | 2 + data/tor/empty.txt | 2 - oonib.conf.example | 4 - oonib/config.py | 2 + oonib/log.py | 61 +---------- oonib/onion.py | 63 +++++------ oonib/oonibackend.py | 280 ++++++++++++++++++++++++++---------------------- requirements.txt | 2 +- setup.py | 6 +- 15 files changed, 213 insertions(+), 259 deletions(-) diff --git a/bin/oonib b/bin/oonib index 109c74e..ccbe246 100755 --- a/bin/oonib +++ b/bin/oonib @@ -2,55 +2,35 @@ import sys import os -from twisted.python import log, usage -from twisted.internet import reactor -from twisted.application import app # Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain. sys.path[:] = map(os.path.abspath, sys.path) sys.path.insert(0, os.path.abspath(os.getcwd())) -from oonib import errors as e +from oonib import errors from oonib.config import config -from oonib.log import LoggerFactory - try: config.load() -except e.ConfigFileNotSpecified: +except errors.ConfigFileNotSpecified: print "Config file not specified!" print "Use -c to specify one!" config.usageOptions() sys.exit(1) -except e.ConfigFileDoesNotExist, path: +except errors.ConfigFileDoesNotExist, path: print "Config file \"%s\" does not exist!" % path sys.exit(2) -except e.InvalidReportDirectory, path: +except errors.InvalidReportDirectory, path: print "Invalid report directory: %s!" % path sys.exit(3) -except e.InvalidArchiveDirectory, path: +except errors.InvalidArchiveDirectory, path: print "Invalid archive directory: %s!" % path sys.exit(4) -except e.InvalidInputDirectory, path: +except errors.InvalidInputDirectory, path: print "Invalid input directory: %s" % path sys.exit(5) -except e.InvalidDeckDirectory, path: +except errors.InvalidDeckDirectory, path: print "Invalid deck directory: %s" % path sys.exit(6) -if config.main.chroot: - sys.argv.append('--chroot') - sys.argv.append(config.chroot) - -if not config.main.nodaemon: - sys.argv.append('-y') - -from oonib.oonibackend import application - -from twisted.scripts._twistd_unix import UnixApplicationRunner -class OBaseRunner(UnixApplicationRunner): - temporary_data_dir = None - def createOrGetApplication(self): - return application - -OBaseRunner.loggerFactory = LoggerFactory -OBaseRunner(config.main).run() +from oonib import oonibackend +oonibackend.start() diff --git a/data/archive/.gitignore b/data/archive/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/archive/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/archive/empty.txt b/data/archive/empty.txt deleted file mode 100644 index 9a166f9..0000000 --- a/data/archive/empty.txt +++ /dev/null @@ -1,2 +0,0 @@ -In here will end up the archived reports. -This file may be deleted at any time. diff --git a/data/decks/.gitignore b/data/decks/.gitignore new file mode 100644 index 0000000..514e47a --- /dev/null +++ b/data/decks/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!README diff --git a/data/inputs/.gitignore b/data/inputs/.gitignore new file mode 100644 index 0000000..a0991ff --- /dev/null +++ b/data/inputs/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!Makefile diff --git a/data/reports/.gitignore b/data/reports/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/reports/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/tor/.gitignore b/data/tor/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/tor/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/tor/empty.txt b/data/tor/empty.txt deleted file mode 100644 index ec3be8f..0000000 --- a/data/tor/empty.txt +++ /dev/null @@ -1,2 +0,0 @@ -In here will end up all tor related files. -This file may be deleted at any time. diff --git a/oonib.conf.example b/oonib.conf.example index 6f4c7f3..bcf8393 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -7,8 +7,6 @@ main: bouncer_file: data/bouncer.yaml logfile: null - database_uri: 'sqlite://oonib_test_db.db' - db_threadpool_size: 10 tor_binary: null socks_port: 9055 tor2webmode: false @@ -21,9 +19,7 @@ main: euid: null uid: null gid: null - uuid: null no_save: true - profile: null debug: false stale_time: 3600 diff --git a/oonib/config.py b/oonib/config.py index 31128a9..be29ec1 100644 --- a/oonib/config.py +++ b/oonib/config.py @@ -28,6 +28,8 @@ class Config(object): raise e.ConfigFileDoesNotExist(config_file) self.main = Storage(configuration['main'].items()) + if self.main.logfile is None: + self.main.logfile = 'oonib.log' self.helpers = Storage() for name, helper in configuration['helpers'].items(): diff --git a/oonib/log.py b/oonib/log.py index a79199e..c229f35 100644 --- a/oonib/log.py +++ b/oonib/log.py @@ -6,18 +6,14 @@ Twisted logger for the ooni backend system. """ -import os import sys import codecs import logging import traceback from twisted.python import log as txlog -from twisted.python import util from twisted.python.failure import Failure -from twisted.python.logfile import DailyLogFile -from oonib import otime from oonib.config import config # Get rid of the annoying "No route found for @@ -47,56 +43,21 @@ def log_encode(logmsg): repr(logmsg))) -class LogWithNoPrefix(txlog.FileLogObserver): - def emit(self, eventDict): - text = txlog.textFromEventDict(eventDict) - if text is None: - return - - util.untilConcludes(self.write, "%s\n" % text) - util.untilConcludes(self.flush) # Hoorj! - - -def start(application_name="oonib"): - daily_logfile = None - - if not config.main.logfile: - logfile = 'oonib.log' - else: - logfile = config.main.logfile - - log_folder = os.path.dirname(logfile) - log_filename = os.path.basename(logfile) - - daily_logfile = DailyLogFile(log_filename, log_folder) - - txlog.msg("Starting %s on %s (%s UTC)" % (application_name, - otime.prettyDateNow(), - otime.utcPrettyDateNow())) - txlog.msg("oonib version %s" % config.backend_version) - txlog.startLoggingWithObserver(LogWithNoPrefix(sys.stdout).emit) - txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit) - - -def stop(): - print "Stopping OONI" - - def msg(msg, *arg, **kw): - print "%s" % log_encode(msg) + txlog.msg("%s" % log_encode(msg)) def debug(msg, *arg, **kw): if config.main.get('debug'): - print "[D] %s" % log_encode(msg) + txlog.msg("[D] %s" % log_encode(msg)) def warn(msg, *arg, **kw): - print "[W] %s" % log_encode(msg) + txlog.msg("[W] %s" % log_encode(msg)) def err(msg, *arg, **kw): - print "[!] %s" % log_encode(msg) + txlog.err("[!] %s" % log_encode(msg)) def exception(error): @@ -109,17 +70,3 @@ def exception(error): else: exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback) - - -class LoggerFactory(object): - """ - This is a logger factory to be used by oonib - """ - def __init__(self, options): - pass - - def start(self, application): - start("OONIB") - - def stop(self): - txlog.msg("Stopping OONIB") diff --git a/oonib/onion.py b/oonib/onion.py index 0a7a155..9a5c2c3 100644 --- a/oonib/onion.py +++ b/oonib/onion.py @@ -1,31 +1,17 @@ +import pwd import tempfile + from oonib import log from oonib.config import config -from twisted.internet import reactor, endpoints, defer +from twisted.internet import defer import os from random import randint import socket -from txtorcon import TCPHiddenServiceEndpoint, TorConfig +from txtorcon import TorConfig from txtorcon import launch_tor -from txtorcon import __version__ as txtorcon_version -if tuple(map(int, txtorcon_version.split('.'))) < (0, 9, 0): - """ - Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint - listens on all interfaces by default. - """ - def create_listener(self, proto): - self._update_onion(self.hiddenservice.dir) - self.tcp_endpoint = endpoints.TCP4ServerEndpoint(self.reactor, - self.listen_port, - interface='127.0.0.1') - d = self.tcp_endpoint.listen(self.protocolfactory) - d.addCallback(self._add_attributes).addErrback(self._retry_local_port) - return d - TCPHiddenServiceEndpoint._create_listener = create_listener - def randomFreePort(addr="127.0.0.1"): """ Args: @@ -56,11 +42,15 @@ def txSetupFailed(failure): def _configTor(): torconfig = TorConfig() - if config.main.socks_port: - torconfig.SocksPort = config.main.socks_port - if config.main.control_port: - torconfig.ControlPort = config.main.control_port - if config.main.tor2webmode: + if config.main.socks_port is None: + config.main.socks_port = int(randomFreePort()) + torconfig.SocksPort = config.main.socks_port + + if config.main.control_port is None: + config.main.control_port = int(randomFreePort()) + torconfig.ControlPort = config.main.control_port + + if config.main.tor2webmode is True: torconfig.Tor2webMode = 1 torconfig.CircuitBuildTimeout = 60 if config.main.tor_datadir is None: @@ -68,26 +58,29 @@ def _configTor(): log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") log.warn("Using %s" % temporary_data_dir) torconfig.DataDirectory = temporary_data_dir + uid = -1 + gid = -1 + if config.main.uid is not None: + uid = config.main.uid + if config.main.gid is not None: + gid = config.main.gid + os.chown(temporary_data_dir, uid, gid) else: if os.path.exists(config.main.tor_datadir): torconfig.DataDirectory = os.path.abspath(config.main.tor_datadir) else: raise Exception("Could not find tor datadir") + if config.main.uid is not None: + try: + user = pwd.getpwuid(config.main.uid)[0] + except KeyError: + raise Exception("Invalid user ID") + torconfig.User = user + tor_log_file = os.path.join(torconfig.DataDirectory, "tor.log") torconfig.Log = ["notice stdout", "notice file %s" % tor_log_file] torconfig.save() - if not hasattr(torconfig, 'ControlPort'): - control_port = int(randomFreePort()) - torconfig.ControlPort = control_port - config.main.control_port = control_port - - if not hasattr(torconfig, 'SocksPort'): - socks_port = int(randomFreePort()) - torconfig.SocksPort = socks_port - config.main.socks_port = socks_port - - torconfig.save() return torconfig # get_global_tor is a near-rip of that from txtorcon (so you can have some @@ -110,7 +103,7 @@ def get_global_tor(reactor): # start Tor launching def updates(prog, tag, summary): - print("%d%%: %s" % (prog, summary)) + log.msg("%d%%: %s" % (prog, summary)) yield launch_tor(_global_tor_config, reactor, progress_updates=updates, tor_binary=config.main.tor_binary) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index ebf3c52..a57bf92 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -7,7 +7,9 @@ # In here we start all the test helpers that are required by ooniprobe and # start the report collector -from distutils.version import LooseVersion +import os +import sys + from oonib.api import ooniBackend, ooniBouncer from oonib.config import config @@ -15,107 +17,22 @@ from oonib.onion import get_global_tor from oonib.testhelpers import dns_helpers, ssl_helpers from oonib.testhelpers import http_helpers, tcp_helpers -import os - +from twisted.scripts import twistd +from twisted.python import usage from twisted.application import internet, service -from twisted.internet import reactor, endpoints, ssl, defer +from twisted.internet import reactor, endpoints, ssl from twisted.names import dns -from txtorcon import TCPHiddenServiceEndpoint, TorConfig -from txtorcon import __version__ as txtorcon_version - -if config.main.uid and config.main.gid: - application = service.Application('oonibackend', uid=config.main.uid, - gid=config.main.gid) -else: - application = service.Application('oonibackend') - -multiService = service.MultiService() - -if config.helpers['ssl'].port: - print "Starting SSL helper on %s" % config.helpers['ssl'].port - ssl_helper = internet.SSLServer(int(config.helpers['ssl'].port), - http_helpers.HTTPReturnJSONHeadersHelper(), - ssl_helpers.SSLContext(config)) - multiService.addService(ssl_helper) - ssl_helper.startService() - -# Start the DNS Server related services -if config.helpers['dns'].tcp_port: - print "Starting TCP DNS Helper on %s" % config.helpers['dns'].tcp_port - tcp_dns_helper = internet.TCPServer(int(config.helpers['dns'].tcp_port), - dns_helpers.DNSTestHelper()) - multiService.addService(tcp_dns_helper) - tcp_dns_helper.startService() - -if config.helpers['dns'].udp_port: - print "Starting UDP DNS Helper on %s" % config.helpers['dns'].udp_port - udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper()) - udp_dns_helper = internet.UDPServer(int(config.helpers['dns'].udp_port), - udp_dns_factory) - multiService.addService(udp_dns_helper) - udp_dns_helper.startService() - -if config.helpers['dns_discovery'].udp_port: - print ("Starting UDP DNS Discovery Helper on %s" % - config.helpers['dns_discovery'].udp_port) - udp_dns_discovery = internet.UDPServer( - int(config.helpers['dns_discovery'].udp_port), - dns.DNSDatagramProtocol(dns_helpers.DNSResolverDiscovery()) - ) - multiService.addService(udp_dns_discovery) - -if config.helpers['dns_discovery'].tcp_port: - print ("Starting TCP DNS Discovery Helper on %s" % - config.helpers['dns_discovery'].tcp_port) - tcp_dns_discovery = internet.TCPServer( - int(config.helpers['dns_discovery'].tcp_port), - dns_helpers.DNSResolverDiscovery() - ) - multiService.addService(tcp_dns_discovery) - tcp_dns_discovery.startService() - - -# XXX this needs to be ported -# Start the OONI daphn3 backend -if config.helpers['daphn3'].port: - print "Starting Daphn3 helper on %s" % config.helpers['daphn3'].port - daphn3_helper = internet.TCPServer(int(config.helpers['daphn3'].port), - tcp_helpers.Daphn3Server()) - multiService.addService(daphn3_helper) - daphn3_helper.startService() - - -if config.helpers['tcp-echo'].port: - print "Starting TCP echo helper on %s" % config.helpers['tcp-echo'].port - tcp_echo_helper = internet.TCPServer(int(config.helpers['tcp-echo'].port), - tcp_helpers.TCPEchoHelper()) - multiService.addService(tcp_echo_helper) - tcp_echo_helper.startService() - -if config.helpers['http-return-json-headers'].port: - print ("Starting HTTP return request helper on %s" % - config.helpers['http-return-json-headers'].port) - http_return_request_helper = internet.TCPServer( - int(config.helpers['http-return-json-headers'].port), - http_helpers.HTTPReturnJSONHeadersHelper()) - multiService.addService(http_return_request_helper) - http_return_request_helper.startService() def getHSEndpoint(endpoint_config): + from txtorcon import TCPHiddenServiceEndpoint hsdir = endpoint_config['hsdir'] hsdir = os.path.expanduser(hsdir) hsdir = os.path.realpath(hsdir) - if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'): - return TCPHiddenServiceEndpoint(reactor, - get_global_tor(reactor), - 80, - hidden_service_dir=hsdir) - else: - return TCPHiddenServiceEndpoint(reactor, - get_global_tor(reactor), - 80, - data_dir=hsdir) + return TCPHiddenServiceEndpoint(reactor, + config=get_global_tor(reactor), + public_port=80, + hidden_service_dir=hsdir) def getTCPEndpoint(endpoint_config): return endpoints.TCP4ServerEndpoint( @@ -159,35 +76,146 @@ def createService(endpoint, role, endpoint_config): endpoint, factory ) service.setName("-".join([endpoint_config['type'], role])) - multiService.addService(service) - service.startService() - -# this is to ensure same behaviour with an old config file -if config.main.tor_hidden_service and \ - config.main.bouncer_endpoints is None and \ - config.main.collector_endpoints is None: - base_dir = '.' - if config.main.tor_datadir is not None: - base_dir = config.main.tor_datadir - bouncer_hsdir = os.path.join(base_dir, 'bouncer') - collector_hsdir = os.path.join(base_dir, 'collector') - config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': bouncer_hsdir} ] - config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': collector_hsdir} ] - -for endpoint_config in config.main.get('bouncer_endpoints', []): - if config.main.bouncer_file: - print "Starting bouncer with config %s" % endpoint_config - endpoint = getEndpoint(endpoint_config) - createService(endpoint, 'bouncer', endpoint_config) - else: - print "No bouncer configured" - -for endpoint_config in config.main.get('collector_endpoints', []): - print "Starting collector with config %s" % endpoint_config - endpoint = getEndpoint(endpoint_config) - createService(endpoint, 'collector', endpoint_config) - -for endpoint_config in config.helpers.web_connectivity.get('endpoints', []): - print "Starting web_connectivity helper with config %s" % endpoint_config - endpoint = getEndpoint(endpoint_config) - createService(endpoint, 'web_connectivity', endpoint_config) + return service + +class StartOONIBackendPlugin: + tapname = "oonibackend" + def makeService(self, so): + ooniBackendService = service.MultiService() + + if config.helpers['ssl'].port: + print "Starting SSL helper on %s" % config.helpers['ssl'].port + ssl_helper = internet.SSLServer(int(config.helpers['ssl'].port), + http_helpers.HTTPReturnJSONHeadersHelper(), + ssl_helpers.SSLContext(config)) + ooniBackendService.addService(ssl_helper) + + # Start the DNS Server related services + if config.helpers['dns'].tcp_port: + print "Starting TCP DNS Helper on %s" % config.helpers['dns'].tcp_port + tcp_dns_helper = internet.TCPServer(int(config.helpers['dns'].tcp_port), + dns_helpers.DNSTestHelper()) + ooniBackendService.addService(tcp_dns_helper) + + if config.helpers['dns'].udp_port: + print "Starting UDP DNS Helper on %s" % config.helpers['dns'].udp_port + udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper()) + udp_dns_helper = internet.UDPServer(int(config.helpers['dns'].udp_port), + udp_dns_factory) + ooniBackendService.addService(udp_dns_helper) + + if config.helpers['dns_discovery'].udp_port: + print ("Starting UDP DNS Discovery Helper on %s" % + config.helpers['dns_discovery'].udp_port) + udp_dns_discovery = internet.UDPServer( + int(config.helpers['dns_discovery'].udp_port), + dns.DNSDatagramProtocol(dns_helpers.DNSResolverDiscovery()) + ) + ooniBackendService.addService(udp_dns_discovery) + + if config.helpers['dns_discovery'].tcp_port: + print ("Starting TCP DNS Discovery Helper on %s" % + config.helpers['dns_discovery'].tcp_port) + tcp_dns_discovery = internet.TCPServer( + int(config.helpers['dns_discovery'].tcp_port), + dns_helpers.DNSResolverDiscovery() + ) + ooniBackendService.addService(tcp_dns_discovery) + + # XXX this needs to be ported + # Start the OONI daphn3 backend + if config.helpers['daphn3'].port: + print "Starting Daphn3 helper on %s" % config.helpers['daphn3'].port + daphn3_helper = internet.TCPServer(int(config.helpers['daphn3'].port), + tcp_helpers.Daphn3Server()) + ooniBackendService.addService(daphn3_helper) + + if config.helpers['tcp-echo'].port: + print "Starting TCP echo helper on %s" % config.helpers['tcp-echo'].port + tcp_echo_helper = internet.TCPServer(int(config.helpers['tcp-echo'].port), + tcp_helpers.TCPEchoHelper()) + ooniBackendService.addService(tcp_echo_helper) + + if config.helpers['http-return-json-headers'].port: + print ("Starting HTTP return request helper on %s" % + config.helpers['http-return-json-headers'].port) + http_return_request_helper = internet.TCPServer( + int(config.helpers['http-return-json-headers'].port), + http_helpers.HTTPReturnJSONHeadersHelper()) + ooniBackendService.addService(http_return_request_helper) + + # this is to ensure same behaviour with an old config file + if config.main.tor_hidden_service and \ + config.main.bouncer_endpoints is None and \ + config.main.collector_endpoints is None: + bouncer_hsdir = os.path.join(config.main.tor_datadir, 'bouncer') + collector_hsdir = os.path.join(config.main.tor_datadir, 'collector') + config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': bouncer_hsdir} ] + config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': collector_hsdir} ] + + for endpoint_config in config.main.get('bouncer_endpoints', []): + if config.main.bouncer_file: + print "Starting bouncer with config %s" % endpoint_config + endpoint = getEndpoint(endpoint_config) + bouncer_service = createService(endpoint, 'bouncer', + endpoint_config) + ooniBackendService.addService(bouncer_service) + else: + print "No bouncer configured" + + for endpoint_config in config.main.get('collector_endpoints', []): + print "Starting collector with config %s" % endpoint_config + endpoint = getEndpoint(endpoint_config) + collector_service = createService(endpoint, 'collector', + endpoint_config) + ooniBackendService.addService(collector_service) + + for endpoint_config in config.helpers.web_connectivity.get('endpoints', []): + print "Starting web_connectivity helper with config %s" % endpoint_config + endpoint = getEndpoint(endpoint_config) + web_connectivity_service = createService(endpoint, + 'web_connectivity', + endpoint_config) + ooniBackendService.addService(web_connectivity_service) + + + return ooniBackendService + +class OONIBackendTwistdConfig(twistd.ServerOptions): + subCommands = [("StartOONIBackend", None, usage.Options, "node")] + +def start(): + twistd_args = [] + + flags = [ + 'nodaemon', + 'no_save' + ] + options = [ + 'pidfile', + 'logfile', + 'rundir', + 'euid', + 'gid', + 'uid', + 'umask' + ] + for option in options: + if config.main.get(option, None) is not None: + twistd_args.append('--%s' % option) + twistd_args.append(config.main.get(option)) + for flag in flags: + if config.main.get(flag, None) is True: + twistd_args.append('--%s'% flag) + + twistd_args.append('StartOONIBackend') # Point to our backend plugin + twistd_config = OONIBackendTwistdConfig() + try: + twistd_config.parseOptions(twistd_args) + except usage.error, ue: + print "Usage error from twistd" + sys.exit(7) + + twistd_config.loadedPlugins = {'StartOONIBackend': StartOONIBackendPlugin()} + twistd.runApp(twistd_config) + return 0 diff --git a/requirements.txt b/requirements.txt index b8c18a8..91a711a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Twisted>=13.2.0 cyclone>=1.1 ipaddr>=2.1.10 pygeoip>=0.2.6 -txtorcon>=0.7 +txtorcon>=0.11.0 pyOpenSSL>=0.13 zope.component>=4.0.0 zope.event>=4.0.0 diff --git a/setup.py b/setup.py index 2113d21..39cb1f9 100644 --- a/setup.py +++ b/setup.py @@ -67,9 +67,9 @@ data_files = [ (data_dir, ['oonib.conf.example']), (os.path.join(data_dir, 'decks'), ['data/decks/README']), (os.path.join(data_dir, 'inputs'), ['data/inputs/Makefile']), - (os.path.join(spool_dir, 'reports'), ['data/reports/empty.txt']), - (os.path.join(spool_dir, 'archive'), ['data/archive/empty.txt']), - (os.path.join(var_dir, 'tor'), ['data/tor/empty.txt']), + (os.path.join(spool_dir, 'reports'), ['data/reports/.gitignore']), + (os.path.join(spool_dir, 'archive'), ['data/archive/.gitignore']), + (os.path.join(var_dir, 'tor'), ['data/tor/.gitignore']), (var_dir, ['data/bouncer.yaml', 'data/policy.yaml']) ]

1 0

[oonib/master] Feature/web connectivity (#63)
by art＠torproject.org 30 May '16

30 May '16

commit 5178fc299d361c9ab57002e619e40b05061f5a7d Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat May 28 19:05:42 2016 +0200 Feature/web connectivity (#63) * Add web connectivity test helper * Forward test helper addresses to the host machine * Align the request headers used by the web_connectivity test helper to those of the probe * Add monkey patch for bug in twisted RedirectAgent: https://twistedmatrix.com/trac/ticket/8265 * Add monkey patch for HTTPClientParser.statusReceived * Use TrueHeaders in control request * Add support for specifying endpoints in web_connectivity test helper * Add endpoint for checking the status of the WebConnectivity test helper * Add support for parsing test-helpers-alternate * Fix key for web_connectivity test in example configuration file * Implement on-disk web_connectivity cache * Add support for Gzip content decoding * Also record CNAME resolution. * Rename ips to addrs * Add support for retries in the http_request * Add support for extracting title * Encode the responses as well when debug mode * Handle partial downloads * Ignore errors when encoding headers * Cast title to unicode and ignore errors * Improvements based on feedback and comments by @bassosimone * Move twisted related patches into txextra module * Add support for returning the responses based on a key sent by the client. * Inherit from OONIBHandler so we can get the error message format * Stylistic improvements * Set defaults in a way that oonib.conf can start from the example * Avoid doing join on nonetype * Address comments by @bassosimone * Properly set the body also when we get a partial body downloaded * Move more code into shared oonib.common module * Fully sync the common module with ooni-probe (pulling in also other shared functionality so the Agents match entirely) * Fix location of comment for the patched HTTPClient * Add unittests and integration tests for web_connectivity --- Vagrantfile | 25 +++ oonib.conf.example | 18 +- oonib/bouncer/handlers.py | 14 +- oonib/common/__init__.py | 10 ++ oonib/common/http_utils.py | 54 ++++++ oonib/common/tcp_utils.py | 10 ++ oonib/common/txextra.py | 202 +++++++++++++++++++++ oonib/oonibackend.py | 29 ++- oonib/test/test_web_connectivity.py | 71 ++++++++ oonib/testhelpers/http_helpers.py | 344 +++++++++++++++++++++++++++++++++++- 10 files changed, 758 insertions(+), 19 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index b71e4e7..7aa5f71 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,6 +4,31 @@ Vagrant.configure("2") do |config| config.vm.box = "precise32" config.vm.box_url = "http://files.vagrantup.com/precise32.box" + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + config.vm.network :forwarded_port, guest: 57001, host: 57001 + config.vm.network :forwarded_port, guest: 57001, host: 57002 + config.vm.network :forwarded_port, guest: 57001, host: 57003 + config.vm.network :forwarded_port, guest: 57004, host: 57004 + config.vm.network :forwarded_port, guest: 57005, host: 57005 + config.vm.network :forwarded_port, guest: 57006, host: 57006 + config.vm.network :forwarded_port, guest: 57007, host: 57007 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network :private_network, ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network :public_network + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. config.vm.synced_folder ".", "/data/oonib" end diff --git a/oonib.conf.example b/oonib.conf.example index 1925d81..6f4c7f3 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -30,12 +30,13 @@ main: tor_datadir: null bouncer_endpoints: - - {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"} + #- {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"} - {type: tcp, port: 10080} - - {type: onion, hsdir: bouncer} + - {type: onion, hsdir: /tmp/bouncer} collector_endpoints: - - {type: tls, port: 11443, cert: "private/ssl-key-and-cert.pem"} + #- {type: tls, port: 11443, cert: "private/ssl-key-and-cert.pem"} + - {type: onion, hsdir: /tmp/collector} report_file_template: '{iso8601_timestamp}-{test_name}-{report_id}-{probe_asn}-{probe_cc}-probe-0.2.0.{ext}' helpers: @@ -62,12 +63,17 @@ helpers: dns_discovery: address: null - udp_port: 53 - tcp_port: 53 + udp_port: null + tcp_port: null resolver_address: null ssl: address: null private_key: 'private.key' certificate: 'certificate.crt' - port: 57006 + #port: 57006 + port: null + + web_connectivity: + endpoints: + - {type: tcp, port: 57007} diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 1c7627c..a9370c0 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -159,9 +159,12 @@ class Bouncer(object): requested_nettest['input-hashes'], requested_nettest['test-helpers']) test_helpers = {} + test_helpers_alternate = {} for test_helper in requested_nettest['test-helpers']: + collector_info = self.bouncerFile['collector'][collector] try: - test_helpers[test_helper] = self.bouncerFile['collector'][collector]['test-helper'][test_helper] + test_helpers[test_helper] = \ + collector_info['test-helper'][test_helper] except KeyError: helpers = self.knownHelpers.get(test_helper) if not helpers: @@ -169,12 +172,19 @@ class Bouncer(object): helper = random.choice(helpers) test_helpers[test_helper] = helper['helper-address'] + try: + test_helpers_alternate[test_helper] = \ + collector_info['test-helper-alternate'][test_helper] + except KeyError: + pass + nettest = { 'name': requested_nettest['name'], 'version': requested_nettest['version'], 'input-hashes': requested_nettest['input-hashes'], 'test-helpers': test_helpers, - 'collector': collector, + 'test-helpers-alternate': test_helpers_alternate, + 'collector': collector } nettests.append(nettest) return {'net-tests': nettests} diff --git a/oonib/common/__init__.py b/oonib/common/__init__.py new file mode 100644 index 0000000..7f6cf73 --- /dev/null +++ b/oonib/common/__init__.py @@ -0,0 +1,10 @@ +""" +This modules contains functionality that is shared amongst ooni-probe and +ooni-backend. If the code in here starts growing too much I think it would +make sense to either: + + * Make the code in here into it's own package that is imported by + ooni-probe and ooni-backend. + + * Merge ooniprobe with oonibackend. +""" diff --git a/oonib/common/http_utils.py b/oonib/common/http_utils.py new file mode 100644 index 0000000..6f9db2d --- /dev/null +++ b/oonib/common/http_utils.py @@ -0,0 +1,54 @@ +import re +import codecs +from base64 import b64encode + +META_CHARSET_REGEXP = re.compile('<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"\']*([^\s"\'/>]*)') + +def representBody(body): + if not body: + return body + body = body.replace('\0', '') + decoded = False + charsets = ['ascii', 'utf-8'] + + # If we are able to detect the charset of body from the meta tag + # try to decode using that one first + charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE) + if charset: + try: + encoding = charset.group(1).lower() + codecs.lookup(encoding) + charsets.insert(0, encoding) + except (LookupError, IndexError): + # Skip invalid codecs and partial regexp match + pass + + for encoding in charsets: + try: + body = unicode(body, encoding) + decoded = True + break + except UnicodeDecodeError: + pass + if not decoded: + body = { + 'data': b64encode(body), + 'format': 'base64' + } + return body + +TITLE_REGEXP = re.compile("<title>(.*?)</title>", re.IGNORECASE | re.DOTALL) + +def extractTitle(body): + m = TITLE_REGEXP.search(body, re.IGNORECASE | re.DOTALL) + if m: + return unicode(m.group(1), errors='ignore') + return '' + +REQUEST_HEADERS = { + 'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, ' + 'like Gecko) Chrome/47.0.2526.106 Safari/537.36'], + 'Accept-Language': ['en-US;q=0.8,en;q=0.5'], + 'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,' + '*/*;q=0.8'] +} diff --git a/oonib/common/tcp_utils.py b/oonib/common/tcp_utils.py new file mode 100644 index 0000000..7b7a8a4 --- /dev/null +++ b/oonib/common/tcp_utils.py @@ -0,0 +1,10 @@ +from twisted.internet.protocol import Factory, Protocol + +class TCPConnectProtocol(Protocol): + def connectionMade(self): + self.transport.loseConnection() + +class TCPConnectFactory(Factory): + noisy = False + def buildProtocol(self, addr): + return TCPConnectProtocol() diff --git a/oonib/common/txextra.py b/oonib/common/txextra.py new file mode 100644 index 0000000..7a84592 --- /dev/null +++ b/oonib/common/txextra.py @@ -0,0 +1,202 @@ +import itertools +from copy import copy + +from twisted.web.http_headers import Headers +from twisted.web import error + +from twisted.web.client import BrowserLikeRedirectAgent +from twisted.web._newclient import ResponseFailed +from twisted.web._newclient import HTTPClientParser, ParseError +from twisted.python.failure import Failure + +from twisted.web import client, _newclient + +from twisted.web._newclient import RequestNotSent, RequestGenerationFailed +from twisted.web._newclient import TransportProxyProducer, STATUS + +from twisted.internet import reactor +from twisted.internet.defer import Deferred, fail, maybeDeferred, failure + +from twisted.python import log + +class TrueHeaders(Headers): + def __init__(self, rawHeaders=None): + self._rawHeaders = dict() + if rawHeaders is not None: + for name, values in rawHeaders.iteritems(): + if type(values) is list: + self.setRawHeaders(name, values[:]) + elif type(values) is str: + self.setRawHeaders(name, values) + + def setRawHeaders(self, name, values): + if name.lower() not in self._rawHeaders: + self._rawHeaders[name.lower()] = dict() + self._rawHeaders[name.lower()]['name'] = name + self._rawHeaders[name.lower()]['values'] = values + + def getAllRawHeaders(self): + for _, v in self._rawHeaders.iteritems(): + yield v['name'], v['values'] + + def getRawHeaders(self, name, default=None): + if name.lower() in self._rawHeaders: + return self._rawHeaders[name.lower()]['values'] + return default + + + def getDiff(self, headers, ignore=[]): + """ + + Args: + + headers: a TrueHeaders object + + ignore: specify a list of header fields to ignore + + Returns: + + a set containing the header names that are not present in + header_dict or not present in self. + """ + diff = set() + field_names = [] + + headers_a = copy(self) + headers_b = copy(headers) + for name in ignore: + try: + del headers_a._rawHeaders[name.lower()] + except KeyError: + pass + try: + del headers_b._rawHeaders[name.lower()] + except KeyError: + pass + + for k, v in itertools.chain(headers_a.getAllRawHeaders(), + headers_b.getAllRawHeaders()): + field_names.append(k) + + for name in field_names: + if self.getRawHeaders(name) and headers.getRawHeaders(name): + pass + else: + diff.add(name) + return list(diff) + +class HTTPClientParser(_newclient.HTTPClientParser): + def logPrefix(self): + return 'HTTPClientParser' + + def connectionMade(self): + self.headers = TrueHeaders() + self.connHeaders = TrueHeaders() + self.state = STATUS + self._partialHeader = None + + def headerReceived(self, name, value): + if self.isConnectionControlHeader(name.lower()): + headers = self.connHeaders + else: + headers = self.headers + headers.addRawHeader(name, value) + + def statusReceived(self, status): + # This is a fix for invalid number of parts + try: + return _newclient.HTTPClientParser.statusReceived(self, status) + except ParseError as exc: + if exc.args[0] == 'wrong number of parts': + return _newclient.HTTPClientParser.statusReceived(self, + status + " XXX") + raise + +class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol): + def request(self, request): + if self._state != 'QUIESCENT': + return fail(RequestNotSent()) + + self._state = 'TRANSMITTING' + _requestDeferred = maybeDeferred(request.writeTo, self.transport) + self._finishedRequest = Deferred() + + self._currentRequest = request + + self._transportProxy = TransportProxyProducer(self.transport) + self._parser = HTTPClientParser(request, self._finishResponse) + self._parser.makeConnection(self._transportProxy) + self._responseDeferred = self._parser._responseDeferred + + def cbRequestWrotten(ignored): + if self._state == 'TRANSMITTING': + self._state = 'WAITING' + self._responseDeferred.chainDeferred(self._finishedRequest) + + def ebRequestWriting(err): + if self._state == 'TRANSMITTING': + self._state = 'GENERATION_FAILED' + self.transport.loseConnection() + self._finishedRequest.errback( + failure.Failure(RequestGenerationFailed([err]))) + else: + log.err(err, 'Error writing request, but not in valid state ' + 'to finalize request: %s' % self._state) + + _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting) + + return self._finishedRequest + + +class _HTTP11ClientFactory(client._HTTP11ClientFactory): + noisy = False + + def buildProtocol(self, addr): + return HTTP11ClientProtocol(self._quiescentCallback) + + +class HTTPConnectionPool(client.HTTPConnectionPool): + _factory = _HTTP11ClientFactory + +class TrueHeadersAgent(client.Agent): + def __init__(self, *args, **kw): + super(TrueHeadersAgent, self).__init__(*args, **kw) + self._pool = HTTPConnectionPool(reactor, False) + +class FixedRedirectAgent(BrowserLikeRedirectAgent): + """ + This is a redirect agent with this patch manually applied: + https://twistedmatrix.com/trac/ticket/8265 + """ + def _handleRedirect(self, response, method, uri, headers, redirectCount): + """ + Handle a redirect response, checking the number of redirects already + followed, and extracting the location header fields. + + This is patched to fix a bug in infinite redirect loop. + """ + if redirectCount >= self._redirectLimit: + err = error.InfiniteRedirection( + response.code, + b'Infinite redirection detected', + location=uri) + raise ResponseFailed([Failure(err)], response) + locationHeaders = response.headers.getRawHeaders(b'location', []) + if not locationHeaders: + err = error.RedirectWithNoLocation( + response.code, b'No location header field', uri) + raise ResponseFailed([Failure(err)], response) + location = self._resolveLocation( + # This is the fix to properly handle redirects + response.request.absoluteURI, + locationHeaders[0] + ) + deferred = self._agent.request(method, location, headers) + + def _chainResponse(newResponse): + newResponse.setPreviousResponse(response) + return newResponse + + deferred.addCallback(_chainResponse) + return deferred.addCallback( + self._handleResponse, method, uri, headers, redirectCount + 1) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 601a972..ebf3c52 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -118,15 +118,22 @@ def getHSEndpoint(endpoint_config): data_dir=hsdir) def getTCPEndpoint(endpoint_config): - return endpoints.TCP4ServerEndpoint(reactor, endpoint_config['port']) + return endpoints.TCP4ServerEndpoint( + reactor=reactor, + port=endpoint_config['port'], + interface=endpoint_config.get('address', '') + ) def getTLSEndpoint(endpoint_config): with open(endpoint_config['cert'], 'r') as f: cert_data = f.read() certificate = ssl.PrivateCertificate.loadPEM(cert_data) - return endpoints.SSL4ServerEndpoint(reactor, - endpoint_config['port'], - certificate.options()) + return endpoints.SSL4ServerEndpoint( + reactor=reactor, + port=endpoint_config['port'], + sslContextFactory=certificate.options(), + interface=endpoint_config.get('address', '') + ) def getEndpoint(endpoint_config): if endpoint_config['type'] == 'onion': @@ -143,6 +150,8 @@ def createService(endpoint, role, endpoint_config): factory = ooniBouncer elif role == 'collector': factory = ooniBackend + elif role == 'web_connectivity': + factory = http_helpers.WebConnectivityHelper else: raise Exception("unknown service type") @@ -157,8 +166,11 @@ def createService(endpoint, role, endpoint_config): if config.main.tor_hidden_service and \ config.main.bouncer_endpoints is None and \ config.main.collector_endpoints is None: - bouncer_hsdir = os.path.join(config.main.tor_datadir, 'bouncer') - collector_hsdir = os.path.join(config.main.tor_datadir, 'collector') + base_dir = '.' + if config.main.tor_datadir is not None: + base_dir = config.main.tor_datadir + bouncer_hsdir = os.path.join(base_dir, 'bouncer') + collector_hsdir = os.path.join(base_dir, 'collector') config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': bouncer_hsdir} ] config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': collector_hsdir} ] @@ -174,3 +186,8 @@ for endpoint_config in config.main.get('collector_endpoints', []): print "Starting collector with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) createService(endpoint, 'collector', endpoint_config) + +for endpoint_config in config.helpers.web_connectivity.get('endpoints', []): + print "Starting web_connectivity helper with config %s" % endpoint_config + endpoint = getEndpoint(endpoint_config) + createService(endpoint, 'web_connectivity', endpoint_config) diff --git a/oonib/test/test_web_connectivity.py b/oonib/test/test_web_connectivity.py new file mode 100644 index 0000000..24e2be0 --- /dev/null +++ b/oonib/test/test_web_connectivity.py @@ -0,0 +1,71 @@ +from hashlib import sha256 +from twisted.internet import defer +from twisted.trial import unittest + +from oonib.testhelpers.http_helpers import WebConnectivityCache + +class WebConnectivityCacheTestCase(unittest.TestCase): + def setUp(self): + self.web_connectivity_cache = WebConnectivityCache() + + def tearDown(self): + return self.web_connectivity_cache.expire_all() + + @defer.inlineCallbacks + def test_http_request(self): + value = yield self.web_connectivity_cache.http_request( + 'https://www.google.com/humans.txt') + self.assertEqual( + value['body_length'], 286 + ) + self.assertEqual( + value['status_code'], 200 + ) + self.assertIsInstance(value['headers'], + dict) + + @defer.inlineCallbacks + def test_dns_consistency(self): + # The twisted.names resolve set a reactor.callLater() on parsing the + # resolv.conf and this leads to the reactor being dirty. Look into + # a clean way to solve this and reactive this integration test. + self.skipTest("Skipping to avoid dirty reactor") + value = yield self.web_connectivity_cache.dns_consistency( + 'www.torproject.org') + self.assertIsInstance( + value['addrs'], + list + ) + self.assertIn( + 'failure', + value.keys() + ) + + @defer.inlineCallbacks + def test_tcp_connect(self): + value = yield self.web_connectivity_cache.tcp_connect( + '216.58.213.14:80') + self.assertIsInstance( + value['status'], + bool + ) + self.assertIn( + 'failure', + value.keys() + ) + + @defer.inlineCallbacks + def test_cache_lifecycle(self): + key = 'http://example.com' + key_hash = sha256(key).hexdigest() + value = {'spam': 'ham'} + + miss = yield self.web_connectivity_cache.lookup('http_request', key) + self.assertEqual(miss, None) + + yield self.web_connectivity_cache.cache_value('http_request', key, + value) + hit = yield self.web_connectivity_cache.lookup('http_request', key) + self.assertEqual(hit, value) + + yield self.web_connectivity_cache.expire('http_request', key_hash) diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py index a28cbad..1cddf24 100644 --- a/oonib/testhelpers/http_helpers.py +++ b/oonib/testhelpers/http_helpers.py @@ -1,16 +1,37 @@ import json +import os import random +import re import string - -from twisted.internet import protocol, defer - -from cyclone.web import RequestHandler, Application - +import tempfile +from hashlib import sha256 +from urlparse import urlparse + +from cyclone.web import RequestHandler, Application, HTTPError +from cyclone.web import asynchronous +from twisted.internet import protocol, defer, reactor +from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.internet.error import ConnectionRefusedError +from twisted.internet.error import DNSLookupError, TimeoutError +from twisted.names import client as dns_client +from twisted.names import dns +from twisted.names.error import DNSNameError, DNSServerError from twisted.protocols import policies, basic +from twisted.web.client import readBody +from twisted.web.client import ContentDecoderAgent, GzipDecoder +from twisted.web.client import PartialDownloadError from twisted.web.http import Request from oonib import log, randomStr +from oonib.common.txextra import FixedRedirectAgent, TrueHeaders +from oonib.common.txextra import TrueHeadersAgent +from oonib.common.http_utils import representBody, extractTitle +from oonib.common.http_utils import REQUEST_HEADERS +from oonib.common.tcp_utils import TCPConnectFactory + +from oonib.handlers import OONIBHandler + class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): """ @@ -168,7 +189,320 @@ class HTTPRandomPage(HTTPTrapAll): length = 100000 self.write(self.genRandomPage(length, keyword)) + +def encodeResponse(response): + body = None + body_length = 0 + if (hasattr(response, 'body') and + response.body is not None): + body = response.body + body_length = len(response.body) + headers = {} + for k, v in response.headers.getAllRawHeaders(): + headers[k.lower()] = unicode(v[0], errors='ignore') + return { + 'headers': headers, + 'code': response.code, + 'body_length': body_length, + 'body': representBody(body) + } + +def encodeResponses(response): + responses = [] + responses += [encodeResponse(response)] + if response.previousResponse: + responses += encodeResponses(response.previousResponse) + return responses + + +class WebConnectivityCache(object): + expiration_time = 200 + enable_caching = True + http_retries = 2 + + def __init__(self): + self._response_types = ( + 'http_request', + 'tcp_connect', + 'dns_consistency' + ) + self._cache_lifecycle = {} + self._cache_dir = tempfile.mkdtemp() + for response_type in self._response_types: + os.mkdir(os.path.join(self._cache_dir, response_type)) + self._cache_lifecycle[response_type] = {} + + @defer.inlineCallbacks + def expire_all(self): + for response_type in self._cache_lifecycle.keys(): + for key_hash in self._cache_lifecycle[response_type].keys(): + yield self.expire(response_type, key_hash) + + @defer.inlineCallbacks + def cache_value(self, response_type, key, value): + if response_type not in self._response_types: + raise Exception("Invalid response type") + if self.enable_caching: + key_hash = sha256(key).hexdigest() + cache_file = os.path.join(self._cache_dir, response_type, key_hash) + + if key_hash in self._cache_lifecycle[response_type]: + yield self.expire(response_type, key_hash) + + self._cache_lifecycle[response_type][key_hash] = { + 'expiration': reactor.callLater(self.expiration_time, + self.expire, + response_type, key_hash), + 'lock': defer.DeferredLock() + } + lock = self._cache_lifecycle[response_type][key_hash]['lock'] + yield lock.acquire() + with open(cache_file, 'w+') as fw: + json.dump(value, fw) + lock.release() + + @defer.inlineCallbacks + def expire(self, response_type, key_hash): + if response_type not in self._response_types: + raise Exception("Invalid response type") + lifecycle = self._cache_lifecycle[response_type][key_hash] + if lifecycle['expiration'].active(): + lifecycle['expiration'].cancel() + + yield lifecycle['lock'].acquire() + try: + os.remove(os.path.join(self._cache_dir, response_type, key_hash)) + except OSError: + pass + lifecycle['lock'].release() + del self._cache_lifecycle[response_type][key_hash] + + @defer.inlineCallbacks + def lookup(self, response_type, key): + if not self.enable_caching: + defer.returnValue(None) + + key_hash = sha256(key).hexdigest() + cache_file = os.path.join(self._cache_dir, response_type, key_hash) + + if key_hash not in self._cache_lifecycle[response_type]: + defer.returnValue(None) + + lock = self._cache_lifecycle[response_type][key_hash]['lock'] + expiration = \ + self._cache_lifecycle[response_type][key_hash]['expiration'] + + yield lock.acquire() + + if not os.path.exists(cache_file): + lock.release() + defer.returnValue(None) + + with open(cache_file, 'r') as fh: + value = json.load(fh) + + expiration.reset(self.expiration_time) + lock.release() + defer.returnValue(value) + + @defer.inlineCallbacks + def http_request(self, url, include_http_responses=False): + cached_value = yield self.lookup('http_request', url) + if cached_value is not None: + if include_http_responses is not True: + cached_value.pop('responses', None) + defer.returnValue(cached_value) + + page_info = { + 'body_length': -1, + 'status_code': -1, + 'headers': {}, + 'failure': None + } + + agent = ContentDecoderAgent( + FixedRedirectAgent(TrueHeadersAgent(reactor)), + [('gzip', GzipDecoder)] + ) + try: + retries = 0 + while True: + try: + response = yield agent.request('GET', url, + TrueHeaders(REQUEST_HEADERS)) + headers = {} + for name, value in response.headers.getAllRawHeaders(): + headers[name] = unicode(value[0], errors='ignore') + body_length = -1 + body = None + try: + body = yield readBody(response) + body_length = len(body) + except PartialDownloadError as pde: + if pde.response: + body_length = len(pde.response) + body = pde.response + page_info['body_length'] = body_length + page_info['status_code'] = response.code + page_info['headers'] = headers + page_info['title'] = extractTitle(body) + response.body = body + page_info['responses'] = encodeResponses(response) + break + except: + if retries > self.http_retries: + raise + retries += 1 + except DNSLookupError: + page_info['failure'] = 'dns_lookup_error' + except TimeoutError: + page_info['failure'] = 'generic_timeout_error' + except ConnectionRefusedError: + page_info['failure'] = 'connection_refused_error' + except: + # XXX map more failures + page_info['failure'] = 'unknown_error' + + yield self.cache_value('http_request', url, page_info) + if include_http_responses is not True: + page_info.pop('responses', None) + defer.returnValue(page_info) + + @defer.inlineCallbacks + def tcp_connect(self, socket): + cached_value = yield self.lookup('tcp_connect', socket) + if cached_value is not None: + defer.returnValue(cached_value) + + socket_info = { + 'status': None, + 'failure': None + } + + ip_address, port = socket.split(":") + try: + point = TCP4ClientEndpoint(reactor, ip_address, int(port)) + yield point.connect(TCPConnectFactory()) + socket_info['status'] = True + except TimeoutError: + socket_info['status'] = False + socket_info['failure'] = 'generic_timeout_error' + except ConnectionRefusedError: + socket_info['status'] = False + socket_info['failure'] = 'connection_refused_error' + except: + socket_info['status'] = False + socket_info['failure'] = 'unknown_error' + yield self.cache_value('tcp_connect', socket, socket_info) + defer.returnValue(socket_info) + + @defer.inlineCallbacks + def dns_consistency(self, hostname): + cached_value = yield self.lookup('dns_consistency', hostname) + if cached_value is not None: + defer.returnValue(cached_value) + + dns_info = { + 'addrs': [], + 'failure': None + } + + try: + records = yield dns_client.lookupAddress(hostname) + answers = records[0] + for answer in answers: + if answer.type is dns.A: + dns_info['addrs'].append(answer.payload.dottedQuad()) + elif answer.type is dns.CNAME: + dns_info['addrs'].append(answer.payload.name.name) + except DNSNameError: + dns_info['failure'] = 'dns_name_error' + except DNSServerError: + dns_info['failure'] = 'dns_server_failure' + except: + dns_info['failure'] = 'unknown_error' + + yield self.cache_value('dns_consistency', hostname, dns_info) + defer.returnValue(dns_info) + + +# Taken from +# http://stackoverflow.com/questions/7160737/python-how-to-validate-a-url-in-… +HTTP_REQUEST_REGEXP = re.compile( + r'^(?:http)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + +SOCKET_REGEXP = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+$') + +web_connectivity_cache = WebConnectivityCache() + +class WebConnectivity(OONIBHandler): + @defer.inlineCallbacks + def control_measurement(self, http_url, socket_list, + include_http_responses): + hostname = urlparse(http_url).netloc + dl = [ + web_connectivity_cache.http_request(http_url, include_http_responses), + web_connectivity_cache.dns_consistency(hostname) + ] + for socket in socket_list: + dl.append(web_connectivity_cache.tcp_connect(socket)) + responses = yield defer.DeferredList(dl) + http_request = responses[0][1] + dns = responses[1][1] + tcp_connect = {} + for idx, response in enumerate(responses[2:]): + tcp_connect[socket_list[idx]] = response[1] + self.finish({ + 'http_request': http_request, + 'tcp_connect': tcp_connect, + 'dns': dns + }) + + def validate_request(self, request): + required_keys = ['http_request', 'tcp_connect'] + for rk in required_keys: + if rk not in request.keys(): + raise HTTPError(400, "Missing %s" % rk) + if not HTTP_REQUEST_REGEXP.match(request['http_request']): + raise HTTPError(400, "Invalid http_request URL") + if any([not SOCKET_REGEXP.match(socket) + for socket in request['tcp_connect']]): + raise HTTPError(400, "Invalid tcp_connect URL") + + @asynchronous + def post(self): + try: + request = json.loads(self.request.body) + self.validate_request(request) + include_http_responses = request.get("include_http_responses", + False) + self.control_measurement( + str(request['http_request']), + request['tcp_connect'], + include_http_responses + ) + except HTTPError: + raise + except Exception as exc: + log.msg("Got invalid request") + log.exception(exc) + raise HTTPError(400, 'invalid request') + +class WebConnectivityStatus(RequestHandler): + def get(self): + self.write({"status": "ok"}) + + HTTPRandomPageHelper = Application([ # XXX add regexps here (r"/(.*)/(.*)", HTTPRandomPage) ]) + +WebConnectivityHelper = Application([ + (r"/status", WebConnectivityStatus), + (r"/", WebConnectivity) +])

1 0

[oonib/master] Handle empty bouncer_endpoints and collector_endpoints
by art＠torproject.org 30 May '16

30 May '16

commit 9437610ec476c2c20150a905fbeac7e01908c8b7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat May 21 21:19:52 2016 +0200 Handle empty bouncer_endpoints and collector_endpoints * Set tor_data_dir to null in the default example --- oonib.conf.example | 2 +- oonib/oonibackend.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/oonib.conf.example b/oonib.conf.example index 0a5f4a5..95df448 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -28,7 +28,7 @@ main: stale_time: 3600 tor_hidden_service: false - tor_datadir: test_datadir + tor_datadir: null bouncer_endpoints: - {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"} diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 18bacf0..33ea92d 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -162,12 +162,12 @@ if config.main.tor_hidden_service and \ config.main.bouncer_endpoints = [ {'type': 'onion', 'hsdir': bouncer_hsdir} ] config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': collector_hsdir} ] -for endpoint_config in config.main.get('bouncer_endpoints'): +for endpoint_config in config.main.get('bouncer_endpoints', []): print "Starting bouncer with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) createService(endpoint, 'bouncer', endpoint_config) -for endpoint_config in config.main.get('collector_endpoints'): +for endpoint_config in config.main.get('collector_endpoints', []): print "Starting collector with config %s" % endpoint_config endpoint = getEndpoint(endpoint_config) createService(endpoint, 'collector', endpoint_config)

1 0

[oonib/master] Vagrant: use box canonical name rather than URL (#84)
by art＠torproject.org 30 May '16

30 May '16

commit 973c9b0c2ae089e3043253890148d25dd550f756 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon May 30 10:50:21 2016 +0200 Vagrant: use box canonical name rather than URL (#84) This allows us to get updates running `vagrant box update`. While at it, add `/.vagrant/` to `.gitignore`. --- .gitignore | 3 +++ Vagrantfile | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 7fc5cbd..3f4029e 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ installed-files.txt oonib.conf oonib.pid + +# Local vagrant files +/.vagrant/ diff --git a/Vagrantfile b/Vagrantfile index 7aa5f71..a2abe0d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,8 +2,7 @@ # vi: set ft=ruby : Vagrant.configure("2") do |config| - config.vm.box = "precise32" - config.vm.box_url = "http://files.vagrantup.com/precise32.box" + config.vm.box = "ubuntu/precise32" # Create a forwarded port mapping which allows access to a specific port # within the machine from a port on the host machine. In the example below,

1 0

[oonib/master] Document configuration (#78)
by art＠torproject.org 30 May '16

30 May '16

commit 0396a55a27e9923537a6cd3d743d37009ef4715b Author: Joe Landers <joe(a)joelanders.net> Date: Thu May 26 12:49:23 2016 +0200 Document configuration (#78) * Update README.rst * remove old configuration option --- README.rst | 17 +++++++++++++++++ oonib.conf.example | 1 - 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 63a242c..a73d776 100644 --- a/README.rst +++ b/README.rst @@ -107,6 +107,23 @@ this would simply be ``'report'``) and ``tor_datadir`` (where you would like the spawned Tor process to keep its data). If you compiled Tor yourself, you'll likely want to specify it for the ``tor_binary`` option. +Configure bouncer and collector endpoints +......................................... + +The bouncer and collector are HTTP applications ("protocols" in twisted terminology) that can be configured to run on top of plain TCP, TLS, or onion service endpoints. +Here is an example of the relevant part of the configuration:: + bouncer_endpoints: + - {type: tls, port: 10443, cert: "private/ssl-key-and-cert.pem"} + - {type: tcp, port: 10080} + - {type: onion, hsdir: "/some/private/bouncer"} + + collector_endpoints: + - {type: tls, port: 11443, cert: "private/ssl-key-and-cert.pem"} + - {type: onion, hsdir: "/some/private/collector"} + +`scripts/gen-ssl-key-cert.sh` in this repo contains the openssl command to generate a self-signed certificate which you can use for the tls endpoint. +txtorcon will use the hostname/private_key from the configured hsdir to start an onion service, or generate a new key if hsdir is empty. + Generate self signed certs for OONIB .................................... If you want to use the HTTPS test helper, you will need to create a diff --git a/oonib.conf.example b/oonib.conf.example index 95df448..1925d81 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -27,7 +27,6 @@ main: debug: false stale_time: 3600 - tor_hidden_service: false tor_datadir: null bouncer_endpoints:

1 0

[oonib/master] [hotfix] Only create bouncer endpoints when it's configured
by art＠torproject.org 30 May '16

30 May '16

commit f03f8055d44d3bbb7cbcfb789946ba67a94e1e52 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu May 26 13:49:28 2016 +0200 [hotfix] Only create bouncer endpoints when it's configured --- oonib/oonibackend.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 33ea92d..601a972 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -163,9 +163,12 @@ if config.main.tor_hidden_service and \ config.main.collector_endpoints = [ {'type': 'onion', 'hsdir': collector_hsdir} ] for endpoint_config in config.main.get('bouncer_endpoints', []): - print "Starting bouncer with config %s" % endpoint_config - endpoint = getEndpoint(endpoint_config) - createService(endpoint, 'bouncer', endpoint_config) + if config.main.bouncer_file: + print "Starting bouncer with config %s" % endpoint_config + endpoint = getEndpoint(endpoint_config) + createService(endpoint, 'bouncer', endpoint_config) + else: + print "No bouncer configured" for endpoint_config in config.main.get('collector_endpoints', []): print "Starting collector with config %s" % endpoint_config

1 0

[oonib/master] Download openssl with wget as well to fix SSL errors on MLAB
by art＠torproject.org 30 May '16

30 May '16

commit 94d34521e9ec7f6a61b1d486dbc53506cb9034cb Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue May 10 19:06:44 2016 +0200 Download openssl with wget as well to fix SSL errors on MLAB --- scripts/build_tor2web_tor.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_tor2web_tor.sh b/scripts/build_tor2web_tor.sh index 080a878..c9d7d1b 100755 --- a/scripts/build_tor2web_tor.sh +++ b/scripts/build_tor2web_tor.sh @@ -12,12 +12,12 @@ SCRIPT_ROOT=`pwd` # Package URLS CURL_URLS="\ http://zlib.net/zlib-$ZLIB_VERSION.tar.gz -https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz.asc -https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz https://github.com/downloads/libevent/libevent/libevent-$LIBEVENT_VERSION.t… https://github.com/downloads/libevent/libevent/libevent-$LIBEVENT_VERSION.t…" WGET_URLS="\ +https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz.asc +https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz https://dist.torproject.org/tor-$TOR_VERSION.tar.gz https://dist.torproject.org/tor-$TOR_VERSION.tar.gz.asc"

1 0

[oonib/master] remove echo server i used for debugging
by art＠torproject.org 30 May '16

30 May '16

commit 8e359977af6f92829a27eb45fe4be0a4cda0435a Author: Joe Landers <joe(a)joelanders.net> Date: Fri May 6 12:57:21 2016 +0200 remove echo server i used for debugging --- oonib/oonibackend.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 1c6a23a..0b8c4ec 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -135,10 +135,6 @@ def getEndpoint(endpoint_config): else: raise Exception("unknown endpoint type") -class Echo(protocol.Protocol): - def dataReceived(self, data): - self.transport.write('server echoes ' + data) - def createService(endpoint, role, endpoint_config): if role == 'bouncer': factory = ooniBouncer

1 0