commit dce255af61ae3ee89a0d37776103919dd86e4374 Author: Arturo Filastò art@fuffa.org Date: Sun Nov 11 15:41:28 2012 +0100

Do some major refactoring of the ooniprobe backend * Implement TCP Echo test helper * Refactor testhelpers into proper namespace * Remove unused imports * Remove files that were not being imported anywhere --- oonib/config.py | 39 ++++++++++++------ oonib/lib/__init__.py | 1 - oonib/lib/ssl.py | 8 ---- oonib/oonibackend.conf.sample | 12 ------ oonib/oonibackend.py | 79 ++++++++++++++++++++---------------- oonib/testhelpers/__init__.py | 5 ++ oonib/testhelpers/dns.py | 16 ------- oonib/testhelpers/dns_helpers.py | 16 +++++++ oonib/testhelpers/http_helpers.py | 80 +++++++++++++++++++++++++++++++++++++ oonib/testhelpers/httph.py | 80 ------------------------------------- oonib/testhelpers/ssl_helpers.py | 9 ++++ oonib/testhelpers/tcp_helpers.py | 14 ++++++ ooniprobe.conf | 3 +- 13 files changed, 195 insertions(+), 167 deletions(-)

diff --git a/oonib/config.py b/oonib/config.py index 0156b69..dc2be2f 100644 --- a/oonib/config.py +++ b/oonib/config.py @@ -9,21 +9,34 @@ def get_root_path():

# XXX convert this to something that is a proper config file main = Storage() -main.reporting_port = 8888 -main.http_port = 8080 -main.dns_udp_port = 5354 -main.dns_tcp_port = 8002 -main.daphn3_port = 9666 -main.server_version = "Apache" +main.collector_port = 8888 + +# XXX make this be the server name that is of main.database_uri = "sqlite:"+get_root_path()+"oonib_test_db.db" main.db_threadpool_size = 10 -#main.ssl_private_key = /path/to/data/private.key -#main.ssl_certificate = /path/to/data/certificate.crt -#main.ssl_port = 8433

helpers = Storage() -helpers.http_return_request_port = 1234

-daphn3 = Storage() -daphn3.yaml_file = "/path/to/data/oonib/daphn3.yaml" -daphn3.pcap_file = "/path/to/data/server.pcap" +helpers.http_return_request = Storage() +helpers.http_return_request.port = 57001 +helpers.http_return_request.server_version = "Apache" + +helpers.tcp_echo = Storage() +helpers.tcp_echo.port = 57002 + +helpers.daphn3 = Storage() +helpers.daphn3.yaml_file = "/path/to/data/oonib/daphn3.yaml" +helpers.daphn3.pcap_file = "/path/to/data/server.pcap" + +helpers.daphn3.port = 57003 + +helpers.dns = Storage() +helpers.dns.udp_port = 57004 +helpers.dns.tcp_port = 57005 + +helpers.ssl = Storage() +#helpers.ssl.private_key = /path/to/data/private.key +#helpers.ssl.certificate = /path/to/data/certificate.crt +#helpers.ssl.port = 57007 + + diff --git a/oonib/lib/__init__.py b/oonib/lib/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/oonib/lib/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/oonib/lib/ssl.py b/oonib/lib/ssl.py deleted file mode 100644 index 20e99e8..0000000 --- a/oonib/lib/ssl.py +++ /dev/null @@ -1,8 +0,0 @@ -from twisted.internet import ssl -from oonib import config - -class SSLContext(ssl.DefaultOpenSSLContextFactory): - def __init__(self, *args, **kw): - ssl.DefaultOpenSSLContextFactory.__init__(self, config.main.ssl_private_key, - config.main.ssl_certificate) - diff --git a/oonib/oonibackend.conf.sample b/oonib/oonibackend.conf.sample deleted file mode 100644 index dc0a662..0000000 --- a/oonib/oonibackend.conf.sample +++ /dev/null @@ -1,12 +0,0 @@ -[main] -reporting_port = 8888 -http_port = 8080 -dns_udp_port = 5354 -dns_tcp_port = 8002 -daphn3_port = 9666 -server_version = Apache -ssl_private_key = /path/to/private.key -ssl_certificate = /path/to/certificate.crt -[daphn3] -pcap_file = /path/to/server.pcap -yaml_file = /path/to/server.yaml diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 799dcf0..8f47e73 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -3,57 +3,53 @@ # # This is the backend system responsible for running certain services that are # useful for censorship detection. - -import json -import random -import string +# +# In here we start all the test helpers that are required by ooniprobe and +# start the report collector

from twisted.application import internet - -from twisted.internet import protocol, reactor, defer +from twisted.internet import reactor from twisted.application import internet, service from twisted.application.service import Application - -from twisted.web import resource, server, static -from twisted.web.microdom import escape -from twisted.protocols import basic from twisted.names import dns

from ooni.utils import log

from oonib.report.api import reportingBackend -from oonib.lib.ssl import SSLContext + from oonib import config

-from oonib.testhelpers.httph import HTTPReturnJSONHeadersHelper -from oonib.testhelpers.dns import ProxyDNSServer +from oonib.testhelpers import dns_helpers, ssl_helpers +from oonib.testhelpers import http_helpers, tcp_helpers + #from oonib.testhelpers.daphn3 import Daphn3Server from oonib import db_threadpool

from cyclone import web

-# This tells twisted to set the -server.version = config.main.server_version - application = service.Application('oonibackend') serviceCollection = service.IServiceCollection(application)

-if config.main.ssl_port: - internet.SSLServer(int(config.main.ssl_port), +if config.helpers.ssl.port: + log.msg("Starting SSL helper") + ssl_helper = internet.SSLServer(int(config.helpers.ssl.port), server.Site(HTTPBackend()), - SSLContext(config), - ).setServiceParent(serviceCollection) + SSLContext(config)) + ssl_helper.setServiceParent(serviceCollection)

# Start the DNS Server related services -if config.main.dns_tcp_port: - TCPDNSServer = ProxyDNSServer() - internet.TCPServer(int(config.main.dns_tcp_port), - TCPDNSServer).setServiceParent(serviceCollection) - -if config.main.dns_udp_port: - UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer) - internet.UDPServer(int(config.main.dns_udp_port), - UDPFactory).setServiceParent(serviceCollection) +if config.helpers.dns.tcp_port: + log.msg("Starting TCP DNS Helper on %s" % config.helpers.dns.tcp_port) + tcp_dns_helper = internet.TCPServer(int(config.helpers.dns.tcp_port), + dns_helpers.DNSTestHelper()) + tcp_dns_helper.setServiceParent(serviceCollection) + +if config.helpers.dns.udp_port: + log.msg("Starting UDP DNS Helper on %s" % config.helpers.dns.udp_port) + udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper()) + udp_dns_helper = internet.UDPServer(int(config.helpers.dns.udp_port), + udp_dns_factory) + udp_dns_helper.setServiceParent(serviceCollection)

# XXX this needs to be ported # Start the OONI daphn3 backend @@ -62,12 +58,23 @@ if config.main.dns_udp_port: # internet.TCPServer(int(config.main.daphn3_port), # daphn3).setServiceParent(serviceCollection)

-if config.main.reporting_port: - internet.TCPServer(int(config.main.reporting_port), - reportingBackend).setServiceParent(serviceCollection) - -if config.helpers.http_return_request_port: - internet.TCPServer(int(config.helpers.http_return_request_port), - HTTPReturnJSONHeadersHelper).setServiceParent(serviceCollection) +if config.main.collector_port: + log.msg("Starting Collector on %s" % config.main.collector_port) + collector = internet.TCPServer(int(config.main.collector_port), + reportingBackend) + collector.setServiceParent(serviceCollection) + +if config.helpers.tcp_echo.port: + log.msg("Starting TCP echo helper on %s" % config.helpers.tcp_echo.port) + tcp_echo_helper = internet.TCPServer(int(config.helpers.tcp_echo.port), + tcp_helpers.TCPEchoHelper()) + tcp_echo_helper.setServiceParent(serviceCollection) + +if config.helpers.http_return_request.port: + log.msg("Starting HTTP return request helper on %s" % config.helpers.http_return_request.port) + http_return_request_helper = internet.TCPServer( + int(config.helpers.http_return_request.port), + http_helpers.HTTPReturnJSONHeadersHelper) + http_return_request_helper.setServiceParent(serviceCollection)

reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop) diff --git a/oonib/testhelpers/__init__.py b/oonib/testhelpers/__init__.py index e69de29..4dbb547 100644 --- a/oonib/testhelpers/__init__.py +++ b/oonib/testhelpers/__init__.py @@ -0,0 +1,5 @@ +from . import dns_helpers +from . import http_helpers +from . import tcp_helpers + +__all__ = ['dns_helpers', 'http_helpers', 'tcp_helpers'] diff --git a/oonib/testhelpers/dns.py b/oonib/testhelpers/dns.py deleted file mode 100644 index 689d1a4..0000000 --- a/oonib/testhelpers/dns.py +++ /dev/null @@ -1,16 +0,0 @@ -from twisted.internet.protocol import Factory, Protocol -from twisted.internet import reactor -from twisted.names import dns -from twisted.names import client, server - -class ProxyDNSServer(server.DNSServerFactory): - def __init__(self, authorities = None, - caches = None, clients = None, - verbose = 0): - resolver = client.Resolver(servers=[('8.8.8.8', 53)]) - server.DNSServerFactory.__init__(self, authorities = authorities, - caches = caches, clients = [resolver], - verbose = verbose) - def handleQuery(self, message, protocol, address): - print message, protocol, address - server.DNSServerFactory.handleQuery(self, message, protocol, address) diff --git a/oonib/testhelpers/dns_helpers.py b/oonib/testhelpers/dns_helpers.py new file mode 100644 index 0000000..cb4ff9f --- /dev/null +++ b/oonib/testhelpers/dns_helpers.py @@ -0,0 +1,16 @@ +from twisted.internet.protocol import Factory, Protocol +from twisted.internet import reactor +from twisted.names import dns +from twisted.names import client, server + +class DNSTestHelper(server.DNSServerFactory): + def __init__(self, authorities = None, + caches = None, clients = None, + verbose = 0): + resolver = client.Resolver(servers=[('8.8.8.8', 53)]) + server.DNSServerFactory.__init__(self, authorities = authorities, + caches = caches, clients = [resolver], + verbose = verbose) + def handleQuery(self, message, protocol, address): + print message, protocol, address + server.DNSServerFactory.handleQuery(self, message, protocol, address) diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py new file mode 100644 index 0000000..7001fa6 --- /dev/null +++ b/oonib/testhelpers/http_helpers.py @@ -0,0 +1,80 @@ +import json +import random +import string + +from twisted.application import internet, service +from twisted.internet import protocol, reactor, defer +from twisted.protocols import basic +from twisted.web import resource, server, static, http +from twisted.web.microdom import escape + +from cyclone.web import RequestHandler, Application + +class HTTPTrapAll(RequestHandler): + """ + Master class to be used to trap all the HTTP methods and make capitalized + requests pass. + """ + def _execute(self, transforms, *args, **kwargs): + self._transforms = transforms + defer.maybeDeferred(self.prepare).addCallbacks( + self._execute_handler, + lambda f: self._handle_request_exception(f.value), + callbackArgs=(args, kwargs)) + + def _execute_handler(self, r, args, kwargs): + if not self._finished: + args = [self.decode_argument(arg) for arg in args] + kwargs = dict((k, self.decode_argument(v, name=k)) + for (k, v) in kwargs.iteritems()) + # This is where we do the patching + # XXX this is somewhat hackish + d = defer.maybeDeferred(self.all, *args, **kwargs) + d.addCallbacks(self._execute_success, self._execute_failure) + self.notifyFinish().addCallback(self.on_connection_close) + +class HTTPReturnJSONHeaders(HTTPTrapAll): + def all(self): + # XXX make sure that the request headers are in the correct order + submitted_data = {'request_body': self.request.body, + 'request_headers': self.request.headers, + 'request_uri': self.request.uri, + 'request_method': self.request.method} + response = json.dumps(submitted_data) + self.write(response) + +HTTPReturnJSONHeadersHelper = Application([ + (r"/*", HTTPReturnJSONHeaders) +]) + + +class HTTPRandomPage(HTTPTrapAll): + """ + This generates a random page of arbitrary length and containing the string + selected by the user. + /<length>/<keyword> + XXX this is currently disabled as it is not of use to any test. + """ + isLeaf = True + def _gen_random_string(self, length): + return ''.join(random.choice(string.letters) for x in range(length)) + + def genRandomPage(self, length=100, keyword=None): + data = self._gen_random_string(length/2) + if keyword: + data += keyword + data += self._gen_random_string(length - length/2) + data += '\n' + return data + + def all(self, length, keyword): + length = 100 + if length > 100000: + length = 100000 + return self.genRandomPage(length, keyword) + +HTTPRandomPageHelper = Application([ + # XXX add regexps here + (r"/(.*)/(.*)", HTTPRandomPage) +]) + diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py deleted file mode 100644 index 7001fa6..0000000 --- a/oonib/testhelpers/httph.py +++ /dev/null @@ -1,80 +0,0 @@ -import json -import random -import string - -from twisted.application import internet, service -from twisted.internet import protocol, reactor, defer -from twisted.protocols import basic -from twisted.web import resource, server, static, http -from twisted.web.microdom import escape - -from cyclone.web import RequestHandler, Application - -class HTTPTrapAll(RequestHandler): - """ - Master class to be used to trap all the HTTP methods and make capitalized - requests pass. - """ - def _execute(self, transforms, *args, **kwargs): - self._transforms = transforms - defer.maybeDeferred(self.prepare).addCallbacks( - self._execute_handler, - lambda f: self._handle_request_exception(f.value), - callbackArgs=(args, kwargs)) - - def _execute_handler(self, r, args, kwargs): - if not self._finished: - args = [self.decode_argument(arg) for arg in args] - kwargs = dict((k, self.decode_argument(v, name=k)) - for (k, v) in kwargs.iteritems()) - # This is where we do the patching - # XXX this is somewhat hackish - d = defer.maybeDeferred(self.all, *args, **kwargs) - d.addCallbacks(self._execute_success, self._execute_failure) - self.notifyFinish().addCallback(self.on_connection_close) - -class HTTPReturnJSONHeaders(HTTPTrapAll): - def all(self): - # XXX make sure that the request headers are in the correct order - submitted_data = {'request_body': self.request.body, - 'request_headers': self.request.headers, - 'request_uri': self.request.uri, - 'request_method': self.request.method} - response = json.dumps(submitted_data) - self.write(response) - -HTTPReturnJSONHeadersHelper = Application([ - (r"/*", HTTPReturnJSONHeaders) -]) - - -class HTTPRandomPage(HTTPTrapAll): - """ - This generates a random page of arbitrary length and containing the string - selected by the user. - /<length>/<keyword> - XXX this is currently disabled as it is not of use to any test. - """ - isLeaf = True - def _gen_random_string(self, length): - return ''.join(random.choice(string.letters) for x in range(length)) - - def genRandomPage(self, length=100, keyword=None): - data = self._gen_random_string(length/2) - if keyword: - data += keyword - data += self._gen_random_string(length - length/2) - data += '\n' - return data - - def all(self, length, keyword): - length = 100 - if length > 100000: - length = 100000 - return self.genRandomPage(length, keyword) - -HTTPRandomPageHelper = Application([ - # XXX add regexps here - (r"/(.*)/(.*)", HTTPRandomPage) -]) - diff --git a/oonib/testhelpers/ssl_helpers.py b/oonib/testhelpers/ssl_helpers.py new file mode 100644 index 0000000..2c07275 --- /dev/null +++ b/oonib/testhelpers/ssl_helpers.py @@ -0,0 +1,9 @@ +from twisted.internet import ssl +from oonib import config + +class SSLContext(ssl.DefaultOpenSSLContextFactory): + def __init__(self, *args, **kw): + ssl.DefaultOpenSSLContextFactory.__init__(self, + config.helpers.ssl.private_key, + config.main.ssl.certificate) + diff --git a/oonib/testhelpers/tcp_helpers.py b/oonib/testhelpers/tcp_helpers.py new file mode 100644 index 0000000..57b93d2 --- /dev/null +++ b/oonib/testhelpers/tcp_helpers.py @@ -0,0 +1,14 @@ +from twisted.internet.protocol import Protocol, Factory + +class TCPEchoProtocol(Protocol): + def dataReceived(self, data): + self.transport.write(data) + +class TCPEchoHelper(Factory): + """ + A very simple echo protocol implementation + """ + protocol = TCPEchoProtocol + + + diff --git a/ooniprobe.conf b/ooniprobe.conf index 47f480a..191d809 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -20,7 +20,8 @@ advanced: # XXX change this to point to the directory where you have stored the GeoIP # database file. This should be the directory in which OONI is installed # /path/to/ooni-probe/data/ - geoip_data_dir: /usr/share/GeoIP/ + #geoip_data_dir: /usr/share/GeoIP/ + geoip_data_dir: /home/x/code/networking/ooni-probe/data/ debug: true threadpool_size: 10