commit 93bcc2f5f9d90b1548968807cf3404e19559b362 Author: Arturo Filastò art@fuffa.org Date: Sat Nov 10 14:16:02 2012 +0100

Fully switch to using only cyclone for HTTP related test helpers * Implement HTTP Request handler backend * Fix the startup script of OONIB * Disable daphn3 test helper script * Various cleanups --- bin/oonib | 41 ++++++++++++++++-- ooni/utils/log.py | 18 +++++++- oonib/lib/__init__.py | 7 --- oonib/lib/ssl.py | 2 +- oonib/oonibackend.py | 29 +++++-------- oonib/report/db/__init__.py | 4 +- oonib/testhelpers/daphn3.py | 4 +- oonib/testhelpers/httph.py | 100 +++++++++++++++++-------------------------- 8 files changed, 107 insertions(+), 98 deletions(-)

diff --git a/bin/oonib b/bin/oonib index df7c62f..0808743 100755 --- a/bin/oonib +++ b/bin/oonib @@ -1,4 +1,37 @@ -#!/bin/sh -ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" -export PYTHONPATH=$PYTHONPATH:$ROOT -twistd -ny $ROOT/oonib/oonibackend.py +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import sys +import os +# Avoid the export PYTHONPATH insanity +sys.path[:] = map(os.path.abspath, sys.path) +this_directory = os.path.dirname(__file__) +root = os.path.abspath(os.path.join(this_directory, '..')) +sys.path.insert(0, root) + +backend_script = os.path.join(root, 'oonib', 'oonibackend.py') + +from twisted.python import log, usage +from twisted.internet import reactor +from twisted.application import app + +from oonib import runner +from oonib.oonibackend import application + +sys.argv[1:] = ['-ny', backend_script] + +def runApp(config): + runner.OBaseRunner(config).run() + +config = runner.ServerOptions() +try: + config.parseOptions() +except usage.error, ue: + print config + print "%s: %s" % (sys.argv[0], ue) +else: + runApp(config) + diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 0cb62ac..c05c776 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -13,7 +13,7 @@ from twisted.python.logfile import DailyLogFile from ooni.utils import otime from ooni import config

-def start(logfile=None): +def start(logfile=None, application_name="OONI"): daily_logfile = None

if not logfile: @@ -24,7 +24,7 @@ def start(logfile=None):

daily_logfile = DailyLogFile(log_filename, log_folder)

- txlog.msg("Starting OONI on %s (%s UTC)" % (otime.prettyDateNow(), + txlog.msg("Starting %s on %s (%s UTC)" % (application_name, otime.prettyDateNow(), otime.utcPrettyDateNow())) logging.basicConfig() python_logging = txlog.PythonLoggingObserver() @@ -53,3 +53,17 @@ def err(msg, *arg, **kw): def exception(*msg): logging.exception(msg)

+class LoggerFactory(object): + """ + This is a logger factory to be used by oonib + """ + def __init__(self, options): + pass + + def start(self, application): + # XXX parametrize this + start('/tmp/oonib.log', "OONIB") + + def stop(self): + txlog.msg("Stopping OONIB") + diff --git a/oonib/lib/__init__.py b/oonib/lib/__init__.py index 5b4380a..8b13789 100644 --- a/oonib/lib/__init__.py +++ b/oonib/lib/__init__.py @@ -1,8 +1 @@ -from ooni.utils import Storage -from ooni.utils.config import Config -import os

-root = os.path.normpath(os.path.join(os.path.realpath(__file__), '../../')) -config = Storage() -config.main = Config('main', os.path.join(root, 'oonibackend.conf')) -config.daphn3 = Config('daphn3', os.path.join(root, 'oonibackend.conf')) diff --git a/oonib/lib/ssl.py b/oonib/lib/ssl.py index 094bdc9..20e99e8 100644 --- a/oonib/lib/ssl.py +++ b/oonib/lib/ssl.py @@ -1,5 +1,5 @@ from twisted.internet import ssl -from oonib.lib import config +from oonib import config

class SSLContext(ssl.DefaultOpenSSLContextFactory): def __init__(self, *args, **kw): diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index eaa52f9..9964854 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -24,36 +24,26 @@ from ooni.utils import log

from oonib.report.api import reportingBackend from oonib.lib.ssl import SSLContext -from oonib.lib import config +from oonib import config

-from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer +from oonib.testhelpers.httph import HTTPReturnJSONHeadersHelper from oonib.testhelpers.dns import ProxyDNSServer -from oonib.testhelpers.daphn3 import Daphn3Server +#from oonib.testhelpers.daphn3 import Daphn3Server

from cyclone import web

-log.start('/tmp/oonib.log') - # This tells twisted to set the server.version = config.main.server_version

application = service.Application('oonibackend') serviceCollection = service.IServiceCollection(application)

-if config.main.http_port: - internet.TCPServer(int(config.main.http_port), - server.Site(HTTPBackend()) - ).setServiceParent(serviceCollection) - if config.main.ssl_port: internet.SSLServer(int(config.main.ssl_port), server.Site(HTTPBackend()), SSLContext(config), ).setServiceParent(serviceCollection)

-debugHTTPServer = DebugHTTPServer() -internet.TCPServer(8090, debugHTTPServer).setServiceParent(serviceCollection) - # Start the DNS Server related services if config.main.dns_tcp_port: TCPDNSServer = ProxyDNSServer() @@ -65,15 +55,18 @@ if config.main.dns_udp_port: internet.UDPServer(int(config.main.dns_udp_port), UDPFactory).setServiceParent(serviceCollection)

+# XXX this needs to be ported # Start the OONI daphn3 backend -if config.main.daphn3_port: - daphn3 = Daphn3Server() - internet.TCPServer(int(config.main.daphn3_port), - daphn3).setServiceParent(serviceCollection) +#if config.main.daphn3_port: +# daphn3 = Daphn3Server() +# internet.TCPServer(int(config.main.daphn3_port), +# daphn3).setServiceParent(serviceCollection)

if config.main.reporting_port: internet.TCPServer(int(config.main.reporting_port), reportingBackend).setServiceParent(serviceCollection)

-reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop) +if config.helpers.http_return_request_port: + internet.TCPServer(int(config.helpers.http_return_request_port), + HTTPReturnJSONHeadersHelper).setServiceParent(serviceCollection)

diff --git a/oonib/report/db/__init__.py b/oonib/report/db/__init__.py index 178abe9..383b24b 100644 --- a/oonib/report/db/__init__.py +++ b/oonib/report/db/__init__.py @@ -1,5 +1,3 @@ -__all__ = ['models'] - from twisted.python import log from twisted.python.threadpool import ThreadPool from twisted.internet.defer import inlineCallbacks @@ -21,7 +19,7 @@ def getStore():

@inlineCallbacks def create_tables(): - from oonib.report.db import models + from oonib.report.models import models

for x in models.__all__: query = getattr(models.__getattribute__(x), 'createQuery') diff --git a/oonib/testhelpers/daphn3.py b/oonib/testhelpers/daphn3.py index facd11f..3b59f93 100644 --- a/oonib/testhelpers/daphn3.py +++ b/oonib/testhelpers/daphn3.py @@ -1,9 +1,9 @@ +# XXX this is currently broken and needs to be ported from twisted.internet import protocol from twisted.internet.error import ConnectionDone

-from oonib.lib import config +from oonib import config

-from ooni.plugoo import reports from ooni.protocols.daphn3 import Mutator, Daphn3Protocol from ooni.protocols.daphn3 import read_pcap, read_yaml

diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py index 217dbd5..b793852 100644 --- a/oonib/testhelpers/httph.py +++ b/oonib/testhelpers/httph.py @@ -8,14 +8,42 @@ from twisted.protocols import basic from twisted.web import resource, server, static, http from twisted.web.microdom import escape

-server.version = "Apache" +from cyclone.web import RequestHandler, Application

-class HTTPRandomPage(resource.Resource): +class HTTPTrapAll(RequestHandler): + """ + Master class to be used to trap all the HTTP methods + """ + def get(self, *arg, **kw): + self.all(*arg, **kw) + + def post(self, *arg, **kw): + self.all(*arg, **kw) + + def put(self, *arg, **kw): + self.all(*arg, **kw) + +class HTTPReturnJSONHeaders(HTTPTrapAll): + def all(self): + # XXX make sure that the request headers are in the correct order + submitted_data = {'request_body': self.request.body, + 'request_headers': self.request.headers, + 'request_uri': self.request.uri, + 'request_method': self.request.method} + response = json.dumps(submitted_data) + self.write(response) + +HTTPReturnJSONHeadersHelper = Application([ + (r"/*", HTTPReturnJSONHeaders) +]) + + +class HTTPRandomPage(HTTPTrapAll): """ This generates a random page of arbitrary length and containing the string selected by the user. - The format is the following: - /random/<length>/<keyword> + /<length>/<keyword> + XXX this is currently disabled as it is not of use to any test. """ isLeaf = True def _gen_random_string(self, length): @@ -29,64 +57,14 @@ class HTTPRandomPage(resource.Resource): data += '\n' return data

- def render(self, request): + def all(self, length, keyword): length = 100 - keyword = None - path_parts = request.path.split('/') - if len(path_parts) > 2: - length = int(path_parts[2]) - if length > 100000: - length = 100000 - - if len(path_parts) > 3: - keyword = escape(path_parts[3]) - + if length > 100000: + length = 100000 return self.genRandomPage(length, keyword)

-class HTTPReturnHeaders(resource.Resource): - """ - This returns the headers being sent by the client in JSON format. - """ - isLeaf = True - def render(self, request): - req_headers = request.getAllHeaders() - return json.dumps(req_headers) - -class HTTPSendHeaders(resource.Resource): - """ - This sends to the client the headers that they send inside of the POST - request encoded in json. - """ - isLeaf = True - def render_POST(self, request): - headers = json.loads(request.content.read()) - for header, value in headers.items(): - request.setHeader(str(header), str(value)) - return '' - -class HTTPBackend(resource.Resource): - def __init__(self): - resource.Resource.__init__(self) - self.putChild('random', HTTPRandomPage()) - self.putChild('returnheaders', HTTPReturnHeaders()) - self.putChild('sendheaders', HTTPSendHeaders()) - -class DebugProtocol(http.HTTPChannel): - def headerReceived(self, line): - print "[HEADER] %s" % line - http.HTTPChannel.headerReceived(self, line) - - def allContentReceived(self): - print self.requests[-1].getAllHeaders() - self.transport.loseConnection() - self.connectionLost("Normal closure") - -class DebugHTTPServer(http.HTTPFactory): - protocol = DebugProtocol - - def buildProtocol(self, addr): - print "Got connection from %s" % addr - p = protocol.ServerFactory.buildProtocol(self, addr) - p.timeOut = self.timeOut - return p +HTTPRandomPageHelper = Application([ + # XXX add regexps here + (r"/(.*)/(.*)", HTTPRandomPage) +])