[tor-commits] [ooni-probe/master] Fully switch to using only cyclone for HTTP related test helpers

art at torproject.org art at torproject.org
Sat Nov 10 14:02:09 UTC 2012


commit 93bcc2f5f9d90b1548968807cf3404e19559b362
Author: Arturo Filastò <art at fuffa.org>
Date:   Sat Nov 10 14:16:02 2012 +0100

    Fully switch to using only cyclone for HTTP related test helpers
    * Implement HTTP Request handler backend
    * Fix the startup script of OONIB
    * Disable daphn3 test helper script
    * Various cleanups
---
 bin/oonib                   |   41 ++++++++++++++++--
 ooni/utils/log.py           |   18 +++++++-
 oonib/lib/__init__.py       |    7 ---
 oonib/lib/ssl.py            |    2 +-
 oonib/oonibackend.py        |   29 +++++--------
 oonib/report/db/__init__.py |    4 +-
 oonib/testhelpers/daphn3.py |    4 +-
 oonib/testhelpers/httph.py  |  100 +++++++++++++++++--------------------------
 8 files changed, 107 insertions(+), 98 deletions(-)

diff --git a/bin/oonib b/bin/oonib
index df7c62f..0808743 100755
--- a/bin/oonib
+++ b/bin/oonib
@@ -1,4 +1,37 @@
-#!/bin/sh
-ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
-export PYTHONPATH=$PYTHONPATH:$ROOT
-twistd -ny $ROOT/oonib/oonibackend.py
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+#
+# :authors: Arturo Filastò
+# :licence: see LICENSE
+
+import sys
+import os
+# Avoid the export PYTHONPATH insanity
+sys.path[:] = map(os.path.abspath, sys.path)
+this_directory = os.path.dirname(__file__)
+root = os.path.abspath(os.path.join(this_directory, '..'))
+sys.path.insert(0, root)
+
+backend_script = os.path.join(root, 'oonib', 'oonibackend.py')
+
+from twisted.python import log, usage
+from twisted.internet import reactor
+from twisted.application import app
+
+from oonib import runner
+from oonib.oonibackend import application
+
+sys.argv[1:] = ['-ny', backend_script]
+
+def runApp(config):
+    runner.OBaseRunner(config).run()
+
+config = runner.ServerOptions()
+try:
+    config.parseOptions()
+except usage.error, ue:
+    print config
+    print "%s: %s" % (sys.argv[0], ue)
+else:
+    runApp(config)
+
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 0cb62ac..c05c776 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -13,7 +13,7 @@ from twisted.python.logfile import DailyLogFile
 from ooni.utils import otime
 from ooni import config
 
-def start(logfile=None):
+def start(logfile=None, application_name="OONI"):
     daily_logfile = None
 
     if not logfile:
@@ -24,7 +24,7 @@ def start(logfile=None):
 
     daily_logfile = DailyLogFile(log_filename, log_folder)
 
-    txlog.msg("Starting OONI on %s (%s UTC)" %  (otime.prettyDateNow(),
+    txlog.msg("Starting %s on %s (%s UTC)" %  (application_name, otime.prettyDateNow(),
                                                  otime.utcPrettyDateNow()))
     logging.basicConfig()
     python_logging = txlog.PythonLoggingObserver()
@@ -53,3 +53,17 @@ def err(msg, *arg, **kw):
 def exception(*msg):
     logging.exception(msg)
 
+class LoggerFactory(object):
+    """
+    This is a logger factory to be used by oonib
+    """
+    def __init__(self, options):
+        pass
+
+    def start(self, application):
+        # XXX parametrize this
+        start('/tmp/oonib.log', "OONIB")
+
+    def stop(self):
+        txlog.msg("Stopping OONIB")
+
diff --git a/oonib/lib/__init__.py b/oonib/lib/__init__.py
index 5b4380a..8b13789 100644
--- a/oonib/lib/__init__.py
+++ b/oonib/lib/__init__.py
@@ -1,8 +1 @@
-from ooni.utils import Storage
-from ooni.utils.config import Config
-import os
 
-root = os.path.normpath(os.path.join(os.path.realpath(__file__), '../../'))
-config = Storage()
-config.main = Config('main', os.path.join(root, 'oonibackend.conf'))
-config.daphn3 = Config('daphn3', os.path.join(root, 'oonibackend.conf'))
diff --git a/oonib/lib/ssl.py b/oonib/lib/ssl.py
index 094bdc9..20e99e8 100644
--- a/oonib/lib/ssl.py
+++ b/oonib/lib/ssl.py
@@ -1,5 +1,5 @@
 from twisted.internet import ssl
-from oonib.lib import config
+from oonib import config
 
 class SSLContext(ssl.DefaultOpenSSLContextFactory):
     def __init__(self, *args, **kw):
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index eaa52f9..9964854 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -24,36 +24,26 @@ from ooni.utils import log
 
 from oonib.report.api import reportingBackend
 from oonib.lib.ssl import SSLContext
-from oonib.lib import config
+from oonib import config
 
-from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer
+from oonib.testhelpers.httph import HTTPReturnJSONHeadersHelper
 from oonib.testhelpers.dns import ProxyDNSServer
-from oonib.testhelpers.daphn3 import Daphn3Server
+#from oonib.testhelpers.daphn3 import Daphn3Server
 
 from cyclone import web
 
-log.start('/tmp/oonib.log')
-
 # This tells twisted to set the
 server.version = config.main.server_version
 
 application = service.Application('oonibackend')
 serviceCollection = service.IServiceCollection(application)
 
-if config.main.http_port:
-    internet.TCPServer(int(config.main.http_port),
-                   server.Site(HTTPBackend())
-                  ).setServiceParent(serviceCollection)
-
 if config.main.ssl_port:
     internet.SSLServer(int(config.main.ssl_port),
                    server.Site(HTTPBackend()),
                    SSLContext(config),
                   ).setServiceParent(serviceCollection)
 
-debugHTTPServer = DebugHTTPServer()
-internet.TCPServer(8090, debugHTTPServer).setServiceParent(serviceCollection)
-
 # Start the DNS Server related services
 if config.main.dns_tcp_port:
     TCPDNSServer = ProxyDNSServer()
@@ -65,15 +55,18 @@ if config.main.dns_udp_port:
     internet.UDPServer(int(config.main.dns_udp_port),
                        UDPFactory).setServiceParent(serviceCollection)
 
+# XXX this needs to be ported
 # Start the OONI daphn3 backend
-if config.main.daphn3_port:
-    daphn3 = Daphn3Server()
-    internet.TCPServer(int(config.main.daphn3_port),
-                       daphn3).setServiceParent(serviceCollection)
+#if config.main.daphn3_port:
+#    daphn3 = Daphn3Server()
+#    internet.TCPServer(int(config.main.daphn3_port),
+#                       daphn3).setServiceParent(serviceCollection)
 
 if config.main.reporting_port:
     internet.TCPServer(int(config.main.reporting_port),
                        reportingBackend).setServiceParent(serviceCollection)
 
-reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop)
+if config.helpers.http_return_request_port:
+    internet.TCPServer(int(config.helpers.http_return_request_port),
+            HTTPReturnJSONHeadersHelper).setServiceParent(serviceCollection)
 
diff --git a/oonib/report/db/__init__.py b/oonib/report/db/__init__.py
index 178abe9..383b24b 100644
--- a/oonib/report/db/__init__.py
+++ b/oonib/report/db/__init__.py
@@ -1,5 +1,3 @@
-__all__ = ['models']
-
 from twisted.python import log
 from twisted.python.threadpool import ThreadPool
 from twisted.internet.defer import inlineCallbacks
@@ -21,7 +19,7 @@ def getStore():
 
 @inlineCallbacks
 def create_tables():
-    from oonib.report.db import models
+    from oonib.report.models import models
 
     for x in models.__all__:
         query = getattr(models.__getattribute__(x), 'createQuery')
diff --git a/oonib/testhelpers/daphn3.py b/oonib/testhelpers/daphn3.py
index facd11f..3b59f93 100644
--- a/oonib/testhelpers/daphn3.py
+++ b/oonib/testhelpers/daphn3.py
@@ -1,9 +1,9 @@
+# XXX this is currently broken and needs to be ported
 from twisted.internet import protocol
 from twisted.internet.error import ConnectionDone
 
-from oonib.lib import config
+from oonib import config
 
-from ooni.plugoo import reports
 from ooni.protocols.daphn3 import Mutator, Daphn3Protocol
 from ooni.protocols.daphn3 import read_pcap, read_yaml
 
diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py
index 217dbd5..b793852 100644
--- a/oonib/testhelpers/httph.py
+++ b/oonib/testhelpers/httph.py
@@ -8,14 +8,42 @@ from twisted.protocols import basic
 from twisted.web import resource, server, static, http
 from twisted.web.microdom import escape
 
-server.version = "Apache"
+from cyclone.web import RequestHandler, Application
 
-class HTTPRandomPage(resource.Resource):
+class HTTPTrapAll(RequestHandler):
+    """
+    Master class to be used to trap all the HTTP methods
+    """
+    def get(self, *arg, **kw):
+        self.all(*arg, **kw)
+
+    def post(self, *arg, **kw):
+        self.all(*arg, **kw)
+
+    def put(self, *arg, **kw):
+        self.all(*arg, **kw)
+
+class HTTPReturnJSONHeaders(HTTPTrapAll):
+    def all(self):
+        # XXX make sure that the request headers are in the correct order
+        submitted_data = {'request_body': self.request.body,
+                'request_headers': self.request.headers,
+                'request_uri': self.request.uri,
+                'request_method': self.request.method}
+        response = json.dumps(submitted_data)
+        self.write(response)
+
+HTTPReturnJSONHeadersHelper = Application([
+    (r"/*", HTTPReturnJSONHeaders)
+])
+
+
+class HTTPRandomPage(HTTPTrapAll):
     """
     This generates a random page of arbitrary length and containing the string
     selected by the user.
-    The format is the following:
-    /random/<length>/<keyword>
+    /<length>/<keyword>
+    XXX this is currently disabled as it is not of use to any test.
     """
     isLeaf = True
     def _gen_random_string(self, length):
@@ -29,64 +57,14 @@ class HTTPRandomPage(resource.Resource):
         data += '\n'
         return data
 
-    def render(self, request):
+    def all(self, length, keyword):
         length = 100
-        keyword = None
-        path_parts = request.path.split('/')
-        if len(path_parts) > 2:
-            length = int(path_parts[2])
-            if length > 100000:
-                length = 100000
-
-        if len(path_parts) > 3:
-            keyword = escape(path_parts[3])
-
+        if length > 100000:
+            length = 100000
         return self.genRandomPage(length, keyword)
 
-class HTTPReturnHeaders(resource.Resource):
-    """
-    This returns the headers being sent by the client in JSON format.
-    """
-    isLeaf = True
-    def render(self, request):
-        req_headers = request.getAllHeaders()
-        return json.dumps(req_headers)
-
-class HTTPSendHeaders(resource.Resource):
-    """
-    This sends to the client the headers that they send inside of the POST
-    request encoded in json.
-    """
-    isLeaf = True
-    def render_POST(self, request):
-        headers = json.loads(request.content.read())
-        for header, value in headers.items():
-            request.setHeader(str(header), str(value))
-        return ''
-
-class HTTPBackend(resource.Resource):
-    def __init__(self):
-        resource.Resource.__init__(self)
-        self.putChild('random', HTTPRandomPage())
-        self.putChild('returnheaders', HTTPReturnHeaders())
-        self.putChild('sendheaders', HTTPSendHeaders())
-
-class DebugProtocol(http.HTTPChannel):
-    def headerReceived(self, line):
-        print "[HEADER] %s" % line
-        http.HTTPChannel.headerReceived(self, line)
-
-    def allContentReceived(self):
-        print self.requests[-1].getAllHeaders()
-        self.transport.loseConnection()
-        self.connectionLost("Normal closure")
-
-class DebugHTTPServer(http.HTTPFactory):
-    protocol = DebugProtocol
-
-    def buildProtocol(self, addr):
-        print "Got connection from %s" % addr
-        p = protocol.ServerFactory.buildProtocol(self, addr)
-        p.timeOut = self.timeOut
-        return p
+HTTPRandomPageHelper = Application([
+    # XXX add regexps here
+    (r"/(.*)/(.*)", HTTPRandomPage)
+])
 





More information about the tor-commits mailing list