[tor-commits] [ooni-probe/master] Do some major refactoring of the ooniprobe backend
art at torproject.org
art at torproject.org
Sun Nov 11 17:18:31 UTC 2012
commit dce255af61ae3ee89a0d37776103919dd86e4374
Author: Arturo Filastò <art at fuffa.org>
Date: Sun Nov 11 15:41:28 2012 +0100
Do some major refactoring of the ooniprobe backend
* Implement TCP Echo test helper
* Refactor testhelpers into proper namespace
* Remove unused imports
* Remove files that were not being imported anywhere
---
oonib/config.py | 39 ++++++++++++------
oonib/lib/__init__.py | 1 -
oonib/lib/ssl.py | 8 ----
oonib/oonibackend.conf.sample | 12 ------
oonib/oonibackend.py | 79 ++++++++++++++++++++----------------
oonib/testhelpers/__init__.py | 5 ++
oonib/testhelpers/dns.py | 16 -------
oonib/testhelpers/dns_helpers.py | 16 +++++++
oonib/testhelpers/http_helpers.py | 80 +++++++++++++++++++++++++++++++++++++
oonib/testhelpers/httph.py | 80 -------------------------------------
oonib/testhelpers/ssl_helpers.py | 9 ++++
oonib/testhelpers/tcp_helpers.py | 14 ++++++
ooniprobe.conf | 3 +-
13 files changed, 195 insertions(+), 167 deletions(-)
diff --git a/oonib/config.py b/oonib/config.py
index 0156b69..dc2be2f 100644
--- a/oonib/config.py
+++ b/oonib/config.py
@@ -9,21 +9,34 @@ def get_root_path():
# XXX convert this to something that is a proper config file
main = Storage()
-main.reporting_port = 8888
-main.http_port = 8080
-main.dns_udp_port = 5354
-main.dns_tcp_port = 8002
-main.daphn3_port = 9666
-main.server_version = "Apache"
+main.collector_port = 8888
+
+# XXX make this be the server name that is of
main.database_uri = "sqlite:"+get_root_path()+"oonib_test_db.db"
main.db_threadpool_size = 10
-#main.ssl_private_key = /path/to/data/private.key
-#main.ssl_certificate = /path/to/data/certificate.crt
-#main.ssl_port = 8433
helpers = Storage()
-helpers.http_return_request_port = 1234
-daphn3 = Storage()
-daphn3.yaml_file = "/path/to/data/oonib/daphn3.yaml"
-daphn3.pcap_file = "/path/to/data/server.pcap"
+helpers.http_return_request = Storage()
+helpers.http_return_request.port = 57001
+helpers.http_return_request.server_version = "Apache"
+
+helpers.tcp_echo = Storage()
+helpers.tcp_echo.port = 57002
+
+helpers.daphn3 = Storage()
+helpers.daphn3.yaml_file = "/path/to/data/oonib/daphn3.yaml"
+helpers.daphn3.pcap_file = "/path/to/data/server.pcap"
+
+helpers.daphn3.port = 57003
+
+helpers.dns = Storage()
+helpers.dns.udp_port = 57004
+helpers.dns.tcp_port = 57005
+
+helpers.ssl = Storage()
+#helpers.ssl.private_key = /path/to/data/private.key
+#helpers.ssl.certificate = /path/to/data/certificate.crt
+#helpers.ssl.port = 57007
+
+
diff --git a/oonib/lib/__init__.py b/oonib/lib/__init__.py
deleted file mode 100644
index 8b13789..0000000
--- a/oonib/lib/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/oonib/lib/ssl.py b/oonib/lib/ssl.py
deleted file mode 100644
index 20e99e8..0000000
--- a/oonib/lib/ssl.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from twisted.internet import ssl
-from oonib import config
-
-class SSLContext(ssl.DefaultOpenSSLContextFactory):
- def __init__(self, *args, **kw):
- ssl.DefaultOpenSSLContextFactory.__init__(self, config.main.ssl_private_key,
- config.main.ssl_certificate)
-
diff --git a/oonib/oonibackend.conf.sample b/oonib/oonibackend.conf.sample
deleted file mode 100644
index dc0a662..0000000
--- a/oonib/oonibackend.conf.sample
+++ /dev/null
@@ -1,12 +0,0 @@
-[main]
-reporting_port = 8888
-http_port = 8080
-dns_udp_port = 5354
-dns_tcp_port = 8002
-daphn3_port = 9666
-server_version = Apache
-ssl_private_key = /path/to/private.key
-ssl_certificate = /path/to/certificate.crt
-[daphn3]
-pcap_file = /path/to/server.pcap
-yaml_file = /path/to/server.yaml
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 799dcf0..8f47e73 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -3,57 +3,53 @@
#
# This is the backend system responsible for running certain services that are
# useful for censorship detection.
-
-import json
-import random
-import string
+#
+# In here we start all the test helpers that are required by ooniprobe and
+# start the report collector
from twisted.application import internet
-
-from twisted.internet import protocol, reactor, defer
+from twisted.internet import reactor
from twisted.application import internet, service
from twisted.application.service import Application
-
-from twisted.web import resource, server, static
-from twisted.web.microdom import escape
-from twisted.protocols import basic
from twisted.names import dns
from ooni.utils import log
from oonib.report.api import reportingBackend
-from oonib.lib.ssl import SSLContext
+
from oonib import config
-from oonib.testhelpers.httph import HTTPReturnJSONHeadersHelper
-from oonib.testhelpers.dns import ProxyDNSServer
+from oonib.testhelpers import dns_helpers, ssl_helpers
+from oonib.testhelpers import http_helpers, tcp_helpers
+
#from oonib.testhelpers.daphn3 import Daphn3Server
from oonib import db_threadpool
from cyclone import web
-# This tells twisted to set the
-server.version = config.main.server_version
-
application = service.Application('oonibackend')
serviceCollection = service.IServiceCollection(application)
-if config.main.ssl_port:
- internet.SSLServer(int(config.main.ssl_port),
+if config.helpers.ssl.port:
+ log.msg("Starting SSL helper")
+ ssl_helper = internet.SSLServer(int(config.helpers.ssl.port),
server.Site(HTTPBackend()),
- SSLContext(config),
- ).setServiceParent(serviceCollection)
+ SSLContext(config))
+ ssl_helper.setServiceParent(serviceCollection)
# Start the DNS Server related services
-if config.main.dns_tcp_port:
- TCPDNSServer = ProxyDNSServer()
- internet.TCPServer(int(config.main.dns_tcp_port),
- TCPDNSServer).setServiceParent(serviceCollection)
-
-if config.main.dns_udp_port:
- UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer)
- internet.UDPServer(int(config.main.dns_udp_port),
- UDPFactory).setServiceParent(serviceCollection)
+if config.helpers.dns.tcp_port:
+ log.msg("Starting TCP DNS Helper on %s" % config.helpers.dns.tcp_port)
+ tcp_dns_helper = internet.TCPServer(int(config.helpers.dns.tcp_port),
+ dns_helpers.DNSTestHelper())
+ tcp_dns_helper.setServiceParent(serviceCollection)
+
+if config.helpers.dns.udp_port:
+ log.msg("Starting UDP DNS Helper on %s" % config.helpers.dns.udp_port)
+ udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper())
+ udp_dns_helper = internet.UDPServer(int(config.helpers.dns.udp_port),
+ udp_dns_factory)
+ udp_dns_helper.setServiceParent(serviceCollection)
# XXX this needs to be ported
# Start the OONI daphn3 backend
@@ -62,12 +58,23 @@ if config.main.dns_udp_port:
# internet.TCPServer(int(config.main.daphn3_port),
# daphn3).setServiceParent(serviceCollection)
-if config.main.reporting_port:
- internet.TCPServer(int(config.main.reporting_port),
- reportingBackend).setServiceParent(serviceCollection)
-
-if config.helpers.http_return_request_port:
- internet.TCPServer(int(config.helpers.http_return_request_port),
- HTTPReturnJSONHeadersHelper).setServiceParent(serviceCollection)
+if config.main.collector_port:
+ log.msg("Starting Collector on %s" % config.main.collector_port)
+ collector = internet.TCPServer(int(config.main.collector_port),
+ reportingBackend)
+ collector.setServiceParent(serviceCollection)
+
+if config.helpers.tcp_echo.port:
+ log.msg("Starting TCP echo helper on %s" % config.helpers.tcp_echo.port)
+ tcp_echo_helper = internet.TCPServer(int(config.helpers.tcp_echo.port),
+ tcp_helpers.TCPEchoHelper())
+ tcp_echo_helper.setServiceParent(serviceCollection)
+
+if config.helpers.http_return_request.port:
+ log.msg("Starting HTTP return request helper on %s" % config.helpers.http_return_request.port)
+ http_return_request_helper = internet.TCPServer(
+ int(config.helpers.http_return_request.port),
+ http_helpers.HTTPReturnJSONHeadersHelper)
+ http_return_request_helper.setServiceParent(serviceCollection)
reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop)
diff --git a/oonib/testhelpers/__init__.py b/oonib/testhelpers/__init__.py
index e69de29..4dbb547 100644
--- a/oonib/testhelpers/__init__.py
+++ b/oonib/testhelpers/__init__.py
@@ -0,0 +1,5 @@
+from . import dns_helpers
+from . import http_helpers
+from . import tcp_helpers
+
+__all__ = ['dns_helpers', 'http_helpers', 'tcp_helpers']
diff --git a/oonib/testhelpers/dns.py b/oonib/testhelpers/dns.py
deleted file mode 100644
index 689d1a4..0000000
--- a/oonib/testhelpers/dns.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from twisted.internet.protocol import Factory, Protocol
-from twisted.internet import reactor
-from twisted.names import dns
-from twisted.names import client, server
-
-class ProxyDNSServer(server.DNSServerFactory):
- def __init__(self, authorities = None,
- caches = None, clients = None,
- verbose = 0):
- resolver = client.Resolver(servers=[('8.8.8.8', 53)])
- server.DNSServerFactory.__init__(self, authorities = authorities,
- caches = caches, clients = [resolver],
- verbose = verbose)
- def handleQuery(self, message, protocol, address):
- print message, protocol, address
- server.DNSServerFactory.handleQuery(self, message, protocol, address)
diff --git a/oonib/testhelpers/dns_helpers.py b/oonib/testhelpers/dns_helpers.py
new file mode 100644
index 0000000..cb4ff9f
--- /dev/null
+++ b/oonib/testhelpers/dns_helpers.py
@@ -0,0 +1,16 @@
+from twisted.internet.protocol import Factory, Protocol
+from twisted.internet import reactor
+from twisted.names import dns
+from twisted.names import client, server
+
+class DNSTestHelper(server.DNSServerFactory):
+ def __init__(self, authorities = None,
+ caches = None, clients = None,
+ verbose = 0):
+ resolver = client.Resolver(servers=[('8.8.8.8', 53)])
+ server.DNSServerFactory.__init__(self, authorities = authorities,
+ caches = caches, clients = [resolver],
+ verbose = verbose)
+ def handleQuery(self, message, protocol, address):
+ print message, protocol, address
+ server.DNSServerFactory.handleQuery(self, message, protocol, address)
diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py
new file mode 100644
index 0000000..7001fa6
--- /dev/null
+++ b/oonib/testhelpers/http_helpers.py
@@ -0,0 +1,80 @@
+import json
+import random
+import string
+
+from twisted.application import internet, service
+from twisted.internet import protocol, reactor, defer
+from twisted.protocols import basic
+from twisted.web import resource, server, static, http
+from twisted.web.microdom import escape
+
+from cyclone.web import RequestHandler, Application
+
+class HTTPTrapAll(RequestHandler):
+ """
+ Master class to be used to trap all the HTTP methods and make capitalized
+ requests pass.
+ """
+ def _execute(self, transforms, *args, **kwargs):
+ self._transforms = transforms
+ defer.maybeDeferred(self.prepare).addCallbacks(
+ self._execute_handler,
+ lambda f: self._handle_request_exception(f.value),
+ callbackArgs=(args, kwargs))
+
+ def _execute_handler(self, r, args, kwargs):
+ if not self._finished:
+ args = [self.decode_argument(arg) for arg in args]
+ kwargs = dict((k, self.decode_argument(v, name=k))
+ for (k, v) in kwargs.iteritems())
+ # This is where we do the patching
+ # XXX this is somewhat hackish
+ d = defer.maybeDeferred(self.all, *args, **kwargs)
+ d.addCallbacks(self._execute_success, self._execute_failure)
+ self.notifyFinish().addCallback(self.on_connection_close)
+
+class HTTPReturnJSONHeaders(HTTPTrapAll):
+ def all(self):
+ # XXX make sure that the request headers are in the correct order
+ submitted_data = {'request_body': self.request.body,
+ 'request_headers': self.request.headers,
+ 'request_uri': self.request.uri,
+ 'request_method': self.request.method}
+ response = json.dumps(submitted_data)
+ self.write(response)
+
+HTTPReturnJSONHeadersHelper = Application([
+ (r"/*", HTTPReturnJSONHeaders)
+])
+
+
+class HTTPRandomPage(HTTPTrapAll):
+ """
+ This generates a random page of arbitrary length and containing the string
+ selected by the user.
+ /<length>/<keyword>
+ XXX this is currently disabled as it is not of use to any test.
+ """
+ isLeaf = True
+ def _gen_random_string(self, length):
+ return ''.join(random.choice(string.letters) for x in range(length))
+
+ def genRandomPage(self, length=100, keyword=None):
+ data = self._gen_random_string(length/2)
+ if keyword:
+ data += keyword
+ data += self._gen_random_string(length - length/2)
+ data += '\n'
+ return data
+
+ def all(self, length, keyword):
+ length = 100
+ if length > 100000:
+ length = 100000
+ return self.genRandomPage(length, keyword)
+
+HTTPRandomPageHelper = Application([
+ # XXX add regexps here
+ (r"/(.*)/(.*)", HTTPRandomPage)
+])
+
diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py
deleted file mode 100644
index 7001fa6..0000000
--- a/oonib/testhelpers/httph.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import json
-import random
-import string
-
-from twisted.application import internet, service
-from twisted.internet import protocol, reactor, defer
-from twisted.protocols import basic
-from twisted.web import resource, server, static, http
-from twisted.web.microdom import escape
-
-from cyclone.web import RequestHandler, Application
-
-class HTTPTrapAll(RequestHandler):
- """
- Master class to be used to trap all the HTTP methods and make capitalized
- requests pass.
- """
- def _execute(self, transforms, *args, **kwargs):
- self._transforms = transforms
- defer.maybeDeferred(self.prepare).addCallbacks(
- self._execute_handler,
- lambda f: self._handle_request_exception(f.value),
- callbackArgs=(args, kwargs))
-
- def _execute_handler(self, r, args, kwargs):
- if not self._finished:
- args = [self.decode_argument(arg) for arg in args]
- kwargs = dict((k, self.decode_argument(v, name=k))
- for (k, v) in kwargs.iteritems())
- # This is where we do the patching
- # XXX this is somewhat hackish
- d = defer.maybeDeferred(self.all, *args, **kwargs)
- d.addCallbacks(self._execute_success, self._execute_failure)
- self.notifyFinish().addCallback(self.on_connection_close)
-
-class HTTPReturnJSONHeaders(HTTPTrapAll):
- def all(self):
- # XXX make sure that the request headers are in the correct order
- submitted_data = {'request_body': self.request.body,
- 'request_headers': self.request.headers,
- 'request_uri': self.request.uri,
- 'request_method': self.request.method}
- response = json.dumps(submitted_data)
- self.write(response)
-
-HTTPReturnJSONHeadersHelper = Application([
- (r"/*", HTTPReturnJSONHeaders)
-])
-
-
-class HTTPRandomPage(HTTPTrapAll):
- """
- This generates a random page of arbitrary length and containing the string
- selected by the user.
- /<length>/<keyword>
- XXX this is currently disabled as it is not of use to any test.
- """
- isLeaf = True
- def _gen_random_string(self, length):
- return ''.join(random.choice(string.letters) for x in range(length))
-
- def genRandomPage(self, length=100, keyword=None):
- data = self._gen_random_string(length/2)
- if keyword:
- data += keyword
- data += self._gen_random_string(length - length/2)
- data += '\n'
- return data
-
- def all(self, length, keyword):
- length = 100
- if length > 100000:
- length = 100000
- return self.genRandomPage(length, keyword)
-
-HTTPRandomPageHelper = Application([
- # XXX add regexps here
- (r"/(.*)/(.*)", HTTPRandomPage)
-])
-
diff --git a/oonib/testhelpers/ssl_helpers.py b/oonib/testhelpers/ssl_helpers.py
new file mode 100644
index 0000000..2c07275
--- /dev/null
+++ b/oonib/testhelpers/ssl_helpers.py
@@ -0,0 +1,9 @@
+from twisted.internet import ssl
+from oonib import config
+
+class SSLContext(ssl.DefaultOpenSSLContextFactory):
+ def __init__(self, *args, **kw):
+ ssl.DefaultOpenSSLContextFactory.__init__(self,
+ config.helpers.ssl.private_key,
+ config.main.ssl.certificate)
+
diff --git a/oonib/testhelpers/tcp_helpers.py b/oonib/testhelpers/tcp_helpers.py
new file mode 100644
index 0000000..57b93d2
--- /dev/null
+++ b/oonib/testhelpers/tcp_helpers.py
@@ -0,0 +1,14 @@
+from twisted.internet.protocol import Protocol, Factory
+
+class TCPEchoProtocol(Protocol):
+ def dataReceived(self, data):
+ self.transport.write(data)
+
+class TCPEchoHelper(Factory):
+ """
+ A very simple echo protocol implementation
+ """
+ protocol = TCPEchoProtocol
+
+
+
diff --git a/ooniprobe.conf b/ooniprobe.conf
index 47f480a..191d809 100644
--- a/ooniprobe.conf
+++ b/ooniprobe.conf
@@ -20,7 +20,8 @@ advanced:
# XXX change this to point to the directory where you have stored the GeoIP
# database file. This should be the directory in which OONI is installed
# /path/to/ooni-probe/data/
- geoip_data_dir: /usr/share/GeoIP/
+ #geoip_data_dir: /usr/share/GeoIP/
+ geoip_data_dir: /home/x/code/networking/ooni-probe/data/
debug: true
threadpool_size: 10
More information about the tor-commits
mailing list