[tor-commits] [ooni-probe/master] Now we have the database properly hooked up to OONIB
art at torproject.org
art at torproject.org
Fri Oct 5 14:09:04 UTC 2012
commit b1ca4257c13dff18664c8b175ed6b57acc8874b3
Author: Arturo Filastò <arturo at filasto.net>
Date: Fri Oct 5 10:58:57 2012 +0000
Now we have the database properly hooked up to OONIB
* Support writing to DB of data relative to a report
---
oonib/oonibackend.conf.sample | 1 +
oonib/oonibackend.py | 36 ++++++++++++++-----
oonib/report/api.py | 75 +++++++++++++++++++++++++++++-----------
oonib/report/db/__init__.py | 20 +++++-----
oonib/report/db/models.py | 44 +++++++++++++++++++++++-
5 files changed, 134 insertions(+), 42 deletions(-)
diff --git a/oonib/oonibackend.conf.sample b/oonib/oonibackend.conf.sample
index 021e0e2..dc0a662 100644
--- a/oonib/oonibackend.conf.sample
+++ b/oonib/oonibackend.conf.sample
@@ -1,4 +1,5 @@
[main]
+reporting_port = 8888
http_port = 8080
dns_udp_port = 5354
dns_tcp_port = 8002
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 0232829..29f3f58 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -16,6 +16,7 @@ from twisted.web import resource, server, static
from twisted.web.microdom import escape
from twisted.names import dns
+from oonib.report.api import reportingBackend
from oonib.lib import config
from oonib.lib.ssl import SSLContext
from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer
@@ -28,11 +29,13 @@ server.version = config.main.server_version
application = service.Application('oonibackend')
serviceCollection = service.IServiceCollection(application)
-internet.TCPServer(int(config.main.http_port),
+if config.main.http_port:
+ internet.TCPServer(int(config.main.http_port),
server.Site(HTTPBackend())
).setServiceParent(serviceCollection)
-internet.SSLServer(int(config.main.ssl_port),
+if config.main.ssl_port:
+ internet.SSLServer(int(config.main.ssl_port),
server.Site(HTTPBackend()),
SSLContext(config),
).setServiceParent(serviceCollection)
@@ -41,11 +44,24 @@ debugHTTPServer = DebugHTTPServer()
internet.TCPServer(8090, debugHTTPServer).setServiceParent(serviceCollection)
# Start the DNS Server related services
-TCPDNSServer = ProxyDNSServer()
-internet.TCPServer(int(config.main.dns_tcp_port), TCPDNSServer).setServiceParent(serviceCollection)
-UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer)
-internet.UDPServer(int(config.main.dns_udp_port), UDPFactory).setServiceParent(serviceCollection)
-
-# Start the ooni backend thing
-daphn3 = Daphn3Server()
-internet.TCPServer(int(config.main.daphn3_port), daphn3).setServiceParent(serviceCollection)
+if config.main.dns_tcp_port:
+ TCPDNSServer = ProxyDNSServer()
+ internet.TCPServer(int(config.main.dns_tcp_port),
+ TCPDNSServer).setServiceParent(serviceCollection)
+
+if config.main.dns_udp_port:
+ UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer)
+ internet.UDPServer(int(config.main.dns_udp_port),
+ UDPFactory).setServiceParent(serviceCollection)
+
+# Start the OONI daphn3 backend
+if config.main.daphn3_port:
+ daphn3 = Daphn3Server()
+ internet.TCPServer(int(config.main.daphn3_port),
+ daphn3).setServiceParent(serviceCollection)
+
+if config.main.reporting_port:
+ internet.TCPServer(int(config.main.reporting_port),
+ reportingBackend).setServiceParent(serviceCollection)
+
+
diff --git a/oonib/report/__init__.py b/oonib/report/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/oonib/report/api.py b/oonib/report/api.py
index 04f70e0..ad0f62c 100644
--- a/oonib/report/api.py
+++ b/oonib/report/api.py
@@ -7,37 +7,61 @@ This is the async pcap reporting system. It requires the client to have created
"""
import random
-import strings
-from twisted.internet import reactor
+import string
+from twisted.internet import reactor, defer
from cyclone import web
+
+from oonib.report.db import models
backend_version = '0.0.1'
def generateReportID():
size = 100
- id = ''.join(random.choice(strings.ascii_letters) for x in range(size))
- return id
+ report_id = ''.join(random.choice(string.ascii_letters) for x in range(size))
+ return report_id
+ at defer.inlineCallbacks
def newReport(software_name, software_version, test_name, test_version,
progress, content):
+
+ report_id = generateReportID()
+
+ new_report = models.Report()
+
+ new_report.report_id = unicode(report_id)
+
+ new_report.software_name = unicode(software_name)
+ new_report.software_version = unicode(software_version)
+ new_report.test_name = unicode(test_name)
+ new_report.test_version = unicode(test_version)
+ new_report.progress = unicode(progress)
+ new_report.content = unicode(content)
+
print "Software Name: %s" % software_name
print "Software Version: %s" % software_version
print "Test Name: %s" % test_name
print "Test Version: %s" % test_version
print "Progress: %s" % progress
print "Content: %s" % content
- reportId = generateReportID()
- return {'backend_version': backend_version, 'report_id': reportID}
+
+ yield new_report.save()
+
+ defer.returnValue({'backend_version': backend_version, 'report_id':
+ report_id})
def updateReport(report_id, content):
print "Report ID: %s" % report_id
print "Content: %s" % content
- return {'backend_version': '0.1', 'report_id': 'FOOBAR'}
-class DoReportHandler(web.RequestHandler):
+ return {'backend_version': backend_version, 'report_id': report_id}
+
+class NewReportHandler(web.RequestHandler):
"""
Responsible for creating and updating reports.
"""
- def post(self):
+
+ @web.asynchronous
+ @defer.inlineCallbacks
+ def get(self):
"""
Creates a new report with the input
@@ -51,6 +75,10 @@ class DoReportHandler(web.RequestHandler):
'content': 'XXX'
}
+ Optional:
+ 'test_helper': 'XXX'
+ 'client_ip': 'XXX'
+
* Response
{'backend_version': 'XXX', 'report_id': 'XXX'}
@@ -61,15 +89,23 @@ class DoReportHandler(web.RequestHandler):
'test_name','test_version',
'progress', 'content']
report = {}
- error = None
for arg in arguments:
if len(self.get_arguments(arg)) == 0:
- print "No %s specified" % arg
- error = arg
- break
+ raise web.HTTPError(400, "%s not specified as argument of POST"
+ % arg)
report[arg] = self.get_argument(arg)
- if not error:
- self.write(newReport(**report))
+
+ try:
+ test_helper = self.get_argument('test_helper')
+
+
+ except web.HTTPError:
+ pass
+
+ new_report = yield newReport(**report)
+
+ self.write(new_report)
+ self.finish()
def put(self):
"""
@@ -88,11 +124,8 @@ class PCAPReportHandler(web.RequestHandler):
def post(self):
pass
-handlers = [(r"/report/do", DoReportHandler),
- (r"/report/pcap", PCAPReportHandler)]
+spec = [(r"/report/new", NewReportHandler),
+ (r"/report/pcap", PCAPReportHandler)]
-reporting = web.Application(handlers)
-reactor.listenTCP(8888, reporting, interface="127.0.0.1")
-print "starting bullshit"
-reactor.run()
+reportingBackend = web.Application(spec)
diff --git a/oonib/report/db/__init__.py b/oonib/report/db/__init__.py
index 1248c19..178abe9 100644
--- a/oonib/report/db/__init__.py
+++ b/oonib/report/db/__init__.py
@@ -6,28 +6,28 @@ from twisted.internet.defer import inlineCallbacks
from storm.twisted.transact import Transactor, transact
from storm.locals import *
-dbtype = "sqlite"
+from storm.uri import URI
+from storm.databases.sqlite import SQLite
-if dbtype == "sqlite":
- from storm.uri import URI
- from storm.databases.sqlite import SQLite
- database = SQLite(URI('sqlite:///test.db'))
+database = SQLite(URI('sqlite:///test.db'))
threadpool = ThreadPool(0, 10)
threadpool.start()
transactor = Transactor(threadpool)
+def getStore():
+ store = Store(database)
+ return store
+
@inlineCallbacks
def create_tables():
- def create(query):
- store = Store(database)
- store.execute(query)
- store.commit()
+ from oonib.report.db import models
for x in models.__all__:
- query = getattr(models.__getattribute__(x), 'create_query')
+ query = getattr(models.__getattribute__(x), 'createQuery')
try:
yield transactor.run(create, query)
except:
log.msg("Failing in creating table for %s. Maybe it already exists?" % x)
+create_tables()
diff --git a/oonib/report/db/models.py b/oonib/report/db/models.py
index b4e56d0..21e60eb 100644
--- a/oonib/report/db/models.py
+++ b/oonib/report/db/models.py
@@ -1,7 +1,26 @@
+__all__ = ['Report', 'TestHelperTMP']
from storm.twisted.transact import transact
from storm.locals import *
-class Report(object):
+from oonib.report.db import getStore, transactor
+
+class OModel(object):
+
+ transactor = transactor
+
+ @transact
+ def create(query):
+ store = Store(database)
+ store.execute(query)
+ store.commit()
+
+ @transact
+ def save(self):
+ store = getStore()
+ store.add(self)
+ store.commit()
+
+class Report(OModel):
"""
This represents an OONI Report as stored in the database.
@@ -27,6 +46,14 @@ class Report(object):
than 100 we should append to the YAML data structure that is
currently stored in such field.
"""
+ __storm_table__ = 'reports'
+
+ createQuery = "CREATE TABLE " + __storm_table__ +\
+ "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\
+ "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\
+ "progress VARCHAR, content VARCHAR)"
+
+
id = Int(primary=True)
report_id = Unicode()
@@ -39,3 +66,18 @@ class Report(object):
content = Unicode()
+class TestHelperTMP(OModel):
+ __storm_table__ = 'testhelpertmp'
+
+ createQuery = "CREATE TABLE " + __storm_table__ +\
+ "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\
+ " client_ip VARCHAR, creation_time VARCHAR)"
+
+ id = Int(primary=True)
+
+ report_id = Unicode()
+
+ test_helper = Unicode()
+ client_ip = Unicode()
+
+ creation_time = Date()
More information about the tor-commits
mailing list