commit b1ca4257c13dff18664c8b175ed6b57acc8874b3 Author: Arturo Filastò <arturo@filasto.net> Date: Fri Oct 5 10:58:57 2012 +0000 Now we have the database properly hooked up to OONIB * Support writing to DB of data relative to a report --- oonib/oonibackend.conf.sample | 1 + oonib/oonibackend.py | 36 ++++++++++++++----- oonib/report/api.py | 75 +++++++++++++++++++++++++++++----------- oonib/report/db/__init__.py | 20 +++++----- oonib/report/db/models.py | 44 +++++++++++++++++++++++- 5 files changed, 134 insertions(+), 42 deletions(-) diff --git a/oonib/oonibackend.conf.sample b/oonib/oonibackend.conf.sample index 021e0e2..dc0a662 100644 --- a/oonib/oonibackend.conf.sample +++ b/oonib/oonibackend.conf.sample @@ -1,4 +1,5 @@ [main] +reporting_port = 8888 http_port = 8080 dns_udp_port = 5354 dns_tcp_port = 8002 diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 0232829..29f3f58 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -16,6 +16,7 @@ from twisted.web import resource, server, static from twisted.web.microdom import escape from twisted.names import dns +from oonib.report.api import reportingBackend from oonib.lib import config from oonib.lib.ssl import SSLContext from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer @@ -28,11 +29,13 @@ server.version = config.main.server_version application = service.Application('oonibackend') serviceCollection = service.IServiceCollection(application) -internet.TCPServer(int(config.main.http_port), +if config.main.http_port: + internet.TCPServer(int(config.main.http_port), server.Site(HTTPBackend()) ).setServiceParent(serviceCollection) -internet.SSLServer(int(config.main.ssl_port), +if config.main.ssl_port: + internet.SSLServer(int(config.main.ssl_port), server.Site(HTTPBackend()), SSLContext(config), ).setServiceParent(serviceCollection) @@ -41,11 +44,24 @@ debugHTTPServer = DebugHTTPServer() internet.TCPServer(8090, debugHTTPServer).setServiceParent(serviceCollection) # Start the DNS Server related services -TCPDNSServer = ProxyDNSServer() -internet.TCPServer(int(config.main.dns_tcp_port), TCPDNSServer).setServiceParent(serviceCollection) -UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer) -internet.UDPServer(int(config.main.dns_udp_port), UDPFactory).setServiceParent(serviceCollection) - -# Start the ooni backend thing -daphn3 = Daphn3Server() -internet.TCPServer(int(config.main.daphn3_port), daphn3).setServiceParent(serviceCollection) +if config.main.dns_tcp_port: + TCPDNSServer = ProxyDNSServer() + internet.TCPServer(int(config.main.dns_tcp_port), + TCPDNSServer).setServiceParent(serviceCollection) + +if config.main.dns_udp_port: + UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer) + internet.UDPServer(int(config.main.dns_udp_port), + UDPFactory).setServiceParent(serviceCollection) + +# Start the OONI daphn3 backend +if config.main.daphn3_port: + daphn3 = Daphn3Server() + internet.TCPServer(int(config.main.daphn3_port), + daphn3).setServiceParent(serviceCollection) + +if config.main.reporting_port: + internet.TCPServer(int(config.main.reporting_port), + reportingBackend).setServiceParent(serviceCollection) + + diff --git a/oonib/report/__init__.py b/oonib/report/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/report/api.py b/oonib/report/api.py index 04f70e0..ad0f62c 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -7,37 +7,61 @@ This is the async pcap reporting system. It requires the client to have created """ import random -import strings -from twisted.internet import reactor +import string +from twisted.internet import reactor, defer from cyclone import web + +from oonib.report.db import models backend_version = '0.0.1' def generateReportID(): size = 100 - id = ''.join(random.choice(strings.ascii_letters) for x in range(size)) - return id + report_id = ''.join(random.choice(string.ascii_letters) for x in range(size)) + return report_id +@defer.inlineCallbacks def newReport(software_name, software_version, test_name, test_version, progress, content): + + report_id = generateReportID() + + new_report = models.Report() + + new_report.report_id = unicode(report_id) + + new_report.software_name = unicode(software_name) + new_report.software_version = unicode(software_version) + new_report.test_name = unicode(test_name) + new_report.test_version = unicode(test_version) + new_report.progress = unicode(progress) + new_report.content = unicode(content) + print "Software Name: %s" % software_name print "Software Version: %s" % software_version print "Test Name: %s" % test_name print "Test Version: %s" % test_version print "Progress: %s" % progress print "Content: %s" % content - reportId = generateReportID() - return {'backend_version': backend_version, 'report_id': reportID} + + yield new_report.save() + + defer.returnValue({'backend_version': backend_version, 'report_id': + report_id}) def updateReport(report_id, content): print "Report ID: %s" % report_id print "Content: %s" % content - return {'backend_version': '0.1', 'report_id': 'FOOBAR'} -class DoReportHandler(web.RequestHandler): + return {'backend_version': backend_version, 'report_id': report_id} + +class NewReportHandler(web.RequestHandler): """ Responsible for creating and updating reports. """ - def post(self): + + @web.asynchronous + @defer.inlineCallbacks + def get(self): """ Creates a new report with the input @@ -51,6 +75,10 @@ class DoReportHandler(web.RequestHandler): 'content': 'XXX' } + Optional: + 'test_helper': 'XXX' + 'client_ip': 'XXX' + * Response {'backend_version': 'XXX', 'report_id': 'XXX'} @@ -61,15 +89,23 @@ class DoReportHandler(web.RequestHandler): 'test_name','test_version', 'progress', 'content'] report = {} - error = None for arg in arguments: if len(self.get_arguments(arg)) == 0: - print "No %s specified" % arg - error = arg - break + raise web.HTTPError(400, "%s not specified as argument of POST" + % arg) report[arg] = self.get_argument(arg) - if not error: - self.write(newReport(**report)) + + try: + test_helper = self.get_argument('test_helper') + + + except web.HTTPError: + pass + + new_report = yield newReport(**report) + + self.write(new_report) + self.finish() def put(self): """ @@ -88,11 +124,8 @@ class PCAPReportHandler(web.RequestHandler): def post(self): pass -handlers = [(r"/report/do", DoReportHandler), - (r"/report/pcap", PCAPReportHandler)] +spec = [(r"/report/new", NewReportHandler), + (r"/report/pcap", PCAPReportHandler)] -reporting = web.Application(handlers) -reactor.listenTCP(8888, reporting, interface="127.0.0.1") -print "starting bullshit" -reactor.run() +reportingBackend = web.Application(spec) diff --git a/oonib/report/db/__init__.py b/oonib/report/db/__init__.py index 1248c19..178abe9 100644 --- a/oonib/report/db/__init__.py +++ b/oonib/report/db/__init__.py @@ -6,28 +6,28 @@ from twisted.internet.defer import inlineCallbacks from storm.twisted.transact import Transactor, transact from storm.locals import * -dbtype = "sqlite" +from storm.uri import URI +from storm.databases.sqlite import SQLite -if dbtype == "sqlite": - from storm.uri import URI - from storm.databases.sqlite import SQLite - database = SQLite(URI('sqlite:///test.db')) +database = SQLite(URI('sqlite:///test.db')) threadpool = ThreadPool(0, 10) threadpool.start() transactor = Transactor(threadpool) +def getStore(): + store = Store(database) + return store + @inlineCallbacks def create_tables(): - def create(query): - store = Store(database) - store.execute(query) - store.commit() + from oonib.report.db import models for x in models.__all__: - query = getattr(models.__getattribute__(x), 'create_query') + query = getattr(models.__getattribute__(x), 'createQuery') try: yield transactor.run(create, query) except: log.msg("Failing in creating table for %s. Maybe it already exists?" % x) +create_tables() diff --git a/oonib/report/db/models.py b/oonib/report/db/models.py index b4e56d0..21e60eb 100644 --- a/oonib/report/db/models.py +++ b/oonib/report/db/models.py @@ -1,7 +1,26 @@ +__all__ = ['Report', 'TestHelperTMP'] from storm.twisted.transact import transact from storm.locals import * -class Report(object): +from oonib.report.db import getStore, transactor + +class OModel(object): + + transactor = transactor + + @transact + def create(query): + store = Store(database) + store.execute(query) + store.commit() + + @transact + def save(self): + store = getStore() + store.add(self) + store.commit() + +class Report(OModel): """ This represents an OONI Report as stored in the database. @@ -27,6 +46,14 @@ class Report(object): than 100 we should append to the YAML data structure that is currently stored in such field. """ + __storm_table__ = 'reports' + + createQuery = "CREATE TABLE " + __storm_table__ +\ + "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\ + "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\ + "progress VARCHAR, content VARCHAR)" + + id = Int(primary=True) report_id = Unicode() @@ -39,3 +66,18 @@ class Report(object): content = Unicode() +class TestHelperTMP(OModel): + __storm_table__ = 'testhelpertmp' + + createQuery = "CREATE TABLE " + __storm_table__ +\ + "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\ + " client_ip VARCHAR, creation_time VARCHAR)" + + id = Int(primary=True) + + report_id = Unicode() + + test_helper = Unicode() + client_ip = Unicode() + + creation_time = Date()