commit fd52ee29a712f23b2e6c82922a27f848c0ffd98b Author: Arturo Filastò art@fuffa.org Date: Sat Jun 8 11:48:32 2013 +0200

Remove dependency from Storm

We are currently not using the db so there is no need for this. --- oonib/__init__.py | 17 ++----- oonib/db/__init__.py | 30 ------------ oonib/db/tables.py | 123 ----------------------------------------------- oonib/models.py | 129 -------------------------------------------------- oonib/oonibackend.py | 3 -- oonib/report/api.py | 55 +-------------------- requirements.txt | 2 - 7 files changed, 4 insertions(+), 355 deletions(-)

diff --git a/oonib/__init__.py b/oonib/__init__.py index 30e446a..c374f53 100644 --- a/oonib/__init__.py +++ b/oonib/__init__.py @@ -6,18 +6,14 @@ In here we shall keep track of all variables and objects that should be instantiated only once and be common to pieces of GLBackend code. """ -__all__ = ['database', 'db_threadpool', 'Storage', 'randomStr']

-from twisted.python.threadpool import ThreadPool +__version__ = '0.9.1' + +__all__ = ['Storage', 'randomStr']

-from storm.uri import URI -from storm.twisted.transact import Transactor -from storm.databases.sqlite import SQLite import string import random

-__version__ = '0.9.1' - class Storage(dict): """ A Storage object is like a dictionary except `obj.foo` can be used @@ -69,10 +65,3 @@ def randomStr(length, num=True): if num: chars += string.digits return ''.join(random.choice(chars) for x in range(length)) - -from oonib import config - -database = SQLite(URI(config.main.database_uri)) -db_threadpool = ThreadPool(0, config.main.db_threadpool_size) -db_threadpool.start() -transactor = Transactor(db_threadpool) diff --git a/oonib/db/__init__.py b/oonib/db/__init__.py deleted file mode 100644 index 2839747..0000000 --- a/oonib/db/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -__all__ = ['createTables', 'database', 'transactor'] - -from twisted.python.threadpool import ThreadPool -from twisted.internet.defer import inlineCallbacks, returnValue, Deferred - -from storm.locals import Store -from storm.uri import URI -from storm.databases.sqlite import SQLite - -from oonib import database, transactor -from oonib import log - -@inlineCallbacks -def createTables(): - """ - XXX this is to be refactored and only exists for experimentation. - """ - from oonib.db import models - for model_name in models.__all__: - try: - model = getattr(m, model_name) - except Exception, e: - log.err("Error in db initting") - log.err(e) - try: - log.debug("Creating %s" % model) - yield tables.runCreateTable(model, transactor, database) - except Exception, e: - log.debug(str(e)) - diff --git a/oonib/db/tables.py b/oonib/db/tables.py deleted file mode 100644 index 908a295..0000000 --- a/oonib/db/tables.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -from twisted.internet.defer import inlineCallbacks - -from storm.locals import Store -from storm.properties import PropertyColumn -from storm.exceptions import StormError - -from storm.variables import BoolVariable, DateTimeVariable, DateVariable -from storm.variables import DecimalVariable, EnumVariable -from storm.variables import FloatVariable, IntVariable, RawStrVariable -from storm.variables import UnicodeVariable, JSONVariable, PickleVariable - -def variableToSQLite(var_type): - """ - We take as input a storm.variable and we output the SQLite string it - represents. - """ - sqlite_type = "VARCHAR" - if isinstance(var_type, BoolVariable): - sqlite_type = "INTEGER" - elif isinstance(var_type, DateTimeVariable): - pass - sqlite_type = "" - elif isinstance(var_type, DateVariable): - pass - elif isinstance(var_type, DecimalVariable): - pass - elif isinstance(var_type, EnumVariable): - sqlite_type = "BLOB" - elif isinstance(var_type, FloatVariable): - sqlite_type = "REAL" - elif isinstance(var_type, IntVariable): - sqlite_type = "INTEGER" - elif isinstance(var_type, RawStrVariable): - sqlite_type = "BLOB" - elif isinstance(var_type, UnicodeVariable): - pass - elif isinstance(var_type, JSONVariable): - sqlite_type = "BLOB" - elif isinstance(var_type, PickleVariable): - sqlite_type = "BLOB" - return "%s" % sqlite_type - -def varsToParametersSQLite(variables, primary_keys): - """ - Takes as input a list of variables (convered to SQLite syntax and in the - form of strings) and primary_keys. - Outputs these variables converted into paramter syntax for SQLites. - - ex. - variables: ["var1 INTEGER", "var2 BOOL", "var3 INTEGER"] - primary_keys: ["var1"] - - output: "(var1 INTEGER, var2 BOOL, var3 INTEGER PRIMARY KEY (var1))" - """ - params = "(" - for var in variables[:-1]: - params += "%s %s, " % var - if len(primary_keys) > 0: - params += "%s %s, " % variables[-1] - params += "PRIMARY KEY (" - for key in primary_keys[:-1]: - params += "%s, " % key - params += "%s))" % primary_keys[-1] - else: - params += "%s %s)" % variables[-1] - return params - -def generateCreateQuery(model): - """ - This takes as input a Storm model and outputs the creation query for it. - """ - query = "CREATE TABLE "+ model.__storm_table__ + " " - - variables = [] - primary_keys = [] - - for attr in dir(model): - a = getattr(model, attr) - if isinstance(a, PropertyColumn): - var_stype = a.variable_factory() - var_type = variableToSQLite(var_stype) - name = a.name - variables.append((name, var_type)) - if a.primary: - primary_keys.append(name) - - query += varsToParametersSQLite(variables, primary_keys) - return query - -def createTable(model, transactor, database): - """ - Create the table for the specified model. - Specification of a transactor and database is useful in unittesting. - """ - if not transactor: - from oonib.db import transactor - if not database: - from oonib.db import database - store = Store(database) - create_query = generateCreateQuery(model) - try: - store.execute(create_query) - # XXX trap the specific error that is raised when the table exists - except StormError, e: - print "Failed to create table!" - print e - store.close() - store.commit() - store.close() - -@inlineCallbacks -def runCreateTable(model, transactor=None, database=None): - """ - Runs the table creation query wrapped in a transaction. - Transactions run in a separate thread. - """ - yield transactor.run(createTable, model, transactor, database) - diff --git a/oonib/models.py b/oonib/models.py deleted file mode 100644 index 60c0472..0000000 --- a/oonib/models.py +++ /dev/null @@ -1,129 +0,0 @@ -__all__ = ['Report', 'TestHelperTMP'] -from storm.twisted.transact import transact -from storm.locals import * - -from oonib import randomStr -from oonib import transactor - -def generateReportID(): - """ - Generates a report ID for usage in the database backed oonib collector. - - XXX note how this function is different from the one in report/api.py - """ - report_id = randomStr(100) - return report_id - -class OModel(object): - - transactor = transactor - - def getStore(self): - return Store(database) - - @transact - def create(self, query): - store = Store(database) - store.execute(query) - store.commit() - - @transact - def save(self): - store = getStore() - store.add(self) - store.commit() - -class Report(OModel): - """ - This represents an OONI Report as stored in the database. - - report_id: this is generated by the backend and is used by the client to - reference a previous report and append to it. It should be - treated as a shared secret between the probe and backend. - - software_name: this indicates the name of the software performing the test - (this will default to ooniprobe) - - software_version: this is the version number of the software running the - test. - - test_name: the name of the test on which the report is being created. - - test_version: indicates the version of the test - - progress: what is the current progress of the report. This allows clients - to report event partial reports up to a certain percentage of - progress. Once the report is complete progress will be 100. - - content: what is the content of the report. If the current progress is less - than 100 we should append to the YAML data structure that is - currently stored in such field. - - XXX this is currently not used. - """ - __storm_table__ = 'reports' - - createQuery = "CREATE TABLE " + __storm_table__ +\ - "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\ - "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\ - "progress VARCHAR, content VARCHAR)" - - - id = Int(primary=True) - - report_id = Unicode() - - software_name = Unicode() - software_version = Unicode() - test_name = Unicode() - test_version = Unicode() - progress = Int() - - content = Unicode() - - @transact - def new(report): - store = self.getStore() - - print "Creating new report %s" % report - - report_id = generateReportID() - new_report = models.Report() - new_report.report_id = unicode(report_id) - - new_report.software_name = report['software_name'] - new_report.software_version = report['software_version'] - new_report.test_name = report['test_name'] - new_report.test_version = report['test_version'] - new_report.progress = report['progress'] - - if 'content' in report: - new_report.content = report['content'] - - print "Report: %s" % report - - store.add(new_report) - try: - store.commit() - except: - store.close() - - defer.returnValue({'backend_version': backend_version, 'report_id': - report_id}) - - -class TestHelperTMP(OModel): - __storm_table__ = 'testhelpertmp' - - createQuery = "CREATE TABLE " + __storm_table__ +\ - "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\ - " client_ip VARCHAR, creation_time VARCHAR)" - - id = Int(primary=True) - - report_id = Unicode() - - test_helper = Unicode() - client_ip = Unicode() - - creation_time = Date() diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 0eef588..afd961f 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -20,7 +20,6 @@ from oonib.testhelpers import http_helpers, tcp_helpers

from oonib import log

-from oonib import db_threadpool from oonib import config

if config.main.uid and config.main.gid: @@ -73,5 +72,3 @@ if config.helpers.http_return_request.port: int(config.helpers.http_return_request.port), http_helpers.HTTPReturnJSONHeadersHelper()) http_return_request_helper.setServiceParent(serviceCollection) - -reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop) diff --git a/oonib/report/api.py b/oonib/report/api.py index d01f47c..7747a1e 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -20,7 +20,7 @@ from cyclone import web from oonib import otime from oonib import randomStr

-from oonib import models, config +from oonib import config from oonib.report import file_collector

def parseUpdateReportRequest(request): @@ -45,59 +45,6 @@ def parseUpdateReportRequest(request):

return parsed_request

- -class NewReportHandlerDB(web.RequestHandler): - """ - Responsible for creating and updating reports via database. - XXX this is not yet fully implemented. - """ - - @web.asynchronous - @defer.inlineCallbacks - def post(self): - """ - Creates a new report with the input to the database. - XXX this is not yet implemented. - - * Request - - {'software_name': 'XXX', - 'software_version': 'XXX', - 'test_name': 'XXX', - 'test_version': 'XXX', - 'progress': 'XXX', - 'content': 'XXX' - } - - Optional: - 'test_helper': 'XXX' - 'client_ip': 'XXX' - - * Response - - {'backend_version': 'XXX', 'report_id': 'XXX'} - - """ - report_data = json.loads(self.request.body) - new_report = models.Report() - log.debug("Got this request %s" % report_data) - result = yield new_report.new(report_data) - self.write(result) - self.finish() - - def put(self): - """ - Update an already existing report with the database. - - XXX this is not yet implemented. - - {'report_id': 'XXX', - 'content': 'XXX' - } - """ - pass - - reportingBackendAPI = [ (r"/report/([a-zA-Z0-9_-]+)/close", file_collector.CloseReportHandlerFile), (r"/report", file_collector.NewReportHandlerFile), diff --git a/requirements.txt b/requirements.txt index b0d7f75..08fd114 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,8 +32,6 @@ pygeoip>=0.2.6 ## https://people.torproject.org/~ioerror/src/mirrors/ooniprobe/scapy-02-25-201... transaction>=1.4.1 -#https://pypi.python.org/packages/source/t/transaction/transaction-1.4.1.zip#... -storm>=0.19 #https://pypi.python.org/packages/source/s/storm/storm-0.19.tar.gz#md5=61d1ee... txtorcon>=0.7 pyOpenSSL>=0.13