[tor-commits] [oonib/master] Remove dependency from Storm
art at torproject.org
art at torproject.org
Wed Sep 11 09:13:51 UTC 2013
commit fd52ee29a712f23b2e6c82922a27f848c0ffd98b
Author: Arturo Filastò <art at fuffa.org>
Date: Sat Jun 8 11:48:32 2013 +0200
Remove dependency from Storm
We are currently not using the db so there is no need for this.
---
oonib/__init__.py | 17 ++-----
oonib/db/__init__.py | 30 ------------
oonib/db/tables.py | 123 -----------------------------------------------
oonib/models.py | 129 --------------------------------------------------
oonib/oonibackend.py | 3 --
oonib/report/api.py | 55 +--------------------
requirements.txt | 2 -
7 files changed, 4 insertions(+), 355 deletions(-)
diff --git a/oonib/__init__.py b/oonib/__init__.py
index 30e446a..c374f53 100644
--- a/oonib/__init__.py
+++ b/oonib/__init__.py
@@ -6,18 +6,14 @@
In here we shall keep track of all variables and objects that should be
instantiated only once and be common to pieces of GLBackend code.
"""
-__all__ = ['database', 'db_threadpool', 'Storage', 'randomStr']
-from twisted.python.threadpool import ThreadPool
+__version__ = '0.9.1'
+
+__all__ = ['Storage', 'randomStr']
-from storm.uri import URI
-from storm.twisted.transact import Transactor
-from storm.databases.sqlite import SQLite
import string
import random
-__version__ = '0.9.1'
-
class Storage(dict):
"""
A Storage object is like a dictionary except `obj.foo` can be used
@@ -69,10 +65,3 @@ def randomStr(length, num=True):
if num:
chars += string.digits
return ''.join(random.choice(chars) for x in range(length))
-
-from oonib import config
-
-database = SQLite(URI(config.main.database_uri))
-db_threadpool = ThreadPool(0, config.main.db_threadpool_size)
-db_threadpool.start()
-transactor = Transactor(db_threadpool)
diff --git a/oonib/db/__init__.py b/oonib/db/__init__.py
deleted file mode 100644
index 2839747..0000000
--- a/oonib/db/__init__.py
+++ /dev/null
@@ -1,30 +0,0 @@
-__all__ = ['createTables', 'database', 'transactor']
-
-from twisted.python.threadpool import ThreadPool
-from twisted.internet.defer import inlineCallbacks, returnValue, Deferred
-
-from storm.locals import Store
-from storm.uri import URI
-from storm.databases.sqlite import SQLite
-
-from oonib import database, transactor
-from oonib import log
-
- at inlineCallbacks
-def createTables():
- """
- XXX this is to be refactored and only exists for experimentation.
- """
- from oonib.db import models
- for model_name in models.__all__:
- try:
- model = getattr(m, model_name)
- except Exception, e:
- log.err("Error in db initting")
- log.err(e)
- try:
- log.debug("Creating %s" % model)
- yield tables.runCreateTable(model, transactor, database)
- except Exception, e:
- log.debug(str(e))
-
diff --git a/oonib/db/tables.py b/oonib/db/tables.py
deleted file mode 100644
index 908a295..0000000
--- a/oonib/db/tables.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# :authors: Arturo Filastò
-# :licence: see LICENSE
-
-from twisted.internet.defer import inlineCallbacks
-
-from storm.locals import Store
-from storm.properties import PropertyColumn
-from storm.exceptions import StormError
-
-from storm.variables import BoolVariable, DateTimeVariable, DateVariable
-from storm.variables import DecimalVariable, EnumVariable
-from storm.variables import FloatVariable, IntVariable, RawStrVariable
-from storm.variables import UnicodeVariable, JSONVariable, PickleVariable
-
-def variableToSQLite(var_type):
- """
- We take as input a storm.variable and we output the SQLite string it
- represents.
- """
- sqlite_type = "VARCHAR"
- if isinstance(var_type, BoolVariable):
- sqlite_type = "INTEGER"
- elif isinstance(var_type, DateTimeVariable):
- pass
- sqlite_type = ""
- elif isinstance(var_type, DateVariable):
- pass
- elif isinstance(var_type, DecimalVariable):
- pass
- elif isinstance(var_type, EnumVariable):
- sqlite_type = "BLOB"
- elif isinstance(var_type, FloatVariable):
- sqlite_type = "REAL"
- elif isinstance(var_type, IntVariable):
- sqlite_type = "INTEGER"
- elif isinstance(var_type, RawStrVariable):
- sqlite_type = "BLOB"
- elif isinstance(var_type, UnicodeVariable):
- pass
- elif isinstance(var_type, JSONVariable):
- sqlite_type = "BLOB"
- elif isinstance(var_type, PickleVariable):
- sqlite_type = "BLOB"
- return "%s" % sqlite_type
-
-def varsToParametersSQLite(variables, primary_keys):
- """
- Takes as input a list of variables (convered to SQLite syntax and in the
- form of strings) and primary_keys.
- Outputs these variables converted into paramter syntax for SQLites.
-
- ex.
- variables: ["var1 INTEGER", "var2 BOOL", "var3 INTEGER"]
- primary_keys: ["var1"]
-
- output: "(var1 INTEGER, var2 BOOL, var3 INTEGER PRIMARY KEY (var1))"
- """
- params = "("
- for var in variables[:-1]:
- params += "%s %s, " % var
- if len(primary_keys) > 0:
- params += "%s %s, " % variables[-1]
- params += "PRIMARY KEY ("
- for key in primary_keys[:-1]:
- params += "%s, " % key
- params += "%s))" % primary_keys[-1]
- else:
- params += "%s %s)" % variables[-1]
- return params
-
-def generateCreateQuery(model):
- """
- This takes as input a Storm model and outputs the creation query for it.
- """
- query = "CREATE TABLE "+ model.__storm_table__ + " "
-
- variables = []
- primary_keys = []
-
- for attr in dir(model):
- a = getattr(model, attr)
- if isinstance(a, PropertyColumn):
- var_stype = a.variable_factory()
- var_type = variableToSQLite(var_stype)
- name = a.name
- variables.append((name, var_type))
- if a.primary:
- primary_keys.append(name)
-
- query += varsToParametersSQLite(variables, primary_keys)
- return query
-
-def createTable(model, transactor, database):
- """
- Create the table for the specified model.
- Specification of a transactor and database is useful in unittesting.
- """
- if not transactor:
- from oonib.db import transactor
- if not database:
- from oonib.db import database
- store = Store(database)
- create_query = generateCreateQuery(model)
- try:
- store.execute(create_query)
- # XXX trap the specific error that is raised when the table exists
- except StormError, e:
- print "Failed to create table!"
- print e
- store.close()
- store.commit()
- store.close()
-
- at inlineCallbacks
-def runCreateTable(model, transactor=None, database=None):
- """
- Runs the table creation query wrapped in a transaction.
- Transactions run in a separate thread.
- """
- yield transactor.run(createTable, model, transactor, database)
-
diff --git a/oonib/models.py b/oonib/models.py
deleted file mode 100644
index 60c0472..0000000
--- a/oonib/models.py
+++ /dev/null
@@ -1,129 +0,0 @@
-__all__ = ['Report', 'TestHelperTMP']
-from storm.twisted.transact import transact
-from storm.locals import *
-
-from oonib import randomStr
-from oonib import transactor
-
-def generateReportID():
- """
- Generates a report ID for usage in the database backed oonib collector.
-
- XXX note how this function is different from the one in report/api.py
- """
- report_id = randomStr(100)
- return report_id
-
-class OModel(object):
-
- transactor = transactor
-
- def getStore(self):
- return Store(database)
-
- @transact
- def create(self, query):
- store = Store(database)
- store.execute(query)
- store.commit()
-
- @transact
- def save(self):
- store = getStore()
- store.add(self)
- store.commit()
-
-class Report(OModel):
- """
- This represents an OONI Report as stored in the database.
-
- report_id: this is generated by the backend and is used by the client to
- reference a previous report and append to it. It should be
- treated as a shared secret between the probe and backend.
-
- software_name: this indicates the name of the software performing the test
- (this will default to ooniprobe)
-
- software_version: this is the version number of the software running the
- test.
-
- test_name: the name of the test on which the report is being created.
-
- test_version: indicates the version of the test
-
- progress: what is the current progress of the report. This allows clients
- to report event partial reports up to a certain percentage of
- progress. Once the report is complete progress will be 100.
-
- content: what is the content of the report. If the current progress is less
- than 100 we should append to the YAML data structure that is
- currently stored in such field.
-
- XXX this is currently not used.
- """
- __storm_table__ = 'reports'
-
- createQuery = "CREATE TABLE " + __storm_table__ +\
- "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\
- "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\
- "progress VARCHAR, content VARCHAR)"
-
-
- id = Int(primary=True)
-
- report_id = Unicode()
-
- software_name = Unicode()
- software_version = Unicode()
- test_name = Unicode()
- test_version = Unicode()
- progress = Int()
-
- content = Unicode()
-
- @transact
- def new(report):
- store = self.getStore()
-
- print "Creating new report %s" % report
-
- report_id = generateReportID()
- new_report = models.Report()
- new_report.report_id = unicode(report_id)
-
- new_report.software_name = report['software_name']
- new_report.software_version = report['software_version']
- new_report.test_name = report['test_name']
- new_report.test_version = report['test_version']
- new_report.progress = report['progress']
-
- if 'content' in report:
- new_report.content = report['content']
-
- print "Report: %s" % report
-
- store.add(new_report)
- try:
- store.commit()
- except:
- store.close()
-
- defer.returnValue({'backend_version': backend_version, 'report_id':
- report_id})
-
-
-class TestHelperTMP(OModel):
- __storm_table__ = 'testhelpertmp'
-
- createQuery = "CREATE TABLE " + __storm_table__ +\
- "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\
- " client_ip VARCHAR, creation_time VARCHAR)"
-
- id = Int(primary=True)
-
- report_id = Unicode()
-
- test_helper = Unicode()
- client_ip = Unicode()
-
- creation_time = Date()
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 0eef588..afd961f 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -20,7 +20,6 @@ from oonib.testhelpers import http_helpers, tcp_helpers
from oonib import log
-from oonib import db_threadpool
from oonib import config
if config.main.uid and config.main.gid:
@@ -73,5 +72,3 @@ if config.helpers.http_return_request.port:
int(config.helpers.http_return_request.port),
http_helpers.HTTPReturnJSONHeadersHelper())
http_return_request_helper.setServiceParent(serviceCollection)
-
-reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop)
diff --git a/oonib/report/api.py b/oonib/report/api.py
index d01f47c..7747a1e 100644
--- a/oonib/report/api.py
+++ b/oonib/report/api.py
@@ -20,7 +20,7 @@ from cyclone import web
from oonib import otime
from oonib import randomStr
-from oonib import models, config
+from oonib import config
from oonib.report import file_collector
def parseUpdateReportRequest(request):
@@ -45,59 +45,6 @@ def parseUpdateReportRequest(request):
return parsed_request
-
-class NewReportHandlerDB(web.RequestHandler):
- """
- Responsible for creating and updating reports via database.
- XXX this is not yet fully implemented.
- """
-
- @web.asynchronous
- @defer.inlineCallbacks
- def post(self):
- """
- Creates a new report with the input to the database.
- XXX this is not yet implemented.
-
- * Request
-
- {'software_name': 'XXX',
- 'software_version': 'XXX',
- 'test_name': 'XXX',
- 'test_version': 'XXX',
- 'progress': 'XXX',
- 'content': 'XXX'
- }
-
- Optional:
- 'test_helper': 'XXX'
- 'client_ip': 'XXX'
-
- * Response
-
- {'backend_version': 'XXX', 'report_id': 'XXX'}
-
- """
- report_data = json.loads(self.request.body)
- new_report = models.Report()
- log.debug("Got this request %s" % report_data)
- result = yield new_report.new(report_data)
- self.write(result)
- self.finish()
-
- def put(self):
- """
- Update an already existing report with the database.
-
- XXX this is not yet implemented.
-
- {'report_id': 'XXX',
- 'content': 'XXX'
- }
- """
- pass
-
-
reportingBackendAPI = [
(r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile),
(r"/report", file_collector.NewReportHandlerFile),
diff --git a/requirements.txt b/requirements.txt
index b0d7f75..08fd114 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -32,8 +32,6 @@ pygeoip>=0.2.6
##
https://people.torproject.org/~ioerror/src/mirrors/ooniprobe/scapy-02-25-2013-tip.zip#egg=scapy
transaction>=1.4.1
-#https://pypi.python.org/packages/source/t/transaction/transaction-1.4.1.zip#md5=8db2680bc0f999219861a67b8f335a88#egg=transaction
-storm>=0.19
#https://pypi.python.org/packages/source/s/storm/storm-0.19.tar.gz#md5=61d1ee4cd2a08639ab917e43fa2c9265#egg=storm
txtorcon>=0.7
pyOpenSSL>=0.13
More information about the tor-commits
mailing list