commit e4f6d98fb9d0c82d154c12cbb5edaa2daf2572dd Author: Cecylia Bocovich <cohosh@torproject.org> Date: Fri Feb 7 19:09:47 2020 -0500 Switch from twisted adbapi to sqlite3 package The rest of GetTor uses the python sqlite3 package, and it's much simpler to use and to test. --- README.md | 2 ++ gettor/utils/db.py | 86 ++++++++++++++++++++++++++---------------------------- tests/conftests.py | 1 + 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index f31d113..6ad9603 100644 --- a/README.md +++ b/README.md @@ -82,5 +82,7 @@ GetTor includes PyTest unit tests. To run the tests, first install the dependenc ``` +$ python3 scripts/create_db -n -c -o -f tests/gettor.db +$ python3 scripts/add_links_to_db -f tests/gettor.db $ pytest-3 tests/ ``` diff --git a/gettor/utils/db.py b/gettor/utils/db.py index 0ca11aa..7c3853f 100644 --- a/gettor/utils/db.py +++ b/gettor/utils/db.py @@ -9,10 +9,10 @@ from __future__ import absolute_import +import sqlite3 from datetime import datetime from twisted.python import log -from twisted.enterprise import adbapi class SQLite3(object): """ @@ -20,94 +20,90 @@ class SQLite3(object): """ def __init__(self, dbname): """Constructor.""" - self.dbpool = adbapi.ConnectionPool( - "sqlite3", dbname, check_same_thread=False - ) - - def query_callback(self, results=None): - """ - Query callback - Log that the database query has been executed and return results - """ - log.msg("Database query executed successfully.") - return results - - def query_errback(self, error=None): - """ - Query error callback - Logs database error - """ - if error: - log.msg("Database error: {}".format(error)) - return None + self.conn = sqlite3.connect(dbname) def new_request(self, id, command, service, platform, language, date, status): """ Perform a new request to the database """ + c = self.conn.cursor() query = "INSERT INTO requests VALUES(?, ?, ?, ?, ?, ?, ?)" - return self.dbpool.runQuery( - query, (id, command, platform, language, service, date, status) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (id, command, platform, language, service, + date, status)) + self.conn.commit() + return def get_requests(self, status, command, service): """ Perform a SELECT request to the database """ + c = self.conn.cursor() query = "SELECT * FROM requests WHERE service=? AND command=? AND "\ "status = ?" - return self.dbpool.runQuery( - query, (service, command, status) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (service, command, status)) + + return c.fetchall() def get_num_requests(self, id, service): """ Get number of requests for statistics """ - query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND service=?" + c = self.conn.cursor() + query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND "\ + "service=?" - return self.dbpool.runQuery( - query, (id, service) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (id, service)) + return c.fetchone()[0] def remove_request(self, id, service, date): """ Removes completed request record from the database """ - query = "DELETE FROM requests WHERE id=? AND service=? AND date=?" + c = self.conn.cursor() + query = "DELETE FROM requests WHERE id=? AND service=? AND "\ + "date=?" - return self.dbpool.runQuery( - query, (id, service, date) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (id, service, date)) + self.conn.commit() + return def update_stats(self, command, service, platform=None, language='en'): """ Update statistics to the database """ + c = self.conn.cursor() now_str = datetime.now().strftime("%Y%m%d") query = "INSERT INTO stats(num_requests, platform, language, command, "\ - "service, date) VALUES (1, ?, ?, ?, ?, ?) ON CONFLICT(platform, "\ - "language, command, service, date) DO UPDATE SET num_requests=num_requests+1" + "service, date) VALUES (1, ?, ?, ?, ?, ?) ON "\ + "CONFLICT(platform, language, command, service, date) "\ + "DO UPDATE SET num_requests=num_requests+1" - return self.dbpool.runQuery( - query, (platform, language, command, service, now_str) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (platform, language, command, service, + now_str)) + self.conn.commit() + return def get_links(self, platform, language, status): """ Get links from the database per platform """ + c = self.conn.cursor() query = "SELECT * FROM links WHERE platform=? AND language=? AND status=?" - return self.dbpool.runQuery( - query, (platform, language, status) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (platform, language, status)) + + return c.fetchall() def get_locales(self): """ Get a list of the supported tor browser binary locales """ + c = self.conn.cursor() query = "SELECT DISTINCT language FROM links" - return self.dbpool.runQuery(query - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query) + + locales = [] + for locale in c.fetchall(): + locales.append(locale[0]) + return locales diff --git a/tests/conftests.py b/tests/conftests.py index cbb4d28..d509776 100644 --- a/tests/conftests.py +++ b/tests/conftests.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals from gettor.utils import options from gettor.utils import strings from gettor.utils import twitter +from gettor.utils.db import SQLite3 from gettor.services.email.sendmail import Sendmail from gettor.services.twitter import twitterdm from gettor.parse.email import EmailParser, AddressError, DKIMError