[tor-commits] [gettor/master] Switch from twisted adbapi to sqlite3 package

[cohosh at torproject.org]

commit e4f6d98fb9d0c82d154c12cbb5edaa2daf2572dd
Author: Cecylia Bocovich <cohosh at torproject.org>
Date:   Fri Feb 7 19:09:47 2020 -0500

    Switch from twisted adbapi to sqlite3 package
    
    The rest of GetTor uses the python sqlite3 package, and it's much
    simpler to use and to test.
---
 README.md          |  2 ++
 gettor/utils/db.py | 86 ++++++++++++++++++++++++++----------------------------
 tests/conftests.py |  1 +
 3 files changed, 44 insertions(+), 45 deletions(-)

diff --git a/README.md b/README.md
index f31d113..6ad9603 100644
--- a/README.md
+++ b/README.md
@@ -82,5 +82,7 @@ GetTor includes PyTest unit tests. To run the tests, first install the dependenc
 
 
 ```
+$ python3 scripts/create_db -n -c -o -f tests/gettor.db
+$ python3 scripts/add_links_to_db -f tests/gettor.db
 $ pytest-3 tests/
 ```
diff --git a/gettor/utils/db.py b/gettor/utils/db.py
index 0ca11aa..7c3853f 100644
--- a/gettor/utils/db.py
+++ b/gettor/utils/db.py
@@ -9,10 +9,10 @@
 
 from __future__ import absolute_import
 
+import sqlite3
 from datetime import datetime
 
 from twisted.python import log
-from twisted.enterprise import adbapi
 
 class SQLite3(object):
 	"""
@@ -20,94 +20,90 @@ class SQLite3(object):
 	"""
 	def __init__(self, dbname):
 		"""Constructor."""
-		self.dbpool = adbapi.ConnectionPool(
-			"sqlite3", dbname, check_same_thread=False
-		)
-
-	def query_callback(self, results=None):
-		"""
-		Query callback
-		Log that the database query has been executed and return results
-		"""
-		log.msg("Database query executed successfully.")
-		return results
-
-	def query_errback(self, error=None):
-		"""
-        Query error callback
-		Logs database error
-		"""
-		if error:
-			log.msg("Database error: {}".format(error))
-		return None
+		self.conn = sqlite3.connect(dbname)
 
 	def new_request(self, id, command, service, platform, language, date, status):
 		"""
 		Perform a new request to the database
 		"""
+		c = self.conn.cursor()
 		query = "INSERT INTO requests VALUES(?, ?, ?, ?, ?, ?, ?)"
 
-		return self.dbpool.runQuery(
-			query, (id, command, platform, language, service, date, status)
-		).addCallback(self.query_callback).addErrback(self.query_errback)
+		c.execute(query, (id, command, platform, language, service,
+                    date, status))
+		self.conn.commit()
+		return
 
 	def get_requests(self, status, command, service):
 		"""
 		Perform a SELECT request to the database
 		"""
+		c = self.conn.cursor()
 		query = "SELECT * FROM requests WHERE service=? AND command=? AND "\
 		"status = ?"
 
-		return self.dbpool.runQuery(
-			query, (service, command, status)
-		).addCallback(self.query_callback).addErrback(self.query_errback)
+		c.execute(query, (service, command, status))
+
+		return c.fetchall()
 
 	def get_num_requests(self, id, service):
 		"""
 		Get number of requests for statistics
 		"""
-		query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND service=?"
+		c = self.conn.cursor()
+		query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND "\
+		"service=?"
 
-		return self.dbpool.runQuery(
-			query, (id, service)
-		).addCallback(self.query_callback).addErrback(self.query_errback)
+		c.execute(query, (id, service))
+		return c.fetchone()[0]
 
 	def remove_request(self, id, service, date):
 		"""
 		Removes completed request record from the database
 		"""
-		query = "DELETE FROM requests WHERE id=? AND service=? AND date=?"
+		c = self.conn.cursor()
+		query = "DELETE FROM requests WHERE id=? AND service=? AND "\
+                "date=?"
 
-		return self.dbpool.runQuery(
-			query, (id, service, date)
-		).addCallback(self.query_callback).addErrback(self.query_errback)
+		c.execute(query, (id, service, date))
+		self.conn.commit()
+		return
 
 	def update_stats(self, command, service, platform=None, language='en'):
 		"""
 		Update statistics to the database
 		"""
+		c = self.conn.cursor()
 		now_str = datetime.now().strftime("%Y%m%d")
 		query = "INSERT INTO stats(num_requests, platform, language, command, "\
-		        "service, date) VALUES (1, ?, ?, ?, ?, ?) ON CONFLICT(platform, "\
-				"language, command, service, date) DO UPDATE SET num_requests=num_requests+1"
+		        "service, date) VALUES (1, ?, ?, ?, ?, ?) ON "\
+                        "CONFLICT(platform, language, command, service, date) "\
+                        "DO UPDATE SET num_requests=num_requests+1"
 
-		return self.dbpool.runQuery(
-			query, (platform, language, command, service, now_str)
-		).addCallback(self.query_callback).addErrback(self.query_errback)
+		c.execute(query, (platform, language, command, service,
+		    now_str))
+		self.conn.commit()
+		return
 
 	def get_links(self, platform, language, status):
 		"""
 		Get links from the database per platform
 		"""
+		c = self.conn.cursor()
 		query = "SELECT * FROM links WHERE platform=? AND language=? AND status=?"
-		return self.dbpool.runQuery(
-			query, (platform, language, status)
-		).addCallback(self.query_callback).addErrback(self.query_errback)
+		c.execute(query, (platform, language, status))
+
+		return c.fetchall()
 
 	def get_locales(self):
 		"""
 		Get a list of the supported tor browser binary locales
 		"""
+		c = self.conn.cursor()
 		query = "SELECT DISTINCT language FROM links"
-		return self.dbpool.runQuery(query
-		).addCallback(self.query_callback).addErrback(self.query_errback)
+		c.execute(query)
+
+		locales = []
+		for locale in c.fetchall():
+		    locales.append(locale[0])
+		return locales
diff --git a/tests/conftests.py b/tests/conftests.py
index cbb4d28..d509776 100644
--- a/tests/conftests.py
+++ b/tests/conftests.py
@@ -5,6 +5,7 @@ from __future__ import unicode_literals
 from gettor.utils import options
 from gettor.utils import strings
 from gettor.utils import twitter
+from gettor.utils.db import SQLite3
 from gettor.services.email.sendmail import Sendmail
 from gettor.services.twitter import twitterdm
 from gettor.parse.email import EmailParser, AddressError, DKIMError