[tor-commits] [gettor/master] Add method to close dbpool and call it
cohosh at torproject.org
cohosh at torproject.org
Fri Feb 21 18:37:12 UTC 2020
commit c413a8ebdaf1c70bc40255e35e7917165fa75552
Author: Cecylia Bocovich <cohosh at torproject.org>
Date: Wed Feb 19 12:15:58 2020 -0500
Add method to close dbpool and call it
We were leaving connections to the database open, which causes some
calls to hang. This adds destructors to the SQLite3 class and the
classes that use it, and refactors some code to make one database
connection per class object. This also makes sure that the destructors
for objects that use databases are actually called.
---
gettor/parse/email.py | 13 +++++++------
gettor/parse/twitter.py | 11 ++++++-----
gettor/services/email/sendmail.py | 2 ++
gettor/services/twitter/twitterdm.py | 3 +++
gettor/utils/db.py | 3 +++
scripts/process_email | 1 +
tests/test_db.py | 4 +---
tests/test_email_service.py | 16 ++++++++++++++++
8 files changed, 39 insertions(+), 14 deletions(-)
diff --git a/gettor/parse/email.py b/gettor/parse/email.py
index 9084cb7..a08f8ca 100644
--- a/gettor/parse/email.py
+++ b/gettor/parse/email.py
@@ -56,6 +56,10 @@ class EmailParser(object):
self.to_addr = to_addr
self.locales = []
self.platforms = self.settings.get("platforms")
+ self.conn = SQLite3(self.settings.get("dbname"))
+
+ def __del__(self):
+ del self.conn
def normalize(self, msg):
# Normalization will convert <Alice Wonderland> alice at wonderland.net
@@ -164,10 +168,8 @@ class EmailParser(object):
@defer.inlineCallbacks
def get_locales(self):
- dbname = self.settings.get("dbname")
- conn = SQLite3(dbname)
- locales = yield conn.get_locales()
+ locales = yield self.conn.get_locales()
for l in locales:
self.locales.append(l[0])
@@ -235,7 +237,6 @@ class EmailParser(object):
now_str = datetime.now().strftime("%Y%m%d%H%M%S")
dbname = self.settings.get("dbname")
test_hid = self.settings.get("test_hid")
- conn = SQLite3(dbname)
if request["command"]:
@@ -247,7 +248,7 @@ class EmailParser(object):
system="email parser"
)
- num_requests = yield conn.get_num_requests(
+ num_requests = yield self.conn.get_num_requests(
id=hid, service=request_service
)
@@ -262,7 +263,7 @@ class EmailParser(object):
), system="email parser"
)
else:
- conn.new_request(
+ self.conn.new_request(
id=request['id'],
command=request['command'],
platform=request['platform'],
diff --git a/gettor/parse/twitter.py b/gettor/parse/twitter.py
index 645bf47..fd197ac 100644
--- a/gettor/parse/twitter.py
+++ b/gettor/parse/twitter.py
@@ -35,7 +35,10 @@ class TwitterParser(object):
"""
self.settings = settings
self.twitter_id = twitter_id
+ self.conn = SQLite3(self.settings.get("dbname"))
+ def __del__(self):
+ del self.conn
def build_request(self, msg_text, twitter_id, languages, platforms):
@@ -110,16 +113,14 @@ class TwitterParser(object):
if request["command"]:
now_str = datetime.now().strftime("%Y%m%d%H%M%S")
- dbname = self.settings.get("dbname")
- conn = SQLite3(dbname)
hid = hashlib.sha256(str(request['id']).encode('utf-8'))
# check limits first
- num_requests = yield conn.get_num_requests(
+ num_requests = yield self.conn.get_num_requests(
id=hid.hexdigest(), service=request['service']
)
- num_requests += yield conn.get_num_requests(
+ num_requests += yield self.conn.get_num_requests(
id=str(request['id']), service=request['service']
)
@@ -130,7 +131,7 @@ class TwitterParser(object):
), system="twitter parser"
)
else:
- conn.new_request(
+ self.conn.new_request(
id=str(request['id']),
command=request['command'],
platform=request['platform'],
diff --git a/gettor/services/email/sendmail.py b/gettor/services/email/sendmail.py
index 9d97567..754fb5b 100644
--- a/gettor/services/email/sendmail.py
+++ b/gettor/services/email/sendmail.py
@@ -45,6 +45,8 @@ class Sendmail(object):
dbname = self.settings.get("dbname")
self.conn = DB(dbname)
+ def __del__(self):
+ del self.conn
def get_interval(self):
"""
diff --git a/gettor/services/twitter/twitterdm.py b/gettor/services/twitter/twitterdm.py
index 5f143ca..b0a1a48 100644
--- a/gettor/services/twitter/twitterdm.py
+++ b/gettor/services/twitter/twitterdm.py
@@ -41,6 +41,8 @@ class Twitterdm(object):
self.twitter = Twitter(settings)
self.conn = DB(dbname)
+ def __del__(self):
+ del self.conn
def get_interval(self):
"""
@@ -115,6 +117,7 @@ class Twitterdm(object):
yield defer.maybeDeferred(
tp.parse, e['message_create']['message_data']['text'], message_id
).addCallback(tp.parse_callback).addErrback(tp.parse_errback)
+ del tp
# Manage help and links messages separately
help_requests = yield self.conn.get_requests(
diff --git a/gettor/utils/db.py b/gettor/utils/db.py
index 0ca11aa..064049e 100644
--- a/gettor/utils/db.py
+++ b/gettor/utils/db.py
@@ -24,6 +24,9 @@ class SQLite3(object):
"sqlite3", dbname, check_same_thread=False
)
+ def __del__(self):
+ self.dbpool.close()
+
def query_callback(self, results=None):
"""
Query callback
diff --git a/scripts/process_email b/scripts/process_email
index cce7bcc..1dd7dfd 100755
--- a/scripts/process_email
+++ b/scripts/process_email
@@ -41,6 +41,7 @@ def process_email(message):
log.err("DKIM error: {}".format(e), system="process email")
reactor.stop()
+ del ep
reactor.stop()
def main():
diff --git a/tests/test_db.py b/tests/test_db.py
index e956850..e04ca39 100644
--- a/tests/test_db.py
+++ b/tests/test_db.py
@@ -2,8 +2,6 @@
import pytest
import pytest_twisted
from twisted.trial import unittest
-from twisted.internet import defer, reactor
-from twisted.internet import task
from . import conftests
@@ -19,7 +17,7 @@ class DatabaseTests(unittest.TestCase):
def tearDown(self):
print("tearDown()")
- return self.conn.dbpool.close()
+ del self.conn
@pytest_twisted.inlineCallbacks
def test_stored_locales(self):
diff --git a/tests/test_email_service.py b/tests/test_email_service.py
index 00795c1..d064aa1 100644
--- a/tests/test_email_service.py
+++ b/tests/test_email_service.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import pytest
+import pytest_twisted
import hashlib
from datetime import datetime
from twisted.trial import unittest
@@ -31,6 +32,7 @@ class EmailServiceTests(unittest.TestCase):
def tearDown(self):
print("tearDown()")
+ del self.sm_client
def test_get_interval(self):
self.assertEqual(self.settings.get("sendmail_interval"), self.sm_client.get_interval())
@@ -39,6 +41,7 @@ class EmailServiceTests(unittest.TestCase):
ep = conftests.EmailParser(self.settings, "gettor at torproject.org")
request = ep.parse("From: \"silvia [hiro]\" <hiro at torproject.org>\n Subject: help\n Reply-To: hiro at torproject.org \nTo: gettor at torproject.org")
self.assertEqual(request["command"], "help")
+ del ep
def test_normalize_msg(self):
ep = conftests.EmailParser(self.settings, "gettor at torproject.org")
@@ -46,6 +49,7 @@ class EmailServiceTests(unittest.TestCase):
msg = conftests.message_from_string(msg_str)
request = ep.normalize(msg)
self.assertEqual(request, ('silvia [hiro]', 'hiro at torproject.org', '', 'gettor at torproject.org'))
+ del ep
def test_validate_msg(self):
ep = conftests.EmailParser(self.settings, "gettor at torproject.org")
@@ -53,6 +57,7 @@ class EmailServiceTests(unittest.TestCase):
msg = conftests.message_from_string(msg_str)
request = ep.validate("hiro at torproject.org", msg)
assert request
+ del ep
def test_dkim_verify(self):
ep = conftests.EmailParser(self.settings, "gettor at torproject.org")
@@ -60,6 +65,7 @@ class EmailServiceTests(unittest.TestCase):
msg = conftests.message_from_string(msg_str)
request = ep.dkim_verify(msg, "hiro at torproject.org")
assert request
+ del ep
def test_build_request(self):
ep = conftests.EmailParser(self.settings, "gettor at torproject.org")
@@ -70,6 +76,7 @@ class EmailServiceTests(unittest.TestCase):
self.assertEqual(request["command"], "links")
self.assertEqual(request["platform"], "osx")
self.assertEqual(request["language"], "es")
+ del ep
def test_too_many_request_exclude(self):
ep = conftests.EmailParser(self.settings, "gettor at torproject.org")
@@ -79,6 +86,7 @@ class EmailServiceTests(unittest.TestCase):
check = ep.too_many_requests(hid, self.settings.get("test_hid"), num_requests, limit)
self.assertEqual(hid, self.settings.get("test_hid"))
self.assertEqual(check, False)
+ del ep
def test_language_email_parser(self):
ep = conftests.EmailParser(self.settings, "gettor at torproject.org")
@@ -137,6 +145,7 @@ class EmailServiceTests(unittest.TestCase):
"Subject: linux es-AR\r\n Reply-To: hiro at torproject.org \nTo:"
"gettor at torproject.org\n linux es")
self.assertEqual(request["language"], "es-AR")
+ del ep
def test_sent_links_message(self):
ep = self.sm_client
@@ -159,6 +168,13 @@ class EmailServiceTests(unittest.TestCase):
help_msg = ep.build_help_body_message()
assert "This is how you can request a tor browser bundle link" in help_msg
+ @pytest_twisted.inlineCallbacks
+ def test_get_locales(self):
+ ep = conftests.EmailParser(self.settings, "gettor at torproject.org")
+ yield ep.get_locales().addErrback(ep.parse_errback)
+ assert "en-US" in ep.locales
+ del ep
+
if __name__ == "__main__":
unittest.main()
More information about the tor-commits
mailing list