commit c413a8ebdaf1c70bc40255e35e7917165fa75552 Author: Cecylia Bocovich cohosh@torproject.org Date: Wed Feb 19 12:15:58 2020 -0500

Add method to close dbpool and call it

We were leaving connections to the database open, which causes some calls to hang. This adds destructors to the SQLite3 class and the classes that use it, and refactors some code to make one database connection per class object. This also makes sure that the destructors for objects that use databases are actually called. --- gettor/parse/email.py | 13 +++++++------ gettor/parse/twitter.py | 11 ++++++----- gettor/services/email/sendmail.py | 2 ++ gettor/services/twitter/twitterdm.py | 3 +++ gettor/utils/db.py | 3 +++ scripts/process_email | 1 + tests/test_db.py | 4 +--- tests/test_email_service.py | 16 ++++++++++++++++ 8 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/gettor/parse/email.py b/gettor/parse/email.py index 9084cb7..a08f8ca 100644 --- a/gettor/parse/email.py +++ b/gettor/parse/email.py @@ -56,6 +56,10 @@ class EmailParser(object): self.to_addr = to_addr self.locales = [] self.platforms = self.settings.get("platforms") + self.conn = SQLite3(self.settings.get("dbname")) + + def __del__(self): + del self.conn

def normalize(self, msg): # Normalization will convert <Alice Wonderland> alice@wonderland.net @@ -164,10 +168,8 @@ class EmailParser(object):

@defer.inlineCallbacks def get_locales(self): - dbname = self.settings.get("dbname") - conn = SQLite3(dbname)

- locales = yield conn.get_locales() + locales = yield self.conn.get_locales() for l in locales: self.locales.append(l[0])

@@ -235,7 +237,6 @@ class EmailParser(object): now_str = datetime.now().strftime("%Y%m%d%H%M%S") dbname = self.settings.get("dbname") test_hid = self.settings.get("test_hid") - conn = SQLite3(dbname)

if request["command"]:

@@ -247,7 +248,7 @@ class EmailParser(object): system="email parser" )

- num_requests = yield conn.get_num_requests( + num_requests = yield self.conn.get_num_requests( id=hid, service=request_service )

@@ -262,7 +263,7 @@ class EmailParser(object): ), system="email parser" ) else: - conn.new_request( + self.conn.new_request( id=request['id'], command=request['command'], platform=request['platform'], diff --git a/gettor/parse/twitter.py b/gettor/parse/twitter.py index 645bf47..fd197ac 100644 --- a/gettor/parse/twitter.py +++ b/gettor/parse/twitter.py @@ -35,7 +35,10 @@ class TwitterParser(object): """ self.settings = settings self.twitter_id = twitter_id + self.conn = SQLite3(self.settings.get("dbname"))

+ def __del__(self): + del self.conn

def build_request(self, msg_text, twitter_id, languages, platforms):

@@ -110,16 +113,14 @@ class TwitterParser(object):

if request["command"]: now_str = datetime.now().strftime("%Y%m%d%H%M%S") - dbname = self.settings.get("dbname") - conn = SQLite3(dbname)

hid = hashlib.sha256(str(request['id']).encode('utf-8')) # check limits first - num_requests = yield conn.get_num_requests( + num_requests = yield self.conn.get_num_requests( id=hid.hexdigest(), service=request['service'] )

- num_requests += yield conn.get_num_requests( + num_requests += yield self.conn.get_num_requests( id=str(request['id']), service=request['service'] )

@@ -130,7 +131,7 @@ class TwitterParser(object): ), system="twitter parser" ) else: - conn.new_request( + self.conn.new_request( id=str(request['id']), command=request['command'], platform=request['platform'], diff --git a/gettor/services/email/sendmail.py b/gettor/services/email/sendmail.py index 9d97567..754fb5b 100644 --- a/gettor/services/email/sendmail.py +++ b/gettor/services/email/sendmail.py @@ -45,6 +45,8 @@ class Sendmail(object): dbname = self.settings.get("dbname") self.conn = DB(dbname)

+ def __del__(self): + del self.conn

def get_interval(self): """ diff --git a/gettor/services/twitter/twitterdm.py b/gettor/services/twitter/twitterdm.py index 5f143ca..b0a1a48 100644 --- a/gettor/services/twitter/twitterdm.py +++ b/gettor/services/twitter/twitterdm.py @@ -41,6 +41,8 @@ class Twitterdm(object): self.twitter = Twitter(settings) self.conn = DB(dbname)

+ def __del__(self): + del self.conn

def get_interval(self): """ @@ -115,6 +117,7 @@ class Twitterdm(object): yield defer.maybeDeferred( tp.parse, e['message_create']['message_data']['text'], message_id ).addCallback(tp.parse_callback).addErrback(tp.parse_errback) + del tp

# Manage help and links messages separately help_requests = yield self.conn.get_requests( diff --git a/gettor/utils/db.py b/gettor/utils/db.py index 0ca11aa..064049e 100644 --- a/gettor/utils/db.py +++ b/gettor/utils/db.py @@ -24,6 +24,9 @@ class SQLite3(object): "sqlite3", dbname, check_same_thread=False )

+ def __del__(self): + self.dbpool.close() + def query_callback(self, results=None): """ Query callback diff --git a/scripts/process_email b/scripts/process_email index cce7bcc..1dd7dfd 100755 --- a/scripts/process_email +++ b/scripts/process_email @@ -41,6 +41,7 @@ def process_email(message): log.err("DKIM error: {}".format(e), system="process email") reactor.stop()

+ del ep reactor.stop()

def main(): diff --git a/tests/test_db.py b/tests/test_db.py index e956850..e04ca39 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -2,8 +2,6 @@ import pytest import pytest_twisted from twisted.trial import unittest -from twisted.internet import defer, reactor -from twisted.internet import task

from . import conftests

@@ -19,7 +17,7 @@ class DatabaseTests(unittest.TestCase):

def tearDown(self): print("tearDown()") - return self.conn.dbpool.close() + del self.conn

@pytest_twisted.inlineCallbacks def test_stored_locales(self): diff --git a/tests/test_email_service.py b/tests/test_email_service.py index 00795c1..d064aa1 100644 --- a/tests/test_email_service.py +++ b/tests/test_email_service.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import pytest +import pytest_twisted import hashlib from datetime import datetime from twisted.trial import unittest @@ -31,6 +32,7 @@ class EmailServiceTests(unittest.TestCase):

def tearDown(self): print("tearDown()") + del self.sm_client

def test_get_interval(self): self.assertEqual(self.settings.get("sendmail_interval"), self.sm_client.get_interval()) @@ -39,6 +41,7 @@ class EmailServiceTests(unittest.TestCase): ep = conftests.EmailParser(self.settings, "gettor@torproject.org") request = ep.parse("From: "silvia [hiro]" hiro@torproject.org\n Subject: help\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org") self.assertEqual(request["command"], "help") + del ep

def test_normalize_msg(self): ep = conftests.EmailParser(self.settings, "gettor@torproject.org") @@ -46,6 +49,7 @@ class EmailServiceTests(unittest.TestCase): msg = conftests.message_from_string(msg_str) request = ep.normalize(msg) self.assertEqual(request, ('silvia [hiro]', 'hiro@torproject.org', '', 'gettor@torproject.org')) + del ep

def test_validate_msg(self): ep = conftests.EmailParser(self.settings, "gettor@torproject.org") @@ -53,6 +57,7 @@ class EmailServiceTests(unittest.TestCase): msg = conftests.message_from_string(msg_str) request = ep.validate("hiro@torproject.org", msg) assert request + del ep

def test_dkim_verify(self): ep = conftests.EmailParser(self.settings, "gettor@torproject.org") @@ -60,6 +65,7 @@ class EmailServiceTests(unittest.TestCase): msg = conftests.message_from_string(msg_str) request = ep.dkim_verify(msg, "hiro@torproject.org") assert request + del ep

def test_build_request(self): ep = conftests.EmailParser(self.settings, "gettor@torproject.org") @@ -70,6 +76,7 @@ class EmailServiceTests(unittest.TestCase): self.assertEqual(request["command"], "links") self.assertEqual(request["platform"], "osx") self.assertEqual(request["language"], "es") + del ep

def test_too_many_request_exclude(self): ep = conftests.EmailParser(self.settings, "gettor@torproject.org") @@ -79,6 +86,7 @@ class EmailServiceTests(unittest.TestCase): check = ep.too_many_requests(hid, self.settings.get("test_hid"), num_requests, limit) self.assertEqual(hid, self.settings.get("test_hid")) self.assertEqual(check, False) + del ep

def test_language_email_parser(self): ep = conftests.EmailParser(self.settings, "gettor@torproject.org") @@ -137,6 +145,7 @@ class EmailServiceTests(unittest.TestCase): "Subject: linux es-AR\r\n Reply-To: hiro@torproject.org \nTo:" "gettor@torproject.org\n linux es") self.assertEqual(request["language"], "es-AR") + del ep

def test_sent_links_message(self): ep = self.sm_client @@ -159,6 +168,13 @@ class EmailServiceTests(unittest.TestCase): help_msg = ep.build_help_body_message() assert "This is how you can request a tor browser bundle link" in help_msg

+ @pytest_twisted.inlineCallbacks + def test_get_locales(self): + ep = conftests.EmailParser(self.settings, "gettor@torproject.org") + yield ep.get_locales().addErrback(ep.parse_errback) + assert "en-US" in ep.locales + del ep +

if __name__ == "__main__": unittest.main()