[tor-commits] [gettor/master] Deleted database interaction from channels and core. Added new logging strings for stats purposes (See #17588).
ilv at torproject.org
ilv at torproject.org
Fri Dec 4 18:20:38 UTC 2015
commit 553519a8d4fd0f1514cca164def386a96e393328
Author: ilv <ilv at users.noreply.github.com>
Date: Mon Nov 16 15:19:44 2015 -0300
Deleted database interaction from channels and core. Added new logging strings for stats purposes (See #17588).
---
gettor/core.py | 11 -----------
gettor/smtp.py | 40 +++++++++-------------------------------
gettor/twitter.py | 51 ++++++++++++++++++++-------------------------------
gettor/utils.py | 15 +++++++++------
gettor/xmpp.py | 28 ++++++++--------------------
log/smtp.log | 3 +++
6 files changed, 49 insertions(+), 99 deletions(-)
diff --git a/gettor/core.py b/gettor/core.py
index 6336e04..3d946c9 100644
--- a/gettor/core.py
+++ b/gettor/core.py
@@ -469,14 +469,3 @@ class Core(object):
else:
self.log.debug("FAILED (links file doesn't seem legit)")
raise LinkFileError("No links file for %s" % provider)
-
- def add_request_to_db(self):
- """Add request to database."""
- self.log.debug("Trying to add request to database")
- try:
- self.db.connect()
- self.db.add_request()
- self.log.debug("Request added!")
- except db.DBError as e:
- self.log.debug("FAILED %s" % str(e))
- raise InternalError("Couldn't add request to database %s" % str(e))
diff --git a/gettor/smtp.py b/gettor/smtp.py
index 3697786..9203039 100644
--- a/gettor/smtp.py
+++ b/gettor/smtp.py
@@ -461,24 +461,15 @@ class SMTP(object):
self.log.debug("Normalizing address...")
norm_from_addr = self._get_normalized_address(from_addr)
except AddressError as e:
- status = 'malformed'
bogus_request = True
- self.log.debug("Address is malformed!")
- # it might be interesting to know what triggered this
- # we are not logging this for now
- # logfile = self._log_email('malformed', content)
+ self.log.info('REQUEST: invalid; OS: none; LC: none')
if norm_from_addr:
- self.log.debug("Anonymizing address...")
anon_addr = utils.get_sha256(norm_from_addr)
if self._is_blacklisted(anon_addr):
- status = 'blacklisted'
bogus_request = True
- self.log.debug("Address is blacklisted!")
- # it might be interesting to know extra info
- # we are not logging this for now
- # logfile = self._log_email(anon_addr, content)
+ self.log.info('REQUEST: blacklist; OS: none; LC: none')
if not bogus_request:
# try to figure out what the user is asking
@@ -487,45 +478,43 @@ class SMTP(object):
# our address should have the locale requested
our_addr = "gettor+%s@%s" % (req['lc'], self.our_domain)
- self.log.debug("Replying from %s" % our_addr)
# possible options: help, links, mirrors
if req['type'] == 'help':
- self.log.debug("Type of request: help")
self.log.debug("Trying to send help...")
+ self.log.info('REQUEST: help; OS: none; LC: %s' %
+ req['lc'])
# make sure we can send emails
try:
self._send_help('en', our_addr, norm_from_addr)
- status = 'success'
except SendEmailError as e:
- status = 'internal_error'
self.log.debug("FAILED: %s" % str(e))
raise InternalError("Something's wrong with the SMTP "
"server: %s" % str(e))
elif req['type'] == 'mirrors':
- self.log.debug("Type of request: mirrors")
self.log.debug("Trying to send the mirrors...")
+ self.log.info('REQUEST: mirrors; OS: none; LC: %s' %
+ req['lc'])
# make sure we can send emails
try:
self._send_mirrors('en', our_addr, norm_from_addr)
- status = 'success'
except SendEmailError as e:
- status = 'internal_error'
self.log.debug("FAILED: %s" % str(e))
raise SendEmailError("Something's wrong with the SMTP "
"server: %s" % str(e))
elif req['type'] == 'links':
- self.log.debug("Type of request: links")
self.log.debug("Trying to obtain the links...")
+ self.log.info('REQUEST: links; OS: %s; LC: %s' %
+ req['os '], req['lc'])
+
try:
links = self.core.get_links(
'SMTP', req['os'], req['lc']
)
# if core fails, we fail too
except (core.InternalError, core.ConfigError) as e:
- status = 'core_error'
self.log.debug("FAILED: %s" % str(e))
# something went wrong with the core
raise InternalError("Error obtaining the links")
@@ -535,19 +524,8 @@ class SMTP(object):
try:
self._send_links(links, req['lc'], req['os'], our_addr,
norm_from_addr)
- status = 'success'
except SendEmailError as e:
- status = 'internal_error'
self.log.debug("FAILED: %s" % str(e))
raise SendEmailError("Something's wrong with the SMTP "
"server: %s" % str(e))
self.log.debug("Mail sent!")
- finally:
- # keep stats
- if req:
- self.log.debug("Adding request to database... ")
- try:
- self.core.add_request_to_db()
- self.log.debug("Request added")
- except InternalError as e:
- self.log.debug("FAILED: %s" % str(e))
diff --git a/gettor/twitter.py b/gettor/twitter.py
index 13666a3..85cad47 100644
--- a/gettor/twitter.py
+++ b/gettor/twitter.py
@@ -24,6 +24,7 @@ import blacklist
"""Twitter channel for distributing links to download Tor Browser."""
+
class ConfigError(Exception):
pass
@@ -51,7 +52,7 @@ class TwitterBot(object):
:param: cfg (string) the path of the configuration file.
"""
-
+
default_cfg = 'twitter.cfg'
config = ConfigParser.ConfigParser()
@@ -107,10 +108,8 @@ class TwitterBot(object):
logfileh.setLevel(logging.getLevelName(loglevel))
log.addHandler(logfileh)
- # stop logging on stdout from now on
- #log.propagate = False
self.log = log
-
+
def _is_blacklisted(self, username):
"""Check if a user is blacklisted.
@@ -141,7 +140,6 @@ class TwitterBot(object):
Return: a string containing the given message.
"""
- #self.log.debug("Getting message '%s' for locale %s" % (msgid, lc))
try:
t = gettext.translation(lc, self.i18ndir, languages=[lc])
_ = t.ugettext
@@ -161,7 +159,7 @@ class TwitterBot(object):
:return: request (list) 3-tuple with locale, os and type of request.
"""
-
+
# core knows what OS are supported
supported_os = self.core.get_supported_os()
supported_lc = self.core.get_supported_lc()
@@ -198,7 +196,7 @@ class TwitterBot(object):
req['type'] = 'mirrors'
if (found_lc and found_os) or (found_lc and found_mirrors):
break
-
+
return req
def parse_request(self, dm):
@@ -220,31 +218,30 @@ class TwitterBot(object):
try:
if self._is_blacklisted(str(sender_id)):
- self.log.info("Request from blacklisted account!")
- status = 'blacklisted'
+ self.log.info('REQUEST: blacklist; OS: none; LC: none')
bogus_request = True
if not bogus_request:
self.log.debug("Request seems legit, let's parse it")
# let's try to guess what the user is asking
request = self.parse_text(str(msg))
-
+
# possible options: links, mirrors, help
if request['type'] == 'links':
- self.log.debug("Type of request: links")
+ self.log.info('REQUEST: links; OS: %s; LC: %s' %
+ (req['os'], req['lc']))
links = self.core.get_links(
'Twitter',
request['os'],
request['lc']
)
-
+
reply = self._get_msg('links', 'en')
reply = reply % (request['os'], request['lc'], links)
-
elif request['type'] == 'mirrors':
- self.log.debug("Type of request: mirrors")
- status = 'success'
+ self.log.info('REQUEST: mirrors; OS: none; LC: %s' %
+ req['lc'])
reply = self._get_msg('mirrors', 'en')
try:
with open(self.mirrors, "r") as list_mirrors:
@@ -255,10 +252,10 @@ class TwitterBot(object):
reply = self._get_msg('mirrors_unavailable', 'en')
else:
- self.log.debug("Type of request: help")
- status = 'success'
+ self.log.info('REQUEST: help; OS: none; LC: %s' %
+ req['lc'])
reply = self._get_msg('help', 'en')
-
+
self.api.send_direct_message(
user_id=sender_id,
text=reply
@@ -267,25 +264,18 @@ class TwitterBot(object):
except (core.ConfigError, core.InternalError) as e:
# if core failes, send the user an error message, but keep going
self.log.error("Something went wrong internally: %s" % str(e))
- status = 'core_error'
reply = self._get_msg('internal_error', 'en')
- finally:
- # keep stats
- if request:
- self.log.debug("Adding request to database... ")
- self.core.add_request_to_db()
-
def start(self):
""" Start the bot for handling requests.
Start a new Twitter bot.
"""
self.auth = tweepy.OAuthHandler(
- self.api_key,
+ self.api_key,
self.api_secret
)
-
+
self.auth.set_access_token(
self.access_token,
self.token_secret
@@ -293,11 +283,10 @@ class TwitterBot(object):
self.api = tweepy.API(self.auth)
self.bot_info = self.api.me()
-
+
stream = tweepy.Stream(
- auth = self.api.auth,
+ auth=self.api.auth,
listener=GetTorStreamListener(self)
)
-
- stream.userstream()
+ stream.userstream()
diff --git a/gettor/utils.py b/gettor/utils.py
index b4ab9c6..8224f25 100644
--- a/gettor/utils.py
+++ b/gettor/utils.py
@@ -16,25 +16,28 @@ import hashlib
"""Common utilities for GetTor modules."""
-LOGGING_FORMAT = "[%(levelname)s] %(asctime)s - %(message)s"
-DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S
+LOGGING_FORMAT = "[%(levelname)s] DATE: %(asctime)s; %(message)s"
+DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S
+
def get_logging_format():
"""Get the logging format.
-
+
:return: (string) the logging format.
-
+
"""
return LOGGING_FORMAT
+
def get_date_format():
"""Get the date format for logging.
-
+
:return: (string) the date format for logging.
-
+
"""
return DATE_FORMAT
+
def get_sha256(string):
"""Get sha256 of a string.
diff --git a/gettor/xmpp.py b/gettor/xmpp.py
index a605f50..072d988 100644
--- a/gettor/xmpp.py
+++ b/gettor/xmpp.py
@@ -277,8 +277,7 @@ class XMPP(object):
self.log.debug("Parsing request")
try:
if self._is_blacklisted(str(account)):
- self.log.info("Request from blacklisted account!")
- status = 'blacklisted'
+ self.log.info('REQUEST: blacklist; OS: none; LC: none')
bogus_request = True
# first let's find out how many words are in the message
@@ -287,7 +286,7 @@ class XMPP(object):
if len(words) > self.max_words:
bogus_request = True
self.log.info("Message way too long")
- status = 'error'
+ self.log.info('REQUEST: invalid; OS: none; LC: none')
reply = self._get_msg('message_error', 'en')
if not bogus_request:
@@ -296,13 +295,13 @@ class XMPP(object):
req = self._parse_text(str(msg))
if req['type'] == 'help':
- self.log.debug("Type of request: help")
- status = 'success'
+ self.log.info('REQUEST: help; OS: none; LC: %s' %
+ req['lc'])
reply = self._get_msg('help', 'en')
elif req['type'] == 'mirrors':
- self.log.debug("Type of request: mirrors")
- status = 'success'
+ self.log.info('REQUEST: mirrors; OS: none; LC: %s' %
+ req['lc'])
reply = self._get_msg('mirrors', 'en')
try:
with open(self.mirrors, "r") as list_mirrors:
@@ -312,7 +311,8 @@ class XMPP(object):
reply = self._get_msg('mirrors_unavailable', 'en')
elif req['type'] == 'links':
- self.log.debug("Type of request: help")
+ self.log.info('REQUEST: links; OS: %s; LC: %s' %
+ (req['os'], req['lc']))
links = self.core.get_links(
"XMPP",
req['os'],
@@ -321,22 +321,10 @@ class XMPP(object):
reply = self._get_msg('links', 'en')
reply = reply % (req['os'], req['lc'], links)
- status = 'success'
-
except (core.ConfigError, core.InternalError) as e:
# if core failes, send the user an error message, but keep going
self.log.error("Something went wrong internally: %s" % str(e))
- status = 'core_error'
reply = self._get_msg('internal_error', req['lc'])
finally:
- # keep stats
- if req:
- self.log.debug("Adding request to database... ")
- self.core.add_request_to_db()
-
- if reply:
- self.log.debug("Everything seems OK. Sending back the reply")
- else:
- self.log.debug("Nothing to reply!")
return reply
diff --git a/log/smtp.log b/log/smtp.log
index e69de29..3950925 100644
--- a/log/smtp.log
+++ b/log/smtp.log
@@ -0,0 +1,3 @@
+[INFO] DATE: 2015-11-16; REQUEST: help; OS: none; LC: fa
+[INFO] DATE: 2015-11-16; REQUEST: links; OS: windows; LC: zh
+[INFO] DATE: 2015-11-16; REQUEST: mirrors; OS: none; LC: tr
More information about the tor-commits
mailing list