commit 553519a8d4fd0f1514cca164def386a96e393328 Author: ilv ilv@users.noreply.github.com Date: Mon Nov 16 15:19:44 2015 -0300

Deleted database interaction from channels and core. Added new logging strings for stats purposes (See #17588). --- gettor/core.py | 11 ----------- gettor/smtp.py | 40 +++++++++------------------------------- gettor/twitter.py | 51 ++++++++++++++++++++------------------------------- gettor/utils.py | 15 +++++++++------ gettor/xmpp.py | 28 ++++++++-------------------- log/smtp.log | 3 +++ 6 files changed, 49 insertions(+), 99 deletions(-)

diff --git a/gettor/core.py b/gettor/core.py index 6336e04..3d946c9 100644 --- a/gettor/core.py +++ b/gettor/core.py @@ -469,14 +469,3 @@ class Core(object): else: self.log.debug("FAILED (links file doesn't seem legit)") raise LinkFileError("No links file for %s" % provider) - - def add_request_to_db(self): - """Add request to database.""" - self.log.debug("Trying to add request to database") - try: - self.db.connect() - self.db.add_request() - self.log.debug("Request added!") - except db.DBError as e: - self.log.debug("FAILED %s" % str(e)) - raise InternalError("Couldn't add request to database %s" % str(e)) diff --git a/gettor/smtp.py b/gettor/smtp.py index 3697786..9203039 100644 --- a/gettor/smtp.py +++ b/gettor/smtp.py @@ -461,24 +461,15 @@ class SMTP(object): self.log.debug("Normalizing address...") norm_from_addr = self._get_normalized_address(from_addr) except AddressError as e: - status = 'malformed' bogus_request = True - self.log.debug("Address is malformed!") - # it might be interesting to know what triggered this - # we are not logging this for now - # logfile = self._log_email('malformed', content) + self.log.info('REQUEST: invalid; OS: none; LC: none')

if norm_from_addr: - self.log.debug("Anonymizing address...") anon_addr = utils.get_sha256(norm_from_addr)

if self._is_blacklisted(anon_addr): - status = 'blacklisted' bogus_request = True - self.log.debug("Address is blacklisted!") - # it might be interesting to know extra info - # we are not logging this for now - # logfile = self._log_email(anon_addr, content) + self.log.info('REQUEST: blacklist; OS: none; LC: none')

if not bogus_request: # try to figure out what the user is asking @@ -487,45 +478,43 @@ class SMTP(object):

# our address should have the locale requested our_addr = "gettor+%s@%s" % (req['lc'], self.our_domain) - self.log.debug("Replying from %s" % our_addr)

# possible options: help, links, mirrors if req['type'] == 'help': - self.log.debug("Type of request: help") self.log.debug("Trying to send help...") + self.log.info('REQUEST: help; OS: none; LC: %s' % + req['lc']) # make sure we can send emails try: self._send_help('en', our_addr, norm_from_addr) - status = 'success' except SendEmailError as e: - status = 'internal_error' self.log.debug("FAILED: %s" % str(e)) raise InternalError("Something's wrong with the SMTP " "server: %s" % str(e))

elif req['type'] == 'mirrors': - self.log.debug("Type of request: mirrors") self.log.debug("Trying to send the mirrors...") + self.log.info('REQUEST: mirrors; OS: none; LC: %s' % + req['lc']) # make sure we can send emails try: self._send_mirrors('en', our_addr, norm_from_addr) - status = 'success' except SendEmailError as e: - status = 'internal_error' self.log.debug("FAILED: %s" % str(e)) raise SendEmailError("Something's wrong with the SMTP " "server: %s" % str(e))

elif req['type'] == 'links': - self.log.debug("Type of request: links") self.log.debug("Trying to obtain the links...") + self.log.info('REQUEST: links; OS: %s; LC: %s' % + req['os '], req['lc']) + try: links = self.core.get_links( 'SMTP', req['os'], req['lc'] ) # if core fails, we fail too except (core.InternalError, core.ConfigError) as e: - status = 'core_error' self.log.debug("FAILED: %s" % str(e)) # something went wrong with the core raise InternalError("Error obtaining the links") @@ -535,19 +524,8 @@ class SMTP(object): try: self._send_links(links, req['lc'], req['os'], our_addr, norm_from_addr) - status = 'success' except SendEmailError as e: - status = 'internal_error' self.log.debug("FAILED: %s" % str(e)) raise SendEmailError("Something's wrong with the SMTP " "server: %s" % str(e)) self.log.debug("Mail sent!") - finally: - # keep stats - if req: - self.log.debug("Adding request to database... ") - try: - self.core.add_request_to_db() - self.log.debug("Request added") - except InternalError as e: - self.log.debug("FAILED: %s" % str(e)) diff --git a/gettor/twitter.py b/gettor/twitter.py index 13666a3..85cad47 100644 --- a/gettor/twitter.py +++ b/gettor/twitter.py @@ -24,6 +24,7 @@ import blacklist

"""Twitter channel for distributing links to download Tor Browser."""

+ class ConfigError(Exception): pass

@@ -51,7 +52,7 @@ class TwitterBot(object):

:param: cfg (string) the path of the configuration file. """ - + default_cfg = 'twitter.cfg' config = ConfigParser.ConfigParser()

@@ -107,10 +108,8 @@ class TwitterBot(object): logfileh.setLevel(logging.getLevelName(loglevel)) log.addHandler(logfileh)

- # stop logging on stdout from now on - #log.propagate = False self.log = log - + def _is_blacklisted(self, username): """Check if a user is blacklisted.

@@ -141,7 +140,6 @@ class TwitterBot(object): Return: a string containing the given message.

""" - #self.log.debug("Getting message '%s' for locale %s" % (msgid, lc)) try: t = gettext.translation(lc, self.i18ndir, languages=[lc]) _ = t.ugettext @@ -161,7 +159,7 @@ class TwitterBot(object):

:return: request (list) 3-tuple with locale, os and type of request. """ - + # core knows what OS are supported supported_os = self.core.get_supported_os() supported_lc = self.core.get_supported_lc() @@ -198,7 +196,7 @@ class TwitterBot(object): req['type'] = 'mirrors' if (found_lc and found_os) or (found_lc and found_mirrors): break - + return req

def parse_request(self, dm): @@ -220,31 +218,30 @@ class TwitterBot(object):

try: if self._is_blacklisted(str(sender_id)): - self.log.info("Request from blacklisted account!") - status = 'blacklisted' + self.log.info('REQUEST: blacklist; OS: none; LC: none') bogus_request = True

if not bogus_request: self.log.debug("Request seems legit, let's parse it") # let's try to guess what the user is asking request = self.parse_text(str(msg)) - + # possible options: links, mirrors, help if request['type'] == 'links': - self.log.debug("Type of request: links") + self.log.info('REQUEST: links; OS: %s; LC: %s' % + (req['os'], req['lc'])) links = self.core.get_links( 'Twitter', request['os'], request['lc'] ) - + reply = self._get_msg('links', 'en') reply = reply % (request['os'], request['lc'], links) -

elif request['type'] == 'mirrors': - self.log.debug("Type of request: mirrors") - status = 'success' + self.log.info('REQUEST: mirrors; OS: none; LC: %s' % + req['lc']) reply = self._get_msg('mirrors', 'en') try: with open(self.mirrors, "r") as list_mirrors: @@ -255,10 +252,10 @@ class TwitterBot(object): reply = self._get_msg('mirrors_unavailable', 'en')

else: - self.log.debug("Type of request: help") - status = 'success' + self.log.info('REQUEST: help; OS: none; LC: %s' % + req['lc']) reply = self._get_msg('help', 'en') - + self.api.send_direct_message( user_id=sender_id, text=reply @@ -267,25 +264,18 @@ class TwitterBot(object): except (core.ConfigError, core.InternalError) as e: # if core failes, send the user an error message, but keep going self.log.error("Something went wrong internally: %s" % str(e)) - status = 'core_error' reply = self._get_msg('internal_error', 'en')

- finally: - # keep stats - if request: - self.log.debug("Adding request to database... ") - self.core.add_request_to_db() - def start(self): """ Start the bot for handling requests.

Start a new Twitter bot. """ self.auth = tweepy.OAuthHandler( - self.api_key, + self.api_key, self.api_secret ) - + self.auth.set_access_token( self.access_token, self.token_secret @@ -293,11 +283,10 @@ class TwitterBot(object):

self.api = tweepy.API(self.auth) self.bot_info = self.api.me() - + stream = tweepy.Stream( - auth = self.api.auth, + auth=self.api.auth, listener=GetTorStreamListener(self) ) - - stream.userstream()

+ stream.userstream() diff --git a/gettor/utils.py b/gettor/utils.py index b4ab9c6..8224f25 100644 --- a/gettor/utils.py +++ b/gettor/utils.py @@ -16,25 +16,28 @@ import hashlib """Common utilities for GetTor modules."""

-LOGGING_FORMAT = "[%(levelname)s] %(asctime)s - %(message)s" -DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S +LOGGING_FORMAT = "[%(levelname)s] DATE: %(asctime)s; %(message)s" +DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S +

def get_logging_format(): """Get the logging format. - + :return: (string) the logging format. - + """ return LOGGING_FORMAT

+ def get_date_format(): """Get the date format for logging. - + :return: (string) the date format for logging. - + """ return DATE_FORMAT

+ def get_sha256(string): """Get sha256 of a string.

diff --git a/gettor/xmpp.py b/gettor/xmpp.py index a605f50..072d988 100644 --- a/gettor/xmpp.py +++ b/gettor/xmpp.py @@ -277,8 +277,7 @@ class XMPP(object): self.log.debug("Parsing request") try: if self._is_blacklisted(str(account)): - self.log.info("Request from blacklisted account!") - status = 'blacklisted' + self.log.info('REQUEST: blacklist; OS: none; LC: none') bogus_request = True

# first let's find out how many words are in the message @@ -287,7 +286,7 @@ class XMPP(object): if len(words) > self.max_words: bogus_request = True self.log.info("Message way too long") - status = 'error' + self.log.info('REQUEST: invalid; OS: none; LC: none') reply = self._get_msg('message_error', 'en')

if not bogus_request: @@ -296,13 +295,13 @@ class XMPP(object): req = self._parse_text(str(msg))

if req['type'] == 'help': - self.log.debug("Type of request: help") - status = 'success' + self.log.info('REQUEST: help; OS: none; LC: %s' % + req['lc']) reply = self._get_msg('help', 'en')

elif req['type'] == 'mirrors': - self.log.debug("Type of request: mirrors") - status = 'success' + self.log.info('REQUEST: mirrors; OS: none; LC: %s' % + req['lc']) reply = self._get_msg('mirrors', 'en') try: with open(self.mirrors, "r") as list_mirrors: @@ -312,7 +311,8 @@ class XMPP(object): reply = self._get_msg('mirrors_unavailable', 'en')

elif req['type'] == 'links': - self.log.debug("Type of request: help") + self.log.info('REQUEST: links; OS: %s; LC: %s' % + (req['os'], req['lc'])) links = self.core.get_links( "XMPP", req['os'], @@ -321,22 +321,10 @@ class XMPP(object): reply = self._get_msg('links', 'en') reply = reply % (req['os'], req['lc'], links)

- status = 'success' - except (core.ConfigError, core.InternalError) as e: # if core failes, send the user an error message, but keep going self.log.error("Something went wrong internally: %s" % str(e)) - status = 'core_error' reply = self._get_msg('internal_error', req['lc'])

finally: - # keep stats - if req: - self.log.debug("Adding request to database... ") - self.core.add_request_to_db() - - if reply: - self.log.debug("Everything seems OK. Sending back the reply") - else: - self.log.debug("Nothing to reply!") return reply diff --git a/log/smtp.log b/log/smtp.log index e69de29..3950925 100644 --- a/log/smtp.log +++ b/log/smtp.log @@ -0,0 +1,3 @@ +[INFO] DATE: 2015-11-16; REQUEST: help; OS: none; LC: fa +[INFO] DATE: 2015-11-16; REQUEST: links; OS: windows; LC: zh +[INFO] DATE: 2015-11-16; REQUEST: mirrors; OS: none; LC: tr