commit d71e6838bda2a158963228f321d1cb682c5f4297 Author: ilv ilv@users.noreply.github.com Date: Thu Aug 27 15:17:21 2015 -0300

Better debugging, error handling and mirrors option for xmpp --- gettor/xmpp.py | 200 +++++++++++++++++++++-------------- lang/xmpp/i18n/en/LC_MESSAGES/en.po | 72 +++++++++++-- process_chat.py | 22 ++++ xmpp.cfg | 2 + 4 files changed, 207 insertions(+), 89 deletions(-)

diff --git a/gettor/xmpp.py b/gettor/xmpp.py index e9d2a10..f9b025d 100644 --- a/gettor/xmpp.py +++ b/gettor/xmpp.py @@ -5,8 +5,8 @@ # :authors: Israel Leiva ilv@riseup.net # see also AUTHORS file # -# :copyright: (c) 2008-2014, The Tor Project, Inc. -# (c) 2014, Israel Leiva +# :copyright: (c) 2008-2015, The Tor Project, Inc. +# (c) 2015, Israel Leiva # # :license: This is Free Software. See LICENSE for license information.

@@ -20,6 +20,7 @@ import logging import ConfigParser

from sleekxmpp import ClientXMPP +from sleekxmpp.xmlstream.stanzabase import JID from sleekxmpp.exceptions import IqError, IqTimeout

import core @@ -30,6 +31,14 @@ import blacklist """XMPP module for processing requests."""

+class ConfigError(Exception): + pass + + +class InternalError(Exception): + pass + + class Bot(ClientXMPP): """XMPP bot.

@@ -52,10 +61,11 @@ class Bot(ClientXMPP): self.get_roster() except IqError as err: # error getting the roster - # logging.error(err.iq['error']['condition']) + self.xmpp.log.error(err.iq['error']['condition']) self.disconnect() except IqTimeout: # server is taking too long to respond + self.xmpp.log.error("Server is taking too long to respond") self.disconnect()

def message(self, msg): @@ -65,14 +75,6 @@ class Bot(ClientXMPP): msg.reply(msg_to_send).send()

-class ConfigError(Exception): - pass - - -class InternalError(Exception): - pass - - class XMPP(object): """Receive and reply requests by XMPP.

@@ -95,75 +97,63 @@ class XMPP(object):

""" # define a set of default values - DEFAULT_CONFIG_FILE = 'xmpp.cfg' - - logging.basicConfig(format='[%(levelname)s] %(asctime)s - %(message)s', - datefmt="%Y-%m-%d %H:%M:%S") - log = logging.getLogger(__name__) + default_cfg = 'xmpp.cfg' config = ConfigParser.ConfigParser()

if cfg is None or not os.path.isfile(cfg): - cfg = DEFAULT_CONFIG_FILE - - config.read(cfg) + cfg = default_cfg

try: - self.user = config.get('account', 'user') - except ConfigParser.Error as e: - raise ConfigError("Couldn't read 'user' from 'account'") + with open(cfg) as f: + config.readfp(f) + except IOError: + raise ConfigError("File %s not found!" % cfg)

try: + self.user = config.get('account', 'user') self.password = config.get('account', 'password') - except ConfigParser.Error as e: - raise ConfigError("Couldn't read 'password' from 'account'")

- try: - self.core_cfg = config.get('general', 'core_cfg') - except ConfigParser.Error as e: - raise ConfigError("Couldn't read 'core_cfg' from 'general'") + self.mirrors = config.get('general', 'mirrors') + self.max_words = config.get('general', 'max_words') + self.max_words = int(self.max_words) + core_cfg = config.get('general', 'core_cfg') + self.core = core.Core(core_cfg) + self.i18ndir = config.get('i18n', 'dir')

- try: blacklist_cfg = config.get('blacklist', 'cfg') - self.bl = blacklist_cfg - except ConfigParser.Error as e: - raise ConfigError("Couldn't read 'cfg' from 'blacklist'") - - try: + self.bl = blacklist.Blacklist(blacklist_cfg) self.bl_max_req = config.get('blacklist', 'max_requests') self.bl_max_req = int(self.bl_max_req) - except ConfigParser.Error as e: - raise ConfigError("Couldn't read 'max_requests' from 'blacklist'") - - try: self.bl_wait_time = config.get('blacklist', 'wait_time') self.bl_wait_time = int(self.bl_wait_time) - except ConfigParser.Error as e: - raise ConfigError("Couldn't read 'wait_time' from 'blacklist'") - - try: - self.i18ndir = config.get('i18n', 'dir') - except ConfigParser.Error as e: - raise ConfigError("Couldn't read 'dir' from 'i18n'")

- try: logdir = config.get('log', 'dir') logfile = os.path.join(logdir, 'xmpp.log') - except ConfigParser.Error as e: - raise ConfigError("Couldn't read 'dir' from 'log'") - - try: loglevel = config.get('log', 'level') + except ConfigParser.Error as e: - raise ConfigError("Couldn't read 'level' from 'log'") + raise ConfigError("Configuration error: %s" % str(e)) + except blacklist.ConfigError as e: + raise InternalError("Blacklist error: %s" % str(e)) + except core.ConfigError as e: + raise InternalError("Core error: %s" % str(e)) + + # logging + log = logging.getLogger(__name__)

- # establish log level and redirect to log file - log.info('Redirecting logging to %s' % logfile) + logging_format = utils.get_logging_format() + date_format = utils.get_date_format() + formatter = logging.Formatter(logging_format, date_format) + + log.info('Redirecting XMPP logging to %s' % logfile) logfileh = logging.FileHandler(logfile, mode='a+') + logfileh.setFormatter(formatter) logfileh.setLevel(logging.getLevelName(loglevel)) log.addHandler(logfileh)

# stop logging on stdout from now on log.propagate = False + self.log = log

def start_bot(self): """Start the bot for handling requests. @@ -171,6 +161,7 @@ class XMPP(object): Start a new sleekxmpp bot.

""" + self.log.info("Starting the bot with account %s" % self.user) xmpp = Bot(self.user, self.password, self) xmpp.connect() xmpp.process(block=True) @@ -184,11 +175,11 @@ class XMPP(object):

""" anon_acc = utils.get_sha256(account) - bl = blacklist.Blacklist(self.bl)

try: - bl.is_blacklisted(anon_acc, 'XMPP', self.bl_max_req, - self.bl_wait_time) + self.bl.is_blacklisted( + anon_acc, 'XMPP', self.bl_max_req, self.bl_wait_time + ) return False except blacklist.BlacklistError as e: return True @@ -203,13 +194,17 @@ class XMPP(object):

""" # obtain the content in the proper language - t = gettext.translation(lc, self.i18ndir, languages=[lc]) - _ = t.ugettext + self.log.debug("Trying to get translated text") + try: + t = gettext.translation(lc, self.i18ndir, languages=[lc]) + _ = t.ugettext

- msgstr = _(msgid) - return msgstr + msgstr = _(msgid) + return msgstr + except IOError as e: + raise ConfigError("%s" % str(e))

- def _parse_text(self, msg, core_obj): + def _parse_text(self, msg): """Parse the text part of a message.

Split the message in words and look for patterns for locale, @@ -223,20 +218,21 @@ class XMPP(object):

""" # core knows what OS are supported - supported_os = core_obj.get_supported_os() - supported_lc = core_obj.get_supported_lc() + supported_os = self.core.get_supported_os() + supported_lc = self.core.get_supported_lc()

+ self.log.debug("Parsing text") # default values req = {} req['lc'] = 'en' req['os'] = None req['type'] = 'help' + found_lc = False found_os = False + found_mirrors = False

# analyze every word - # request shouldn't be more than 10 words long, so there should - # be a limit for the amount of words for word in msg.split(' '): # look for lc and os if not found_lc: @@ -250,6 +246,14 @@ class XMPP(object): found_os = True req['os'] = os req['type'] = 'links' + # mirrors + if not found_mirrors: + if re.match("mirrors?", word, re.IGNORECASE): + found_mirrors = True + req['type'] = 'mirrors' + if (found_lc and found_os) or (found_lc and found_mirrors): + break + return req

def parse_request(self, account, msg): @@ -269,36 +273,70 @@ class XMPP(object): reply = '' status = '' req = None - core_obj = core.Core(self.core_cfg)

+ self.log.debug("Parsing request") try: if self._is_blacklisted(str(account)): + self.log.info("Request from blacklisted account!") status = 'blacklisted' bogus_request = True

+ # first let's find out how many words are in the message + # request shouldn't be longer than 3 words, but just in case + words = msg.split(' ') + if len(words) > self.max_words: + bogus_request = True + self.log.info("Message way too long") + status = 'error' + reply = self._get_msg('message_error', 'en') + if not bogus_request: + self.log.debug("Request seems legit, let's parse it") # let's try to guess what the user is asking - req = self._parse_text(str(msg), core_obj) + req = self._parse_text(str(msg))

if req['type'] == 'help': + self.log.debug("Type of request: help") status = 'success' - reply = self._get_msg('help', req['lc']) - elif req['type'] == 'links': + reply = self._get_msg('help', 'en') + + elif req['type'] == 'mirrors': + self.log.debug("Type of request: mirrors") + status = 'success' + reply = self._get_msg('mirrors', 'en') try: - links = core_obj.get_links("XMPP", req['os'], - req['lc']) - reply = self._get_msg('links', req['lc']) - reply = reply % (req['os'], req['lc'], links) - - status = 'success' - except (core.ConfigError, core.InternalError) as e: - # if core failes, send the user an error message, but - # keep going - status = 'core_error' - reply = self._get_msg('internal_error', req['lc']) + with open(self.mirrors, "r") as list_mirrors: + mirrors = list_mirrors.read() + reply = reply % mirrors + except IOError as e: + reply = self._get_msg('mirrors_unavailable', 'en') + + elif req['type'] == 'links': + self.log.debug("Type of request: help") + links = self.core.get_links( + "XMPP", + req['os'], + req['lc'] + ) + reply = self._get_msg('links', 'en') + reply = reply % (req['os'], req['lc'], links) + + status = 'success' + + except (core.ConfigError, core.InternalError) as e: + # if core failes, send the user an error message, but keep going + self.log.error("Something went wrong internally: %s" % str(e)) + status = 'core_error' + reply = self._get_msg('internal_error', req['lc']) + finally: # keep stats if req: - core_obj.add_request_to_db() + self.log.debug("Adding request to database... ") + self.core.add_request_to_db()

+ if reply: + self.log.debug("Everything seems OK. Sending back the reply") + else: + self.log.debug("Nothing to reply!") return reply diff --git a/lang/xmpp/i18n/en/LC_MESSAGES/en.po b/lang/xmpp/i18n/en/LC_MESSAGES/en.po index d242625..8cc715e 100644 --- a/lang/xmpp/i18n/en/LC_MESSAGES/en.po +++ b/lang/xmpp/i18n/en/LC_MESSAGES/en.po @@ -2,21 +2,77 @@ domain "en"

#: Links msgid "links" -msgstr "Links %s-%s:\n %s" +msgstr "Hello there! this is the 'GetTor' robot.\n\ +\n\ +Below are the links for your request (Tor Browser for %s, %s package):\n\ +\n\ +%s\n\ +\n\ +===========================================================================\n\ +Still need help? If you have any questions, trouble connecting to Tor\n\ +network, or need to talk to a human, please contact our support team at:\n\ +\n\ + help@rt.torproject.org\n\ +\n\ +We are ready to answer your queries in English, Farsi, Chinese, Arabic,\n\ +French and Spanish."

-#: Links -msgid "links_pt" -msgstr "Links-PT for %s-%s:\n %s" +#: Mirrors message +msgid "mirrors" +msgstr "Hello there! this is the 'GetTor' robot.\n\ +\n\ +Thank you for your request. Below you will find an updated list of mirrors\n\ +of Tor Project's website.\n\ +\n\ +%s\n\ +\n\ +Still need help? If you have any questions, trouble connecting to Tor\n\ +network, or need to talk to a human, please contact our support team at:\n\ +\n\ + help@rt.torproject.org\n\ +\n\ +We are ready to answer your queries in English, Farsi, Chinese, Arabic,\n\ +French and Spanish."

#: Help msgid "help" -msgstr "*help*" +msgstr "Hello there! this is the 'GetTor' robot.\n\ +\n\ +Thank you for your request. I am here to help you download the latest\n\ +Tor Browser.\n\ +\n\ +Please reply to this message with one of the options below (where lc stands\n\ +for the locale of the package you want e.g. en):\n\ +\n\ + windows lc\n\ + linux lc\n\ + osx lc\n\ + mirrors\n\ +\n\ +The currently supported locales are:\n\ +\n\ + en: English\n\ +\n\ +And I will send you the download/access instructions quickly."

-#: Unsupported locale -msgid "unsupported_lc" -msgstr "Unsupported locale" +#: Mirrors unavailable message +msgid "mirrors_unavailable" +msgstr "Hello there! this is the 'GetTor' robot.\n\ +\n\ +I'm sorry, I can't send you mirrors right now, please try again later.\n\ +\n\ +Still need help? If you have any questions, trouble connecting to Tor\n\ +network, or need to talk to a human, please contact our support team at:\n\ +\n\ + help@rt.torproject.org\n\ +\n\ +We are ready to answer your queries in English, Farsi, Chinese, Arabic,\n\ +French and Spanish."

#: Internal error msgid "internal_error" msgstr "Internal error"

+#: Internal error +msgid "message_error" +msgstr "Message too long." diff --git a/process_chat.py b/process_chat.py new file mode 100644 index 0000000..6ad7b59 --- /dev/null +++ b/process_chat.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import logging + +import gettor.xmpp + +def main(): + try: + bot = gettor.xmpp.XMPP() + bot.start_bot() + except gettor.xmpp.ConfigError as e: + print "Configuration error: %s" % str(e) + except gettor.xmpp.InternalError as e: + print "Core module not working: %s" % str(e) + except Exception as e: + # in case something unexpected happens + print "Unexpected error: %s" % str(e) + +if __name__ == '__main__': + main() diff --git a/xmpp.cfg b/xmpp.cfg index 0b2f547..d81d4df 100644 --- a/xmpp.cfg +++ b/xmpp.cfg @@ -5,6 +5,8 @@ password: [general] basedir: /path/to/gettor/xmpp/ core_cfg: /path/to/core.cfg +max_words: 10 +db: /path/to/gettor.db

[blacklist] cfg: /path/to/blacklist.cfg