[tor-commits] [gettor/master] Remove unused code for twitter and xmpp
hiro at torproject.org
hiro at torproject.org
Fri Aug 30 12:00:06 UTC 2019
commit 1136236221c615901d4e6e81d6e893fa33309fc5
Author: hiro <hiro at torproject.org>
Date: Fri Aug 30 13:59:58 2019 +0200
Remove unused code for twitter and xmpp
---
gettor/services/twitter/twitter.py | 296 --------------------------------
gettor/services/xmpp/xmpp.py | 336 -------------------------------------
2 files changed, 632 deletions(-)
diff --git a/gettor/services/twitter/twitter.py b/gettor/services/twitter/twitter.py
deleted file mode 100644
index fcdc353..0000000
--- a/gettor/services/twitter/twitter.py
+++ /dev/null
@@ -1,296 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This file is part of GetTor.
-#
-# :authors: Israel Leiva <ilv at torproject.org>
-# Based on BridgeDB Twitter distributor (PoC) by wfn
-# - https://github.com/wfn/twidibot
-#
-# :copyright: (c) 2008-2015, The Tor Project, Inc.
-# (c) 2015, Israel Leiva
-#
-# :license: This is Free Software. See LICENSE for license information.
-
-import os
-import re
-import tweepy
-import logging
-import gettext
-import configparser
-
-import core
-import utils
-import blacklist
-
-"""
-Twitter channel for distributing links to download Tor Browser.
-Needs to be refactored to work with twisted and updates to twitter apis
-"""
-
-OS = {
- 'osx': 'Mac OS X',
- 'linux': 'Linux',
- 'windows': 'Windows'
-}
-
-
-class ConfigError(Exception):
- pass
-
-
-class InternalError(Exception):
- pass
-
-
-class GetTorStreamListener(tweepy.StreamListener):
- """ Basic listener for Twitter's streaming API."""
- def __init__(self, bot):
- self.bot = bot
- super(GetTorStreamListener, self).__init__(self.bot.api)
-
- def on_direct_message(self, status):
- """ Right now we only care about direct messages. """
- if status.direct_message['sender']['id_str'] != self.bot.bot_info.id_str:
- self.bot.parse_request(status.direct_message)
-
-
-class TwitterBot(object):
- """ Receive and reply requests via Twitter. """
- def __init__(self, cfg=None):
- """ Create new object by reading a configuration file.
-
- :param: cfg (string) the path of the configuration file.
- """
-
- default_cfg = 'twitter.cfg'
- config = ConfigParser.ConfigParser()
-
- if cfg is None or not os.path.isfile(cfg):
- cfg = default_cfg
-
- try:
- with open(cfg) as f:
- config.readfp(f)
- except IOError:
- raise ConfigError("File %s not found!" % cfg)
-
- try:
- self.api_key = config.get('access_config', 'api_key')
- self.api_secret = config.get('access_config', 'api_secret')
- self.access_token = config.get('access_config', 'access_token')
- self.token_secret = config.get('access_config', 'token_secret')
-
- self.mirrors = config.get('general', 'mirrors')
- self.i18ndir = config.get('i18n', 'dir')
-
- logdir = config.get('log', 'dir')
- logfile = os.path.join(logdir, 'twitter.log')
- loglevel = config.get('log', 'level')
-
- blacklist_cfg = config.get('blacklist', 'cfg')
- self.bl = blacklist.Blacklist(blacklist_cfg)
- self.bl_max_request = config.get('blacklist', 'max_requests')
- self.bl_max_request = int(self.bl_max_request)
- self.bl_wait_time = config.get('blacklist', 'wait_time')
- self.bl_wait_time = int(self.bl_wait_time)
-
- core_cfg = config.get('general', 'core_cfg')
- self.core = core.Core(core_cfg)
-
- except ConfigParser.Error as e:
- raise ConfigError("Configuration error: %s" % str(e))
- except blacklist.ConfigError as e:
- raise InternalError("Blacklist error: %s" % str(e))
- except core.ConfigError as e:
- raise InternalError("Core error: %s" % str(e))
-
- # logging
- log = logging.getLogger(__name__)
-
- logging_format = utils.get_logging_format()
- date_format = utils.get_date_format()
- formatter = logging.Formatter(logging_format, date_format)
-
- log.info('Redirecting Twitter logging to %s' % logfile)
- logfileh = logging.FileHandler(logfile, mode='a+')
- logfileh.setFormatter(formatter)
- logfileh.setLevel(logging.getLevelName(loglevel))
- log.addHandler(logfileh)
-
- self.log = log
-
- def _is_blacklisted(self, username):
- """Check if a user is blacklisted.
-
- :param: addr (string) the hashed username.
-
- :return: true is the username is blacklisted, false otherwise.
-
- """
- hashed_username = utils.get_sha256(username)
-
- try:
- self.bl.is_blacklisted(
- hashed_username,
- 'Twitter',
- self.bl_max_request,
- self.bl_wait_time
- )
- return False
- except blacklist.BlacklistError as e:
- return True
-
- def _get_msg(self, msgid, lc):
- """Get message identified by msgid in a specific locale.
-
- Params: msgid: the identifier of a string.
- lc: the locale.
-
- Return: a string containing the given message.
-
- """
- try:
- t = gettext.translation(lc, self.i18ndir, languages=[lc])
- _ = t.ugettext
-
- msgstr = _(msgid)
- return msgstr
- except IOError as e:
- raise ConfigError("%s" % str(e))
-
- def parse_text(self, msg):
- """ Parse the text part of a message.
-
- Split the message in words and look for patterns for locale,
- operating system and mirrors requests.
-
- :param: msg (string) the message received.
-
- :return: request (list) 3-tuple with locale, os and type of request.
- """
-
- # core knows what OS are supported
- supported_os = self.core.get_supported_os()
- supported_lc = self.core.get_supported_lc()
-
- # default values
- req = {}
- req['lc'] = 'en'
- req['os'] = None
- req['type'] = 'help'
-
- found_lc = False
- found_os = False
- found_mirrors = False
-
- # analyze every word
- words = re.split('\s+', msg.strip())
- for word in words:
- # look for lc and os
- if not found_lc:
- for lc in supported_lc:
- if re.match(lc, word, re.IGNORECASE):
- found_lc = True
- req['lc'] = lc
- if not found_os:
- for os in supported_os:
- if re.match(os, word, re.IGNORECASE):
- found_os = True
- req['os'] = os
- req['type'] = 'links'
- # mirrors
- if not found_mirrors:
- if re.match("mirrors?", word, re.IGNORECASE):
- found_mirrors = True
- req['type'] = 'mirrors'
- if (found_lc and found_os) or (found_lc and found_mirrors):
- break
-
- return req
-
- def parse_request(self, dm):
- """ Process the request received.
-
- Check if the user is not blacklisted and then check the body of
- the message to find out what is asking.
-
- :param: dm (status.direct_message) the direct message object received
- via Twitter API.
-
- """
-
- sender_id = dm['sender']['id_str']
- msg = dm['text'].strip().lower()
- bogus_req = False
- req = None
- status = ''
-
- try:
- if self._is_blacklisted(str(sender_id)):
- self.log.info('blacklist; none; none')
- bogus_req = True
-
- if not bogus_req:
- self.log.debug("Request seems legit, let's parse it")
- # let's try to guess what the user is asking
- req = self.parse_text(str(msg))
-
- # possible options: links, mirrors, help
- if req['type'] == 'links':
- self.log.info('links; %s; %s' % (req['os'], req['lc']))
- links = self.core.get_links(
- 'twitter', req['os'], req['lc']
- )
-
- reply = self._get_msg('links', 'en')
- reply = reply % (OS[req['os']], links)
-
- elif req['type'] == 'mirrors':
- self.log.info('mirrors; none; %s' % req['lc'])
- reply = self._get_msg('mirrors', 'en')
- try:
- with open(self.mirrors, "r") as list_mirrors:
- mirrors = list_mirrors.read()
- reply = reply % mirrors
-
- except IOError as e:
- reply = self._get_msg('mirrors_unavailable', 'en')
-
- else:
- self.log.info('help; none; %s' % req['lc'])
- reply = self._get_msg('help', 'en')
-
- self.api.send_direct_message(
- user_id=sender_id,
- text=reply
- )
-
- except (core.ConfigError, core.InternalError) as e:
- # if core failes, send the user an error message, but keep going
- self.log.error("Something went wrong internally: %s" % str(e))
- reply = self._get_msg('internal_error', 'en')
-
- def start(self):
- """ Start the bot for handling requests.
-
- Start a new Twitter bot.
- """
- self.auth = tweepy.OAuthHandler(
- self.api_key,
- self.api_secret
- )
-
- self.auth.set_access_token(
- self.access_token,
- self.token_secret
- )
-
- self.api = tweepy.API(self.auth)
- self.bot_info = self.api.me()
-
- stream = tweepy.Stream(
- auth=self.api.auth,
- listener=GetTorStreamListener(self)
- )
-
- stream.userstream()
diff --git a/gettor/services/xmpp/xmpp.py b/gettor/services/xmpp/xmpp.py
deleted file mode 100644
index cfcf8bf..0000000
--- a/gettor/services/xmpp/xmpp.py
+++ /dev/null
@@ -1,336 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This file is part of GetTor, a Tor Browser distribution system.
-#
-# :authors: Israel Leiva <ilv at riseup.net>
-# see also AUTHORS file
-#
-# :copyright: (c) 2008-2015, The Tor Project, Inc.
-# (c) 2015, Israel Leiva
-#
-# :license: This is Free Software. See LICENSE for license information.
-
-import os
-import re
-import sys
-import time
-import gettext
-import hashlib
-import logging
-import configparser
-
-from sleekxmpp import ClientXMPP
-from sleekxmpp.xmlstream.stanzabase import JID
-from sleekxmpp.exceptions import IqError, IqTimeout
-
-import core
-import utils
-import blacklist
-
-
-"""
-XMPP module for processing requests.
-Needs to be refactored to work with twisted
-"""
-
-OS = {
- 'osx': 'Mac OS X',
- 'linux': 'Linux',
- 'windows': 'Windows'
-}
-
-
-class ConfigError(Exception):
- pass
-
-
-class InternalError(Exception):
- pass
-
-
-class Bot(ClientXMPP):
- """XMPP bot.
-
- Handle messages and pass them to XMPP module for parsing.
-
- """
-
- def __init__(self, jid, password, xmpp_obj):
- ClientXMPP.__init__(self, jid, password)
-
- self.xmpp = xmpp_obj
- self.add_event_handler("session_start", self.session_start)
- self.add_event_handler("message", self.message)
-
- def session_start(self, event):
- self.send_presence()
- self.get_roster()
-
- try:
- self.get_roster()
- except IqError as err:
- # error getting the roster
- self.xmpp.log.error(err.iq['error']['condition'])
- self.disconnect()
- except IqTimeout:
- # server is taking too long to respond
- self.xmpp.log.error("Server is taking too long to respond")
- self.disconnect()
-
- def message(self, msg):
- if msg['type'] in ('chat', 'normal'):
- msg_to_send = self.xmpp.parse_request(msg['from'], msg['body'])
- if msg_to_send:
- msg.reply(msg_to_send).send()
-
-
-class XMPP(object):
- """Receive and reply requests by XMPP.
-
- Public methods:
-
- parse_request(): parses a message and tries to figure out what the user
- is asking for.
-
- Exceptions:
-
- ConfigError: Bad configuration.
- InternalError: Something went wrong internally.
-
- """
-
- def __init__(self, cfg=None):
- """Create new object by reading a configuration file.
-
- :param: cfg (string) the path of the configuration file.
-
- """
- # define a set of default values
- default_cfg = 'xmpp.cfg'
- config = ConfigParser.ConfigParser()
-
- if cfg is None or not os.path.isfile(cfg):
- cfg = default_cfg
-
- try:
- with open(cfg) as f:
- config.readfp(f)
- except IOError:
- raise ConfigError("File %s not found!" % cfg)
-
- try:
- self.user = config.get('account', 'user')
- self.password = config.get('account', 'password')
-
- self.mirrors = config.get('general', 'mirrors')
- self.max_words = config.get('general', 'max_words')
- self.max_words = int(self.max_words)
- core_cfg = config.get('general', 'core_cfg')
- self.core = core.Core(core_cfg)
- self.i18ndir = config.get('i18n', 'dir')
-
- blacklist_cfg = config.get('blacklist', 'cfg')
- self.bl = blacklist.Blacklist(blacklist_cfg)
- self.bl_max_req = config.get('blacklist', 'max_requests')
- self.bl_max_req = int(self.bl_max_req)
- self.bl_wait_time = config.get('blacklist', 'wait_time')
- self.bl_wait_time = int(self.bl_wait_time)
-
- logdir = config.get('log', 'dir')
- logfile = os.path.join(logdir, 'xmpp.log')
- loglevel = config.get('log', 'level')
-
- except ConfigParser.Error as e:
- raise ConfigError("Configuration error: %s" % str(e))
- except blacklist.ConfigError as e:
- raise InternalError("Blacklist error: %s" % str(e))
- except core.ConfigError as e:
- raise InternalError("Core error: %s" % str(e))
-
- # logging
- log = logging.getLogger(__name__)
-
- logging_format = utils.get_logging_format()
- date_format = utils.get_date_format()
- formatter = logging.Formatter(logging_format, date_format)
-
- log.info('Redirecting XMPP logging to %s' % logfile)
- logfileh = logging.FileHandler(logfile, mode='a+')
- logfileh.setFormatter(formatter)
- logfileh.setLevel(logging.getLevelName(loglevel))
- log.addHandler(logfileh)
-
- # stop logging on stdout from now on
- log.propagate = False
- self.log = log
-
- def start_bot(self):
- """Start the bot for handling requests.
-
- Start a new sleekxmpp bot.
-
- """
- self.log.info("Starting the bot with account %s" % self.user)
- xmpp = Bot(self.user, self.password, self)
- xmpp.connect()
- xmpp.process(block=True)
-
- def _is_blacklisted(self, account):
- """Check if a user is blacklisted.
-
- :param: addr (string) the hashed address of the user.
-
- :return: true is the address is blacklisted, false otherwise.
-
- """
- anon_acc = utils.get_sha256(account)
-
- try:
- self.bl.is_blacklisted(
- anon_acc, 'XMPP', self.bl_max_req, self.bl_wait_time
- )
- return False
- except blacklist.BlacklistError as e:
- return True
-
- def _get_msg(self, msgid, lc):
- """Get message identified by msgid in a specific locale.
-
- :param: msgid (string) the identifier of a string.
- :param: lc (string) the locale.
-
- :return: (string) the message from the .po file.
-
- """
- # obtain the content in the proper language
- self.log.debug("Trying to get translated text")
- try:
- t = gettext.translation(lc, self.i18ndir, languages=[lc])
- _ = t.ugettext
-
- msgstr = _(msgid)
- return msgstr
- except IOError as e:
- raise ConfigError("%s" % str(e))
-
- def _parse_text(self, msg):
- """Parse the text part of a message.
-
- Split the message in words and look for patterns for locale,
- operating system and built-in pluggable transport info.
-
- :param: msg (string) the message received.
- :param: core_obj (object) the object of gettor core module.
-
- :return: request (list) 4-tuple with locale, os, type of request
- and pt info.
-
- """
- # core knows what OS are supported
- supported_os = self.core.get_supported_os()
- supported_lc = self.core.get_supported_lc()
-
- self.log.debug("Parsing text")
- # default values
- req = {}
- req['lc'] = 'en'
- req['os'] = None
- req['type'] = 'help'
-
- found_lc = False
- found_os = False
- found_mirrors = False
-
- # analyze every word
- for word in msg.split(' '):
- # look for lc and os
- if not found_lc:
- for lc in supported_lc:
- if re.match(lc, word, re.IGNORECASE):
- found_lc = True
- req['lc'] = lc
- if not found_os:
- for os in supported_os:
- if re.match(os, word, re.IGNORECASE):
- found_os = True
- req['os'] = os
- req['type'] = 'links'
- # mirrors
- if not found_mirrors:
- if re.match("mirrors?", word, re.IGNORECASE):
- found_mirrors = True
- req['type'] = 'mirrors'
- if (found_lc and found_os) or (found_lc and found_mirrors):
- break
-
- return req
-
- def parse_request(self, account, msg):
- """Process the request received.
-
- Check if the user is not blacklisted and then check the body of
- the message to find out what is asking.
-
- :param: account (string) the account that did the request.
- :param: msg (string) the body of the message sent to us.
-
- :return: (string/None) the message to be sent to the user via the
- bot, or None if the user is blacklisted.
-
- """
- bogus_request = False
- reply = ''
- status = ''
- req = None
-
- self.log.debug("Parsing request")
- try:
- if self._is_blacklisted(str(account)):
- self.log.info('blacklist; none; none')
- bogus_request = True
-
- # first let's find out how many words are in the message
- # request shouldn't be longer than 3 words, but just in case
- words = re.split('\s+', msg.strip())
- if len(words) > self.max_words:
- bogus_request = True
- self.log.info("Message way too long")
- self.log.info('invalid; none; none')
- reply = self._get_msg('message_error', 'en')
-
- if not bogus_request:
- self.log.debug("Request seems legit, let's parse it")
- # let's try to guess what the user is asking
- req = self._parse_text(str(msg))
-
- if req['type'] == 'help':
- self.log.info('help; none; %s' % req['lc'])
- reply = self._get_msg('help', 'en')
-
- elif req['type'] == 'mirrors':
- self.log.info('mirrors; none; %s' % req['lc'])
- reply = self._get_msg('mirrors', 'en')
- try:
- with open(self.mirrors, "r") as list_mirrors:
- mirrors = list_mirrors.read()
- reply = reply % mirrors
- except IOError as e:
- reply = self._get_msg('mirrors_unavailable', 'en')
-
- elif req['type'] == 'links':
- self.log.info('links; %s; %s' % (req['os'], req['lc']))
- links = self.core.get_links(
- "XMPP",
- req['os'],
- req['lc']
- )
- reply = self._get_msg('links', 'en')
- reply = reply % (OS[req['os']], links)
-
- except (core.ConfigError, core.InternalError) as e:
- # if core failes, send the user an error message, but keep going
- self.log.error("Something went wrong internally: %s" % str(e))
- reply = self._get_msg('internal_error', req['lc'])
-
- finally:
- return reply
More information about the tor-commits
mailing list