[tor-commits] [gettor/master] Lots of changes: added blacklist and stats via sqlite database, cleaned the code (including comments), and minor changes on SMTP module. Both SMTP and XMPP tested. This is the code that will be submitted to melange.
ilv at torproject.org
ilv at torproject.org
Tue Sep 22 23:39:12 UTC 2015
commit 02435673554fc7240cafae72edb644e5ea6767e6
Author: ilv <ilv at users.noreply.github.com>
Date: Mon Aug 25 17:45:34 2014 -0400
Lots of changes: added blacklist and stats via sqlite database, cleaned the code (including comments), and minor changes on SMTP module. Both SMTP and XMPP tested. This is the code that will be submitted to melange.
---
src/LICENSE | 31 ++
src/blacklist.cfg | 6 +
src/core.cfg | 7 +-
src/dropbox.py | 14 +-
src/gettor/blacklist.py | 143 ++++++++
src/gettor/core.py | 37 +-
src/gettor/db.py | 113 ++++++
src/gettor/smtp.py | 666 ++++++++++++++++++------------------
src/gettor/utils.py | 27 +-
src/gettor/xmpp.py | 389 ++++++++++-----------
src/i18n/en/LC_MESSAGES/en.mo | Bin 1967 -> 1730 bytes
src/i18n/en/LC_MESSAGES/en.po | 11 +-
src/i18n/es/LC_MESSAGES/es.mo | Bin 2073 -> 1811 bytes
src/i18n/es/LC_MESSAGES/es.po | 15 +-
src/providers/dropbox.links | 5 +-
src/scripts/blacklist.py | 111 ++++++
src/scripts/create_db.py | 54 +++
src/scripts/stats.py | 123 +++++++
src/smtp.cfg | 14 +-
src/xmpp.cfg | 13 +-
src/xmpp/i18n/en/LC_MESSAGES/en.po | 20 +-
src/xmpp/i18n/es/LC_MESSAGES/es.po | 18 +-
22 files changed, 1216 insertions(+), 601 deletions(-)
diff --git a/src/LICENSE b/src/LICENSE
new file mode 100644
index 0000000..c71c94c
--- /dev/null
+++ b/src/LICENSE
@@ -0,0 +1,31 @@
+gettor is distributed under this license:
+
+Copyright (c) 2008-2014, The Tor Project, Inc.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+ * Neither the names of the copyright owners nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/blacklist.cfg b/src/blacklist.cfg
new file mode 100644
index 0000000..bb8a3d8
--- /dev/null
+++ b/src/blacklist.cfg
@@ -0,0 +1,6 @@
+[general]
+db: /path/to/gettor.db
+
+[log]
+level: DEBUG
+dir: /path/to/log
diff --git a/src/core.cfg b/src/core.cfg
index 3492681..0b3231e 100644
--- a/src/core.cfg
+++ b/src/core.cfg
@@ -1,10 +1,11 @@
[general]
-basedir: ./
+basedir: /path/to/gettor
+db: gettor.db
[links]
dir: providers/
-os: linux, windows, osx
-locales: es, en
+os: linux,windows,osx
+locales: es,en
[log]
dir: log/
diff --git a/src/dropbox.py b/src/dropbox.py
index 2b619d3..baaafee 100644
--- a/src/dropbox.py
+++ b/src/dropbox.py
@@ -2,6 +2,14 @@
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
+# :authors: Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
+
import re
import os
import gnupg
@@ -46,11 +54,7 @@ def get_bundle_info(file):
raise ValueError("Bundle invalid format %s" % file)
def get_file_sha256(file):
- """Get the sha256 of a file.
-
- Desc.
-
- """
+ """Get the sha256 of a file."""
# as seen on the internet
BLOCKSIZE = 65536
diff --git a/src/gettor/blacklist.py b/src/gettor/blacklist.py
new file mode 100644
index 0000000..1c721bb
--- /dev/null
+++ b/src/gettor/blacklist.py
@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of GetTor, a Tor Browser Bundle distribution system.
+#
+# :authors: Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+import os
+import time
+import logging
+import sqlite3
+import datetime
+import ConfigParser
+
+import db
+import utils
+
+"""Blacklist module for managing blacklisting of users."""
+
+
+class BlacklistError(Exception):
+ pass
+
+
+class Blacklist(object):
+ """Manage blacklisting of users.
+
+ Public methods:
+
+ is_blacklisted(): Check if someone is blacklisted.
+
+ Exceptions:
+
+ ConfigurationError: Bad configuration.
+ BlacklistError: User is blacklisted.
+
+ """
+
+ def __init__(self, cfg=None):
+ """Create new object by reading a configuration file.
+
+ :param: cfg (string) path of the configuration file.
+
+ """
+ # define a set of default values
+ DEFAULT_CONFIG_FILE = 'blacklist.cfg'
+
+ logging.basicConfig(format='[%(levelname)s] %(asctime)s - %(message)s',
+ datefmt="%Y-%m-%d %H:%M:%S")
+ log = logging.getLogger(__name__)
+ config = ConfigParser.ConfigParser()
+
+ if cfg is None or not os.path.isfile(cfg):
+ cfg = DEFAULT_CONFIG_FILE
+ log.info("Using default configuration")
+
+ log.info("Reading configuration file %s" % cfg)
+ config.read(cfg)
+
+ try:
+ dbname = config.get('general', 'db')
+ self.db = db.DB(dbname)
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'db' from 'general' (%s)" % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
+ self.logdir = config.get('log', 'dir')
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'dir' from 'log' (%s)" % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
+ self.loglevel = config.get('log', 'level')
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'level' from 'log' (%s)" % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ # keep log levels separated
+ self.log = utils.filter_logging(log, self.logdir, self.loglevel)
+ self.log.setLevel(logging.getLevelName(self.loglevel))
+ log.debug('Redirecting logging to %s' % self.logdir)
+
+ # stop logging on stdout from now on
+ log.propagate = False
+ self.log.debug("New blacklist object created")
+
+ def is_blacklisted(self, user, service, max_req, wait_time):
+ """Check if a user is blacklisted.
+
+ The user is blacklisted if:
+
+ a) The 'blocked' field is set to one, meaning that is permanently
+ blacklisted.
+
+ b) Does too many requests on a short period of time. For now, a user
+ that makes more than 'max_req' requests should wait 'wait_time'
+ minutes to make a new request.
+
+ :param: user (string) the hashed user.
+ :param: service (string) the service the user is making a request to.
+ :param: max_req (int) maximum number of requests a user can make
+ in a row.
+ :param: wait_time (int) amount of time the user must wait before
+ making requests again after 'max_req' requests is reached.
+ For now this is considered in minutes.
+
+ :raise: BlacklistError if the user is blacklisted
+
+ """
+ r = self.db.get_user(user, service)
+ if r:
+ # permanently blacklisted
+ if r['blocked']:
+ self.log.info("Request from blocked user %s" % user)
+ self.db.update_user(user, service, r['times']+1, 1)
+ raise BlacklistError("Blocked user")
+ # don't be greedy
+ elif r['times'] >= max_req:
+ last = datetime.datetime.fromtimestamp(float(
+ r['last_request']))
+ next = last + datetime.timedelta(minutes=wait_time)
+
+ if datetime.datetime.now() < next:
+ self.log.info("Too many requests from user %s" % user)
+ self.db.update_user(user, service, r['times']+1, 0)
+ raise BlacklistError("Too many requests")
+ else:
+ # fresh user again!
+ self.log.debug("Request after wait time, cleaning up for"
+ " %s" % user)
+ self.db.update_user(user, service, 1, 0)
+ else:
+ self.log.debug("Adding up a request for %s" % user)
+ self.db.update_user(user, service, r['times']+1, 0)
+ else:
+ self.log.debug("New request for %s" % user)
+ self.db.add_user(user, service, 0)
diff --git a/src/gettor/core.py b/src/gettor/core.py
index 407e080..f25b27d 100644
--- a/src/gettor/core.py
+++ b/src/gettor/core.py
@@ -2,6 +2,13 @@
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
+# :authors: Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
import os
import re
@@ -10,6 +17,7 @@ import logging
import tempfile
import ConfigParser
+import db
import utils
"""Core module for getting links from providers."""
@@ -48,7 +56,7 @@ class Core(object):
create_links_file(): Create a file to store links of a provider.
add_link(): Add a link to a links file of a provider.
get_supported_os(): Get a list of supported operating systems.
- get_supported_locale(): Get a list of supported locales.
+ get_supported_lc(): Get a list of supported locales.
Exceptions:
@@ -92,6 +100,14 @@ class Core(object):
raise ConfigurationError("Error with conf. See log file.")
try:
+ dbname = config.get('general', 'db')
+ dbname = os.path.join(self.basedir, dbname)
+ self.db = db.DB(dbname)
+ except ConfigParser.Error as e:
+ logger.warning("Couldn't read 'db' from 'general' (%s)" % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
self.linksdir = config.get('links', 'dir')
self.linksdir = os.path.join(self.basedir, self.linksdir)
except ConfigParser.Error as e:
@@ -273,7 +289,7 @@ class Core(object):
"""
return self.supported_os.split(',')
- def get_supported_locales(self):
+ def get_supported_lc(self):
"""Public method to get the list of supported locales.
Returns: List of strings.
@@ -400,3 +416,20 @@ class Core(object):
% operating_system)
else:
raise LinkFileError("There is no links file for %s" % provider)
+
+ def add_request_to_db(self, service, type, os, lc, pt, status, logfile):
+ """Add request to database.
+
+ This is for keeping stats about what is the most, less requested
+ and stuff like that. Hopefully it will help to improve user experience.
+
+ :param: type (string) the type of the request.
+ :param: os (string) the operating system.
+ :param: lc (string) the locale.
+ :param: pt (bool) true if the user asked about pt, false otherwise.
+ :param: status (string) short text describing the status.
+ :param: logfile (string) path of the logfile of the email in case
+ something went really wrong (address blacklisted/malformed).
+
+ """
+ self.db.add_request(service, type, os, lc, pt, status, logfile)
diff --git a/src/gettor/db.py b/src/gettor/db.py
new file mode 100644
index 0000000..1f6cfbd
--- /dev/null
+++ b/src/gettor/db.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of GetTor, a Tor Browser Bundle distribution system.
+#
+# :authors: Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+import time
+import sqlite3
+import datetime
+
+"""DB interface for comunicating with sqlite3"""
+
+
+class DB(object):
+ """
+
+ Public methods:
+
+ add_request(): add a request to the database (requests table).
+ get_user(): get user info from the database (users table).
+ add_user(): add a user to the database (users table).
+ update_user(): update a user on the database (users table).
+
+ """
+
+ def __init__(self, dbname):
+ """Create a new db object.
+
+ :param: dbname (string) the path of the database.
+
+ """
+ self.con = sqlite3.connect(dbname)
+ self.con.row_factory = sqlite3.Row
+
+ def add_request(self, service, type, os, lc, pt, status, logfile):
+ """Add a request to the database.
+
+ This is for keeping stats about what is the most, less requested
+ and stuff like that. Hopefully it will help to improve user experience.
+
+ :param: type (string) the type of the request.
+ :param: os (string) the operating system.
+ :param: lc (string) the locale.
+ :param: pt (bool) true if the user asked about pt, false otherwise.
+ :param: status (string) short text describing the status.
+ :param: logfile (string) path of the logfile of the email in case
+ something went really wrong (address blacklisted/malformed).
+
+ """
+ now = datetime.datetime.now()
+
+ with self.con:
+ cur = self.con.cursor()
+ cur.execute("INSERT INTO requests VALUES(?,?,?,?,?,?,?,?,?,?)",
+ (service, type, os, lc, pt, now.year, now.month,
+ now.day, status, logfile))
+
+ def get_user(self, user, service):
+ """Get user info from the database.
+
+ :param: user (string) unique (hashed) string that represents the user.
+ :param: service (string) the service related to the user (e.g. SMTP).
+
+ :return: (dict) the user information, with fields as indexes
+ (e.g. row['user']).
+
+ """
+ with self.con:
+ cur = self.con.cursor()
+ cur.execute("SELECT * FROM users WHERE id =? AND service =?",
+ (user, service))
+
+ row = cur.fetchone()
+ return row
+
+ def add_user(self, user, service, blocked):
+ """Add a user to the database.
+
+ We add a user with one 'times' and the current time as 'last_request'
+ by default.
+
+ :param: user (string) unique (hashed) string that represents the user.
+ :param: service (string) the service related to the user (e.g. SMTP).
+ :param: blocked (int) one if user is blocked, zero otherwise.
+
+ """
+ with self.con:
+ cur = self.con.cursor()
+ cur.execute("INSERT INTO users VALUES(?,?,?,?,?)",
+ (user, service, 1, blocked, str(time.time())))
+
+ def update_user(self, user, service, times, blocked):
+ """Update a user on the database.
+
+ We update the user info with the current time as 'last_request'.
+
+ :param: user (string) unique (hashed) string that represents the user.
+ :param: service (string) the service related to the user (e.g. SMTP).
+ :param: times (int) the number of requests the user has made.
+ :param: blocked (int) one if user is blocked, zero otherwise.
+
+ """
+ with self.con:
+ cur = self.con.cursor()
+ cur.execute("UPDATE users SET times =?, blocked =?,"
+ " last_request =? WHERE id =? AND service =?",
+ (times, blocked, str(time.time()), user, service))
diff --git a/src/gettor/smtp.py b/src/gettor/smtp.py
index b563600..b4fa3cd 100644
--- a/src/gettor/smtp.py
+++ b/src/gettor/smtp.py
@@ -2,6 +2,13 @@
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
+# :authors: Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
import os
@@ -10,12 +17,16 @@ import sys
import time
import email
import gettext
-import hashlib
import logging
+import smtplib
+import datetime
import ConfigParser
-import utils
+from email.mime.text import MIMEText
+
import core
+import utils
+import blacklist
"""SMTP module for processing email requests."""
@@ -24,10 +35,6 @@ class ConfigurationError(Exception):
pass
-class BlacklistError(Exception):
- pass
-
-
class AddressError(Exception):
pass
@@ -50,7 +57,6 @@ class SMTP(object):
Exceptions:
ConfigurationError: Bad configuration.
- BlacklistError: Address of the sender is blacklisted.
AddressError: Address of the sender malformed.
SendEmailError: SMTP server not responding.
InternalError: Something went wrong internally.
@@ -60,168 +66,171 @@ class SMTP(object):
def __init__(self, cfg=None):
"""Create new object by reading a configuration file.
- Params: cfg - path of the configuration file.
+ :param: cfg (string) path of the configuration file.
"""
- # Define a set of default values
+ # define a set of default values
DEFAULT_CONFIG_FILE = 'smtp.cfg'
logging.basicConfig(format='[%(levelname)s] %(asctime)s - %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
- logger = logging.getLogger(__name__)
+ log = logging.getLogger(__name__)
config = ConfigParser.ConfigParser()
if cfg is None or not os.path.isfile(cfg):
cfg = DEFAULT_CONFIG_FILE
- logger.info("Using default configuration")
+ log.info("Using default configuration")
- logger.info("Reading configuration file %s" % cfg)
+ log.info("Reading configuration file %s" % cfg)
config.read(cfg)
try:
self.basedir = config.get('general', 'basedir')
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'basedir' from 'general' (%s)" % cfg)
+ log.warning("Couldn't read 'basedir' from 'general' (%s)" % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
+ self.our_domain = config.get('general', 'our_domain')
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'our_domain' from 'general' (%s)" % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
+ core_cfg = config.get('general', 'core_cfg')
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'core_cfg' from 'general' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
- self.delay = config.get('general', 'delay')
- # There has to be a better way for doing this...
- if self.delay == 'False':
- self.delay = False
+ blacklist_cfg = config.get('blacklist', 'cfg')
+ self.bl = blacklist.Blacklist(blacklist_cfg)
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'cfg' from 'blacklist' (%s)" % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
+ self.bl_max_req = config.get('blacklist', 'max_requests')
+ self.bl_max_req = int(self.bl_max_req)
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'max_requests' from 'blacklist' (%s)"
+ % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+ try:
+ self.bl_wait_time = config.get('blacklist', 'wait_time')
+ self.bl_wait_time = int(self.bl_wait_time)
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'delay' from 'general' (%s)" % cfg)
+ log.warning("Couldn't read 'wait_time' from 'blacklist' (%s)"
+ % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
- self.our_addr = config.get('general', 'our_addr')
+ self.i18ndir = config.get('i18n', 'dir')
+ self.i18ndir = os.path.join(self.basedir, self.i18ndir)
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'our_addr' from 'general' (%s)" %
- cfg)
+ log.warning("Couldn't read 'dir' from 'i18n' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.logdir = config.get('log', 'dir')
self.logdir = os.path.join(self.basedir, self.logdir)
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'dir' from 'log' (%s)" % cfg)
+ log.warning("Couldn't read 'dir' from 'log' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.logdir_emails = config.get('log', 'emails_dir')
self.logdir_emails = os.path.join(self.logdir, self.logdir_emails)
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'emails_dir' from 'log' (%s)" % cfg)
+ log.warning("Couldn't read 'emails_dir' from 'log' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.loglevel = config.get('log', 'level')
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'level' from 'log' (%s)" % cfg)
+ log.warning("Couldn't read 'level' from 'log' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
- # Use default values
- self.core = core.Core()
-
- # Keep log levels separated
- self.logger = utils.filter_logging(logger, self.logdir, self.loglevel)
- self.logger.setLevel(logging.getLevelName(self.loglevel))
- logger.debug('Redirecting logging to %s' % self.logdir)
-
- # Stop logging on stdout from now on
- logger.propagate = False
- self.logger.debug("New smtp object created")
+ # use default values
+ self.core = core.Core(core_cfg)
- def _get_sha256(self, string):
- """Get sha256 of a string.
+ # keep log levels separated
+ self.log = utils.filter_logging(log, self.logdir, self.loglevel)
+ self.log.setLevel(logging.getLevelName(self.loglevel))
+ log.debug('Redirecting logging to %s' % self.logdir)
- Used whenever we want to do things with addresses (log, blacklist,
- etc.)
+ # stop logging on stdout from now on
+ log.propagate = False
+ self.log.debug("New smtp object created")
- Params: The string to be hashed.
-
- Returns: sha256 of string.
-
- """
- return str(hashlib.sha256(string).hexdigest())
-
- def _log_request(self, addr, raw_msg):
+ def _log_email(self, addr, content):
"""Log a request.
This should be called when something goes wrong. It saves the
email content that triggered the malfunctioning.
- Raises: InternalError: if something goes wrong while trying to
- save the email.
-
- Params: addr - The address of the sender.
- content - The content of the email received.
+ :param: addr (string) the address of the sender.
+ :param: content (string) the content of the email received.
+ :return: (string) the path of the logfile of the email.
"""
-
- # to do: obtain content of msg, not the entire email
- content = raw_msg
-
- # We store the sha256 of the original address in order to know when
+ # we store the sha256 of the original address in order to know when
# specific addresses are doing weird requests
- log_addr = self._get_sha256(addr)
- filename = str(time.time()) + '.log'
- path = self.logdir_emails + filename
+ filename = "%s.log" % str(time.time())
+ path = os.path.join(self.logdir_emails, filename)
abs_path = os.path.abspath(path)
- if os.path.isfile(abs_path):
- log_file = open(abs_path, 'w+')
- log_file.write(content)
- log_file.close()
- self.logger.debug("Logging request from %s in %s" %
- (log_addr, abs_path))
- else:
- self.logger.warning("Couldn't open emails' log file (%s)" %
- abs_path)
- raise InternalError("Error while saving the email content.")
+ log_file = open(abs_path, 'w+')
+ log_file.write("Address: %s " % addr)
+ log_file.write("\nBody:\n%s" % content)
+ log_file.close()
+ self.log.debug("Logging request from %s in %s" % (addr, abs_path))
- def _check_blacklist(self, addr):
- """Check if an email address is blacklisted.
+ return filename
- Look for the address in the file of blacklisted addresses.
+ def _is_blacklisted(self, addr):
+ """Check if a user is blacklisted.
- Raises: BlacklistError if the user is blacklisted.
+ :param: addr (string) the hashed address of the user.
- Params: addr - the address we want to check.
+ :return: true is the address is blacklisted, false otherwise.
"""
- anon_addr = self._get_sha256(addr)
- self.logger.debug("Checking if address %s is blacklisted" %
- anon_addr)
-
- # if blacklisted:
- # raise BlacklistError("Address %s is blacklisted!" % anon_addr)
+ self.log.debug("Checking if address %s is blacklisted" % addr)
- def _get_locale(self, addr):
+ try:
+ self.bl.is_blacklisted(addr, 'SMTP', self.bl_max_req,
+ self.bl_wait_time)
+ return False
+ except blacklist.BlacklistError as e:
+ self.log.info("Blacklisted address %s. Reason: %s" % (addr, e))
+ return True
+
+ def _get_lc(self, addr):
"""Get the locale from an email address.
Process the email received and look for the locale in the recipient
address (e.g. gettor+en at torproject.org). If no locale found, english
by default.
- Params: The email address we want to get the locale.
+ :param: (string) the email address we want to get the locale from.
- Returns: String containing the locale.
+ :return: (string) the locale (english if none).
"""
- self.logger.debug("Trying to obtain locale from recipient address")
+ self.log.debug("Trying to obtain locale from recipient address")
- # If no match found, english by default
- locale = 'en'
+ # if no match found, english by default
+ lc = 'en'
- # Look for gettor+locale at torproject.org
- m = re.match('gettor\+(\w\w)@torproject\.org', addr)
+ # look for gettor+locale at torproject.org
+ m = re.match('gettor\+(\w\w)@\w+\.\w+', addr)
if m:
- self.logger.debug("Request for locale %s" % m.groups())
- locale = "%s" % m.groups()
+ self.log.debug("Request for locale %s" % m.groups())
+ lc = "%s" % m.groups()
- return locale.lower()
+ return lc.lower()
def _get_normalized_address(self, addr):
"""Get normalized address.
@@ -229,11 +238,11 @@ class SMTP(object):
We look for anything inside the last '<' and '>'. Code taken from
the old GetTor (utils.py).
- Raises: AddressError: if address can't be normalized.
+ :param: addr (string) the address we want to normalize.
- Params: addr - the address we want to normalize.
+ :raise: AddressError if the address can't be normalized.
- Returns: String with the normalized address on success.
+ :return: (string) the normalized address.
"""
if '<' in addr:
@@ -241,101 +250,137 @@ class SMTP(object):
addr = addr[idx:]
m = re.search(r'<([^>]*)>', addr)
if m is None:
+ self.log.info("Malformed address: %s" % addr)
raise AddressError("Couldn't extract normalized address "
"from %s" % self_get_sha256(addr))
addr = m.group(1)
return addr
- def _parse_email(self, raw_msg, addr):
+ def _get_content(self, email):
+ """Get the body content of an email.
+
+ :param: email (object) the email object to extract the content from.
+
+ :return: (string) body of the message.
+
+ """
+ self.log.debug("Getting the body content of the email")
+ maintype = email.get_content_maintype()
+ if maintype == 'multipart':
+ for part in email.get_payload():
+ if part.get_content_maintype() == 'text':
+ return part.get_payload()
+ elif maintype == 'text':
+ return email.get_payload()
+
+ def _get_msg(self, msgid, lc):
+ """Get message identified by msgid in a specific locale.
+
+ :param: msgid (string) the identifier of a string.
+ :param: lc (string) the locale.
+
+ :return: (string) the message from the .po file.
+
+ """
+ self.log.debug("Getting message '%s' for locale %s" % (msgid, lc))
+ # obtain the content in the proper language
+ t = gettext.translation(lc, self.i18ndir, languages=[lc])
+ _ = t.ugettext
+
+ msgstr = _(msgid)
+ return msgstr
+
+ def _parse_email(self, msg, addr):
"""Parse the email received.
Get the locale and parse the text for the rest of the info.
- Params: raw_msg - content of the email to be parsed.
- addr - address of the recipient (i.e. us).
+ :param: msg (string) the content of the email to be parsed.
+ :param: addr (string) the address of the recipient (i.e. us).
- Returns: a 3-tuple with locale, os and type.
+ :return: (list) 4-tuple with locale, os and type of request.
"""
- self.logger.debug("Parsing email")
+ self.log.debug("Parsing email")
- request = self._parse_text(raw_msg)
- locale = self._get_locale(addr)
- request['locale'] = locale
+ req = self._parse_text(msg)
+ lc = self._get_lc(addr)
+ req['lc'] = lc
- return request
+ return req
- def _parse_text(self, raw_msg):
+ def _parse_text(self, msg):
"""Parse the text part of the email received.
Try to figure out what the user is asking, namely, the type
of request, the package and os required (if applies).
- Params: raw_msg - content of the email to be parsed.
+ :param: msg (string) the content of the email to be parsed.
+
+ :return: (list) 3-tuple with the type of request, os and pt info.
- Returns: Tuple with the type of request and os
- (None if request is for help).
"""
- self.logger.debug("Parsing email text part")
+ self.log.debug("Parsing email text part")
- # By default we asume the request is asking for help
- request = {}
- request['type'] = 'help'
- request['os'] = None
+ # by default we asume the request is asking for help
+ req = {}
+ req['type'] = 'help'
+ req['os'] = None
+ req['pt'] = False
# core knows what OS are supported
supported_os = self.core.get_supported_os()
- lines = raw_msg.split('\n')
+ # we must search word by word, otherwise we could misinterpret
+ # stuff (i.e. 'option' will match .*pt.*)
found_os = False
- for line in lines:
- # Check for help request
- if re.match('.*help.*', line, re.IGNORECASE):
- self.logger.info("Request for help found")
- request['type'] = 'help'
+ found_help = False
+ lines = msg.split(' ')
+ for word in lines:
+ # check for help request
+ if not found_os and re.match('help', word, re.IGNORECASE):
+ self.log.info("Request for help found")
+ req['type'] = 'help'
+ found_help = True
break
- # Check for os
- if not found_os:
- for supported in supported_os:
- p = '.*' + supported + '.*'
- if re.match(p, line, re.IGNORECASE):
- request['os'] = supported
- request['type'] = 'links'
- self.logger.debug("Request for links found")
+ # check for os
+ if not found_os and not found_help:
+ for os in supported_os:
+ if re.match(os, word, re.IGNORECASE):
+ req['os'] = os
+ req['type'] = 'links'
+ self.log.debug("Request for links found")
found_os = True
break
- # Check if the user is asking for terms related to pt
- if re.match("[obfs|plugabble transport|pt]", line, re.IGNORECASE):
- self.logger.info("Request for PT found")
- request['pt'] = True
+ # check if the user is asking for terms related to pt
+ if re.match(".*obfs.*|pluggable|transport|pt",
+ word, re.IGNORECASE):
+ self.log.info("Request for PT found")
+ req['pt'] = True
- return request
+ return req
def _create_email(self, from_addr, to_addr, subject, msg):
"""Create an email object.
This object will be used to construct the reply.
- Params: from_addr - address of the sender.
- to_addr - address of the recipient.
- subject - subject of the email.
- msg - content of the email.
+ :param: from_addr (string) the address of the sender.
+ :param: to_addr (string) the address of the recipient.
+ :param: subject (string) the subject of the email.
+ :param: msg (string) the content of the email.
- Returns: The email object.
+ :return: (object) the email object.
"""
- self.logger.debug("Creating email object for replying")
- # try:
- # email_obj = MIMEtext(msg)
- # email_obj['Subject'] = subject
- # email_obj['From'] = from_addr
- # email_obj['To'] = to_addr
+ self.log.debug("Creating email object for replying")
+ email_obj = MIMEText(msg)
+ email_obj.set_charset("utf-8")
+ email_obj['Subject'] = subject
+ email_obj['From'] = from_addr
+ email_obj['To'] = to_addr
- reply = "From: " + from_addr + ", To: " + to_addr
- reply = reply + ", Subject: " + subject + "\n\n" + msg
-
- # return email_obj
- return reply
+ return email_obj
def _send_email(self, from_addr, to_addr, subject, msg):
"""Send an email.
@@ -343,171 +388,105 @@ class SMTP(object):
Take a 'from' and 'to' addresses, a subject and the content, creates
the email and send it.
- Params: from_addr - address of the sender.
- to_addr - address of the recipient.
- subject - subject of the email.
- msg - content of the email.
+ :param: from_addr (string) the address of the sender.
+ :param: to_addr (string) the address of the recipient.
+ :param: subject (string) the subject of the email.
+ :param: msg (string) the content of the email.
"""
email_obj = self._create_email(from_addr, to_addr, subject, msg)
- # try:
- # s = smtplib.SMTP("localhost")
- # s.sendmail(from_addr, to_addr, msg.as_string())
- # s.quit()
- # except SMTPException as e:
- # self.logger.error("Couldn't send the email: %s" % str(e))
- # raise SendEmailError("Error with SMTP: %s" % str(e))
- print email_obj
- self.logger.debug("Email sent")
-
- def _send_delay(self, locale, from_addr, to_addr):
- """Send delay message.
-
- If the config says so, send a delay message.
-
- Params: locale - two-character string describing a locale.
- from_addr - address of the sender.
- to_addr - address of the recipient.
-
- """
- self.logger.debug("Delay is ON. Sending a delay message.")
-
- # Obtain the content in the proper language and send it
- t = gettext.translation(locale, './i18n', languages=[locale])
- _ = t.ugettext
-
- delay_subject = _('delay_subject')
- delay_msg = _('delay_msg')
try:
- self._send_email(from_addr, to_addr, delay_subject, delay_msg)
- except SendEmailError as e:
- self.logger.warning("Couldn't send delay message")
- raise InternalError("Error while sending delay message")
+ s = smtplib.SMTP("localhost")
+ s.sendmail(from_addr, to_addr, email_obj.as_string())
+ s.quit()
+ except smtplib.SMTPException as e:
+ self.log.error("Couldn't send the email: %s" % str(e))
+ raise SendEmailError("Error with SMTP: %s" % str(e))
+
+ self.log.debug("Email sent")
- def _send_links(self, links, locale, operating_system, from_addr, to_addr,
- pt):
+ def _send_links(self, links, lc, os, from_addr, to_addr, pt):
"""Send links to the user.
Get the message in the proper language (according to the locale),
replace variables and send the email.
- Params: links - links to be sent.
- locale - two-character string describing a locale.
- from_addr - address of the sender.
- to_addr - address of the recipient.
- pt - True/False if the user did a PT request.
+ :param: links (string) the links to be sent.
+ :param: lc (string) the locale.
+ :param: os (string) the operating system.
+ :param: from_addr (string) the address of the sender.
+ :param: to_addr (string) the address of the recipient.
+ :param: pt (bool) true if the used asked for PT info; false otherwise.
"""
- self.logger.debug("Request for links in %s" % locale)
+ self.log.debug("Request for links in %s" % lc)
- # Obtain the content in the proper language and send it
- t = gettext.translation(locale, './i18n', languages=[locale])
- _ = t.ugettext
-
- links_subject = _('links_subject')
- links_msg = _('links_msg')
- links_msg = links_msg % (operating_system, locale, links, links)
+ # obtain the content in the proper language and send it
+ links_subject = self._get_msg('links_subject', lc)
+ links_msg = self._get_msg('links_msg', lc)
+ links_msg = links_msg % (os, lc, links)
- # Don't forget to check if user did a PT request
+ # don't forget to check if user did a PT request
if pt:
- # If so, we get the links message + info about PT included.
- links_subject = _('links_pt_subject')
- links_msg = _('links_pt_msg')
- links_msg = links_msg % (operating_system, locale, links, links)
+ # if so, we get the links message + info about PT included.
+ links_subject = self._get_msg('links_pt_subject', lc)
+ links_msg = self._get_msg('links_pt_msg', lc)
+ links_msg = links_msg % (os, lc, links)
try:
self._send_email(from_addr, to_addr, links_subject, links_msg)
except SendEmailError as e:
- self.logger.warning("Couldn't send links message")
+ self.log.warning("Couldn't send links message")
raise InternalError("Error while sending links message")
- def _send_help(self, locale, from_addr, to_addr):
+ def _send_help(self, lc, from_addr, to_addr):
"""Send help message.
Get the message in the proper language (according to the locale),
replace variables (if any) and send the email.
- Params: locale - two-character string describing a locale.
- from_addr - address of the sender.
- to_addr - address of the recipient.
+ :param: lc (string) the locale.
+ :param: from_addr (string) the address of the sender.
+ :param: to_addr (string) the address of the recipient.
"""
- self.logger.debug("Request for help in %s" % locale)
+ self.log.debug("Request for help in %s" % lc)
- # Obtain the content in the proper language and send it
- t = gettext.translation(locale, './i18n', languages=[locale])
- _ = t.ugettext
-
- help_subject = _('help_subject')
- help_msg = _('help_msg')
+ # obtain the content in the proper language and send it
+ help_subject = self._get_msg('help_subject', lc)
+ help_msg = self._get_msg('help_msg', lc)
try:
self._send_email(from_addr, to_addr, help_subject, help_msg)
except SendEmailError as e:
- self.logger.warning("Couldn't send help message")
+ self.log.warning("Couldn't send help message")
raise InternalError("Error while sending help message")
- def _send_unsupported_os(self, operating_system, locale, from_addr,
- to_addr):
- """Send unsupported OS message.
-
- Get the message for unsupported OS in the proper language
- (according to the locale, or in english if the locale is
- unsupported too), replace variables (if any) and send the email.
-
- Params: locale - two-character string describing a locale.
- from_addr - address of the sender.
- to_addr - address of the recipient.
-
- """
- # Check if the locale is unsupported too
- # If so, english by default
- supported_locales = self.core.get_supported_locales()
- if locale not in supported_locales:
- locale = 'en'
-
- # Obtain the content in the proper language and send it
- t = gettext.translation(locale, './i18n', languages=[locale])
- _ = t.ugettext
-
- unsupported_os_subject = _('unsupported_os_subject')
- unsupported_os_msg = _('unsupported_os_msg')
- unsupported_os_msg = unsupported_os_msg % operating_system
-
- try:
- self._send_email(from_addr, to_addr, unsupported_os_subject,
- unsupported_os_msg)
- except SendEmailError as e:
- self.logger.warning("Couldn't send unsupported OS message")
- raise InternalError("Error while sending unsupported OS message")
-
- def _send_unsupported_locale(self, locale, operating_system, from_addr,
- to_addr):
+ def _send_unsupported_lc(self, lc, os, from_addr, to_addr):
"""Send unsupported locale message.
Get the message for unsupported locale in english replace variables
(if any) and send the email.
- Params: operating_system - name of the operating system.
- from_addr - address of the sender.
- to_addr - address of the recipient.
+ :param: lc (string) the locale.
+ :param: os (string) the operating system.
+ :param: from_addr (string) the address of the sender.
+ :param: to_addr (string) the address of the recipient.
"""
- # Obtain the content in english and send it
- t = gettext.translation('en', './i18n', languages=['en'])
- _ = t.ugettext
-
- unsupported_locale_subject = _('unsupported_locale_subject')
- unsupported_locale_msg = _('unsupported_locale_msg')
- unsupported_locale_msg = unsupported_locale_msg % locale
+ # obtain the content in english and send it
+ unsupported_lc_subject = self._get_msg('unsupported_lc_subject', 'en')
+ unsupported_lc_msg = self._get_msg('unsupported_lc_msg', 'en')
+ unsupported_lc_msg = unsupported_lc_msg % lc
try:
- self._send_email(from_addr, to_addr, unsupported_locale_subject,
- unsupported_locale_msg)
+ self._send_email(from_addr, to_addr, unsupported_lc_subject,
+ unsupported_lc_msg)
+
except SendEmailError as e:
- self.logger.warning("Couldn't send unsupported locale message")
+ self.log.warning("Couldn't send unsupported locale message")
raise InternalError("Error while sending unsupported locale"
"message")
@@ -522,94 +501,103 @@ class SMTP(object):
- Check the type of request.
- Send reply.
- Raises: InternalError if something goes wrong while asking for the
- links to the Core module.
+ :param: raw_msg (string) the email received.
- Params: raw_msg - the email received.
+ :raise: InternalError if something goes wrong while asking for the
+ links to the Core module.
"""
parsed_msg = email.message_from_string(raw_msg)
+ content = self._get_content(parsed_msg)
from_addr = parsed_msg['From']
to_addr = parsed_msg['To']
bogus_request = False
+ logfile = ''
+ status = ''
+ req = None
try:
- norm_from_addr = self._get_normalized_address(from_addr)
- except AddressError as e:
- # This shouldn't stop us from receiving other requests
- self.logger.warning(str(e))
- bogus_request = True
-
- if norm_from_addr:
+ # two ways for a request to be bogus: address malformed or
+ # blacklisted
try:
- self._check_blacklist(self._get_sha256(norm_from_addr))
- except BlacklistError as e:
- # This shouldn't stop us from receiving other requests
- self.logger.warning(str(e))
+ norm_from_addr = self._get_normalized_address(from_addr)
+ except AddressError as e:
+ status = 'malformed'
bogus_request = True
+ # it might be interesting to know what triggered this
+ logfile = self._log_email('malformed', content)
+
+ if norm_from_addr:
+ anon_addr = utils.get_sha256(norm_from_addr)
+
+ if self._is_blacklisted(anon_addr):
+ status = 'blacklisted'
+ bogus_request = True
+ # it might be interesting to know extra info
+ logfile = self._log_email(anon_addr, content)
+
+ if not bogus_request:
+ # try to figure out what the user is asking
+ req = self._parse_email(content, to_addr)
+
+ # our address should have the locale requested
+ our_addr = "gettor+%s@%s" % (req['lc'], self.our_domain)
- if not bogus_request:
- # Try to figure out what the user is asking
- request = self._parse_email(raw_msg, to_addr)
-
- # Two possible options: asking for help or for the links
- self.logger.info("New request for %s" % request['type'])
- if request['type'] == 'help':
- # make sure we can send emails
- try:
- self._send_help(request['locale'], self.our_addr,
- norm_from_addr)
- except SendEmailError as e:
- raise InternalError("Something's wrong with the SMTP "
- "server: %s" % str(e))
-
- elif request['type'] == 'links':
- if self.delay:
+ # two possible options: asking for help or for the links
+ self.log.info("New request for %s" % req['type'])
+ if req['type'] == 'help':
# make sure we can send emails
try:
- self._send_delay(request['locale'], self.our_addr,
- norm_from_addr)
+ self._send_help(req['lc'], our_addr, norm_from_addr)
except SendEmailError as e:
+ status = 'internal_error'
raise InternalError("Something's wrong with the SMTP "
"server: %s" % str(e))
- try:
- self.logger.info("Asking core for links in %s for %s" %
- (request['locale'], request['os']))
-
- links = self.core.get_links('SMTP', request['os'],
- request['locale'])
-
- except core.UnsupportedOSError as e:
- self.logger.info("Request for unsupported OS: %s (%s)" %
- (request['os'], str(e)))
- # if we got here, the address of the sender should be valid
- # so we send him/her a message about the unsupported OS
- self._send_unsupported_os(request['os'], request['locale'],
- self.our_addr, norm_from_addr)
- return
-
- except core.UnsupportedLocaleError as e:
- self.logger.info("Request for unsupported locale: %s (%s)"
- % (request['locale'], str(e)))
- # if we got here, the address of the sender should be valid
- # so we send him/her a message about the unsupported locale
- self._send_unsupported_locale(request['locale'],
- request['os'], self.our_addr,
- norm_from_addr)
- return
-
- # if core fails, we fail too
- except (core.InternalError, core.ConfigurationError) as e:
- self.logger.error("Something's wrong with the Core module:"
- " %s" % str(e))
- raise InternalError("Error obtaining the links.")
-
- # make sure we can send emails
- try:
- self._send_links(links, request['locale'], request['os'],
- self.our_addr, norm_from_addr,
- request['pt'])
- except SendEmailError as e:
- raise SendEmailError("Something's wrong with the SMTP "
- "server: %s" % str(e))
+ elif req['type'] == 'links':
+ try:
+ self.log.info("Asking core for links in %s for %s" %
+ (req['lc'], req['os']))
+
+ links = self.core.get_links('SMTP', req['os'],
+ req['lc'])
+
+ except core.UnsupportedLocaleError as e:
+ self.log.info("Request for unsupported locale: %s (%s)"
+ % (req['lc'], str(e)))
+ # if we got here, the address of the sender should
+ # be valid so we send him/her a message about the
+ # unsupported locale
+ status = 'unsupported_lc'
+ self._send_unsupported_lc(req['lc'], req['os'],
+ our_addr, norm_from_addr)
+ return
+
+ # if core fails, we fail too
+ except (core.InternalError, core.ConfigurationError) as e:
+ status = 'core_error'
+ self.log.error("Something went wrong with Core: %s"
+ % str(e))
+ raise InternalError("Error obtaining the links.")
+
+ # make sure we can send emails
+ try:
+ self._send_links(links, req['lc'], req['os'], our_addr,
+ norm_from_addr, req['pt'])
+ except SendEmailError as e:
+ status = 'internal_error'
+ raise SendEmailError("Something's wrong with the SMTP "
+ "server: %s" % str(e))
+ status = 'success'
+ finally:
+ # keep stats
+ if req:
+ self.core.add_request_to_db('SMTP',
+ req['type'], req['os'],
+ req['lc'], req['pt'],
+ status, logfile)
+ else:
+ # invalid request, so no info about it
+ # logfiles were created for this
+ self.core.add_request_to_db('SMTP', '', '', '', '',
+ status, logfile)
diff --git a/src/gettor/utils.py b/src/gettor/utils.py
index cc927fe..53891ea 100644
--- a/src/gettor/utils.py
+++ b/src/gettor/utils.py
@@ -2,9 +2,17 @@
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
+# :authors: Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
import os
import logging
+import hashlib
"""Common utilities for GetTor modules."""
@@ -23,7 +31,7 @@ class SingleLevelFilter(logging.Filter):
If reject value is false, all but the passlevel will be filtered.
Useful for logging in separated files.
- Params: passlevel - name of a logging level.
+ :param: passlevel (string) the name of a logging level.
"""
@@ -40,11 +48,11 @@ class SingleLevelFilter(logging.Filter):
def filter_logging(logger, dir, level):
"""Create separated files for each level of logging.
- Params: logger - a logging object.
- dir - directory to put the log files.
- level - the level of logging for the all.log file.
+ :param: logger (object) a logging object.
+ :param: dir (string) directory to put the log files.
+ :param: level (string) the level of logging for the all.log file.
- Returns: logger object.
+ :return: (object) a logging object.
"""
# Keep a good format
@@ -85,3 +93,12 @@ def filter_logging(logger, dir, level):
return logger
+def get_sha256(string):
+ """Get sha256 of a string.
+
+ :param: (string) the string to be hashed.
+
+ :return: (string) the sha256 of string.
+
+ """
+ return str(hashlib.sha256(string).hexdigest())
diff --git a/src/gettor/xmpp.py b/src/gettor/xmpp.py
index 9c4bb89..f296559 100644
--- a/src/gettor/xmpp.py
+++ b/src/gettor/xmpp.py
@@ -2,6 +2,13 @@
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
+# :authors: Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
import os
import re
@@ -15,8 +22,9 @@ import ConfigParser
from sleekxmpp import ClientXMPP
from sleekxmpp.exceptions import IqError, IqTimeout
-import utils
import core
+import utils
+import blacklist
"""XMPP module for processing requests."""
@@ -61,10 +69,6 @@ class ConfigurationError(Exception):
pass
-class BlacklistError(Exception):
- pass
-
-
class InternalError(Exception):
pass
@@ -80,7 +84,6 @@ class XMPP(object):
Exceptions:
ConfigurationError: Bad configuration.
- BlacklistError: User is blacklisted.
InternalError: Something went wrong internally.
"""
@@ -88,74 +91,106 @@ class XMPP(object):
def __init__(self, cfg=None):
"""Create new object by reading a configuration file.
- Params: cfg - path of the configuration file.
+ :param: cfg (string) the path of the configuration file.
"""
- # Define a set of default values
+ # define a set of default values
DEFAULT_CONFIG_FILE = 'xmpp.cfg'
logging.basicConfig(format='[%(levelname)s] %(asctime)s - %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
- logger = logging.getLogger(__name__)
+ log = logging.getLogger(__name__)
config = ConfigParser.ConfigParser()
if cfg is None or not os.path.isfile(cfg):
cfg = DEFAULT_CONFIG_FILE
- logger.info("Using default configuration")
+ log.info("Using default configuration")
- logger.info("Reading configuration file %s" % cfg)
+ log.info("Reading configuration file %s" % cfg)
config.read(cfg)
try:
self.user = config.get('account', 'user')
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'user' from 'account' (%s)" % cfg)
+ log.warning("Couldn't read 'user' from 'account' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.password = config.get('account', 'password')
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'password' from 'account' (%s)" %
- cfg)
+ log.warning("Couldn't read 'password' from 'account' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.basedir = config.get('general', 'basedir')
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'basedir' from 'general' (%s)" % cfg)
+ log.warning("Couldn't read 'basedir' from 'general' (%s)" % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
+ self.core_cfg = config.get('general', 'core_cfg')
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'core_cfg' from 'general' (%s)" % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
+ blacklist_cfg = config.get('blacklist', 'cfg')
+ self.bl = blacklist_cfg
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'cfg' from 'blacklist' (%s)" % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
+ self.bl_max_req = config.get('blacklist', 'max_requests')
+ self.bl_max_req = int(self.bl_max_req)
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'max_requests' from 'blacklist' (%s)"
+ % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
+ self.bl_wait_time = config.get('blacklist', 'wait_time')
+ self.bl_wait_time = int(self.bl_wait_time)
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'wait_time' from 'blacklist' (%s)"
+ % cfg)
+ raise ConfigurationError("Error with conf. See log file.")
+
+ try:
+ self.i18ndir = config.get('i18n', 'dir')
+ self.i18ndir = os.path.join(self.basedir, self.i18ndir)
+ except ConfigParser.Error as e:
+ log.warning("Couldn't read 'dir' from 'i18n' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.logdir = config.get('log', 'dir')
self.logdir = os.path.join(self.basedir, self.logdir)
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'dir' from 'log' (%s)" % cfg)
+ log.warning("Couldn't read 'dir' from 'log' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.logdir_msgs = config.get('log', 'msgs_dir')
self.logdir_msgs = os.path.join(self.logdir, self.logdir_msgs)
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'msgs_dir' from 'log' (%s)" % cfg)
+ log.warning("Couldn't read 'msgs_dir' from 'log' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.loglevel = config.get('log', 'level')
except ConfigParser.Error as e:
- logger.warning("Couldn't read 'level' from 'log' (%s)" % cfg)
+ log.warning("Couldn't read 'level' from 'log' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
- # Use default values
- self.core = core.Core()
+ # keep log levels separated
+ self.log = utils.filter_logging(log, self.logdir, self.loglevel)
+ self.log.setLevel(logging.getLevelName(self.loglevel))
+ log.debug('Redirecting logging to %s' % self.logdir)
- # Keep log levels separated
- self.logger = utils.filter_logging(logger, self.logdir, self.loglevel)
- self.logger.setLevel(logging.getLevelName(self.loglevel))
- logger.debug('Redirecting logging to %s' % self.logdir)
-
- # Stop logging on stdout from now on
- logger.propagate = False
- self.logger.debug("New xmpp object created")
+ # stop logging on stdout from now on
+ log.propagate = False
+ self.log.debug("New xmpp object created")
def start_bot(self):
"""Start the bot for handling requests.
@@ -164,177 +199,104 @@ class XMPP(object):
"""
- self.logger.debug("Calling sleekmppp bot")
+ self.log.debug("Calling sleekmppp bot")
xmpp = Bot(self.user, self.password, self)
xmpp.connect()
xmpp.process(block=True)
- def _get_sha256(self, string):
- """Get sha256 of a string.
-
- Used whenever we want to do things with accounts (log, blacklist,
- etc.)
-
- Params: The string to be hashed.
-
- Returns: sha256 of string.
-
- """
- return str(hashlib.sha256(string).hexdigest())
-
- def _check_blacklist(self, account):
- """Check if an account is blacklisted.
-
- Look for the account in the file of blacklisted accounts.
-
- Raises: BlacklistError if the user is blacklisted.
-
- Params: account - the account we want to check.
-
- """
- anon_account = self._get_sha256(account)
- self.logger.debug("Checking if address %s is blacklisted" %
- anon_account)
-
- # if blacklisted:
- # raise BlacklistError("Account %s is blacklisted!" % anon_account)
-
- def _get_help_msg(self, locale):
- """Get help message for a given locale.
-
- Get the message in the proper language (according to the locale),
- replace variables (if any) and return the message.
-
- Return: a string containing the message.
-
- """
- self.logger.debug("Getting help message")
- # Obtain the content in the proper language
- t = gettext.translation(locale, './xmpp/i18n', languages=[locale])
- _ = t.ugettext
-
- help_msg = _('help_msg')
- return help_msg
-
- def _get_unsupported_locale_msg(self, locale):
- """Get unsupported locale message for a given locale.
+ def _is_blacklisted(self, account):
+ """Check if a user is blacklisted.
- Get the message in the proper language (according to the locale),
- replace variables (if any) and return the message.
+ :param: addr (string) the hashed address of the user.
- Return: a string containing the message.
+ :return: true is the address is blacklisted, false otherwise.
"""
- self.logger.debug("Getting unsupported locale message")
- # Obtain the content in the proper language
- t = gettext.translation(locale, './xmpp/i18n', languages=[locale])
- _ = t.ugettext
-
- unsupported_locale_msg = _('unsupported_locale_msg')
- return unsupported_locale_msg
-
- def _get_unsupported_os_msg(self, locale):
- """Get unsupported OS message for a given locale.
-
- Get the message in the proper language (according to the locale),
- replace variables (if any) and return the message.
-
- Return: a string containing the message.
-
- """
- self.logger.debug("Getting unsupported os message")
- # Obtain the content in the proper language
- t = gettext.translation(locale, './xmpp/i18n', languages=[locale])
- _ = t.ugettext
-
- unsupported_os_msg = _('unsupported_os_msg')
- return unsupported_os_msg
+ anon_acc = utils.get_sha256(account)
+ bl = blacklist.Blacklist(self.bl)
+ self.log.debug("Checking if address %s is blacklisted" % anon_acc)
- def _get_internal_error_msg(self, locale):
- """Get internal error message for a given locale.
-
- Get the message in the proper language (according to the locale),
- replace variables (if any) and return the message.
-
- Return: a string containing the message.
-
- """
- self.logger.debug("Getting internal error message")
- # Obtain the content in the proper language
- t = gettext.translation(locale, './xmpp/i18n', languages=[locale])
- _ = t.ugettext
+ try:
+ bl.is_blacklisted(anon_acc, 'XMPP', self.bl_max_req,
+ self.bl_wait_time)
+ return False
+ except blacklist.BlacklistError as e:
+ self.log.info("Blacklisted address %s. Reason: %s" % (anon_acc, e))
+ return True
- internal_error_msg = _('internal_error_msg')
- return internal_error_msg
+ def _get_msg(self, msgid, lc):
+ """Get message identified by msgid in a specific locale.
- def _get_links_msg(self, locale, operating_system, pt, links):
- """Get links message for a given locale, operating system and PT
- request.
+ :param: msgid (string) the identifier of a string.
+ :param: lc (string) the locale.
- Get the message in the proper language (according to the locale),
- replace variables (if any) and return the message.
+ :return: (string) the message from the .po file.
- Return: a string containing the message.
"""
- self.logger.debug("Getting links message")
- # Obtain the content in the proper language
- t = gettext.translation(locale, './xmpp/i18n', languages=[locale])
+ self.log.debug("Getting message '%s' for locale %s" % (msgid, lc))
+ # obtain the content in the proper language
+ t = gettext.translation(lc, self.i18ndir, languages=[lc])
_ = t.ugettext
- if pt:
- links_msg = _('links_pt_msg')
- else:
- links_msg = _('links_msg')
-
- links_msg = links_msg % links
-
- return links_msg
+ msgstr = _(msgid)
+ return msgstr
- def _parse_text(self, msg):
+ def _parse_text(self, msg, core_obj):
"""Parse the text part of a message.
Split the message in words and look for patterns for locale,
operating system and built-in pluggable transport info.
+ :param: msg (string) the message received.
+ :param: core_obj (object) the object of gettor core module.
+
+ :return: request (list) 4-tuple with locale, os, type of request
+ and pt info.
+
"""
- self.logger.debug("Starting text parsing")
+ self.log.debug("Starting text parsing")
# core knows what OS are supported
- supported_os = self.core.get_supported_os()
- supported_locales = self.core.get_supported_locales()
+ supported_os = core_obj.get_supported_os()
+ supported_lc = core_obj.get_supported_lc()
# default values
- request = {}
- request['locale'] = 'en'
- request['os'] = 'windows'
- request['type'] = 'help'
- request['pt'] = False
- found_locale = False
+ req = {}
+ req['lc'] = 'en'
+ req['os'] = ''
+ req['type'] = 'help'
+ req['pt'] = False
+ found_lc = False
found_os = False
+ found_help = False
# analyze every word
# request shouldn't be more than 10 words long, so there should
# be a limit for the amount of words
for word in msg.split(' '):
+ # check for help request
+ if not found_os and re.match('help', word, re.IGNORECASE):
+ self.log.info("Request for help found")
+ req['type'] = 'help'
+ found_help = True
# look for locale, os and pt
- if not found_locale:
- for locale in supported_locales:
- if re.match(locale, word, re.IGNORECASE):
- found_locale = True
- request['locale'] = locale
- self.logger.debug("Found locale: %s" % locale)
- if not found_os:
- for operating_system in supported_os:
- if re.match(operating_system, word, re.IGNORECASE):
+ if not found_lc:
+ for lc in supported_lc:
+ if re.match(lc, word, re.IGNORECASE):
+ found_lc = True
+ req['lc'] = lc
+ self.log.debug("Found locale: %s" % lc)
+ if not found_os and not found_help:
+ for os in supported_os:
+ if re.match(os, word, re.IGNORECASE):
found_os = True
- request['os'] = operating_system
- request['type'] = 'links'
- self.logger.debug("Found OS: %s" % operating_system)
- if re.match("obfs|plugabble transport|pt", word,
- re.IGNORECASE):
- request['pt'] = True
- self.logger.debug("Found PT request")
+ req['os'] = os
+ req['type'] = 'links'
+ self.log.debug("Found OS: %s" % os)
+ if re.match("obfs|plugabble|transport|pt", word, re.IGNORECASE):
+ req['pt'] = True
+ self.log.debug("Found PT request")
- return request
+ return req
def parse_request(self, account, msg):
"""Process the request received.
@@ -342,36 +304,75 @@ class XMPP(object):
Check if the user is not blacklisted and then check the body of
the message to find out what is asking.
- Params: account - the account that did the request.
- msg - the body of the message sent to us.
+ :param: account (string) the account that did the request.
+ :param: msg (string) the body of the message sent to us.
- """
- try:
- self._check_blacklist(str(account))
- except BlacklistError as e:
- return None
-
- # let's try to guess what the user is asking
- request = self._parse_text(str(msg))
-
- if request['type'] == 'help':
- return_msg = self._get_help_msg(request['locale'])
- elif request['type'] == 'links':
- try:
- links = self.core.get_links("XMPP", request['os'],
- request['locale'])
-
- return_msg = self._get_links_msg(request['locale'],
- request['os'], request['pt'],
- links)
+ :return: (string/None) the message to be sent to the user via the
+ bot, or None if the user is blacklisted.
- except (core.ConfigurationError, core.InternalError) as e:
- return_msg = self._get_internal_error_msg(request['locale'])
-
- except core.UnsupportedLocaleError as e:
- self.core._get_unsupported_locale_msg(request['locale'])
-
- except core.UnsupportedOSError as e:
- self.core._get_unsupported_os_msg(request['locale'])
+ """
+ bogus_request = False
+ reply = ''
+ logfile = ''
+ status = ''
+ req = None
+ core_obj = core.Core(self.core_cfg)
- return return_msg
+ try:
+ if self._is_blacklisted(str(account)):
+ status = 'blacklisted'
+ bogus_request = True
+
+ if not bogus_request:
+ self.log.debug("Request seems legit, let's parse it")
+ # let's try to guess what the user is asking
+ req = self._parse_text(str(msg), core_obj)
+
+ if req['type'] == 'help':
+ status = 'success'
+ self.log.debug("Got a help request")
+ reply = self._get_msg('help', req['lc'])
+ elif req['type'] == 'links':
+ self.log.debug("Got a links request")
+ try:
+ links = core_obj.get_links("XMPP", req['os'],
+ req['lc'])
+ # did the user asked for PT stuff?
+ if req['pt']:
+ self.log.debug("Also asked for PT info")
+ reply = self._get_msg('links_pt', req['lc'])
+ reply = reply % (req['os'], req['lc'], links)
+ else:
+ reply = self._get_msg('links', req['lc'])
+ reply = reply % (req['os'], req['lc'], links)
+
+ status = 'success'
+ except (core.ConfigurationError, core.InternalError) as e:
+ # if core failes, send the user an error message, but
+ # keep going
+ status = 'core_error'
+ self.log.debug("Something went wrong with Core")
+ reply = self._get_msg('internal_error', req['lc'])
+
+ # if the user asked for an unsupported locale, warn him
+ # and keep going
+ except core.UnsupportedLocaleError as e:
+ status = 'unsupported_lc'
+ self.log.debug("User asked for unsupported locale")
+ reply = self._get_msg('unsupported_lc', req['lc'])
+ finally:
+ # keep stats
+ self.log.debug("Request processed, saving stats.")
+
+ if req:
+ core_obj.add_request_to_db('XMPP',
+ req['type'], req['os'],
+ req['lc'], req['pt'],
+ status, logfile)
+ else:
+ # invalid request, so no info about it
+ # logfiles were created for this
+ core_obj.add_request_to_db('XMPP', '', '', '', '',
+ status, logfile)
+
+ return reply
diff --git a/src/i18n/en/LC_MESSAGES/en.mo b/src/i18n/en/LC_MESSAGES/en.mo
index 3499e93..e686fcb 100644
Binary files a/src/i18n/en/LC_MESSAGES/en.mo and b/src/i18n/en/LC_MESSAGES/en.mo differ
diff --git a/src/i18n/en/LC_MESSAGES/en.po b/src/i18n/en/LC_MESSAGES/en.po
index 6251c8c..6f78da2 100644
--- a/src/i18n/en/LC_MESSAGES/en.po
+++ b/src/i18n/en/LC_MESSAGES/en.po
@@ -35,11 +35,6 @@ Tor Browser Bundle:\n\
===\n\
%s\n\
===\n\
-Pluggable Transport Bundle:\n\
-===\n\
-%s\n\
-Tip: If you are in Iran, China, or under heavy censorship, you need the\n\
-Pluggable Transport Bundle.\n\
\n\
===\n\
Support:\n\
@@ -64,11 +59,7 @@ Tor Browser Bundle:\n\
===\n\
%s\n\
===\n\
-Pluggable Transport Bundle:\n\
-===\n\
-%s\n\
-Tip: If you are in Iran, China, or under heavy censorship, you need the\n\
-Pluggable Transport Bundle.\n\
+Info about pluggable transports.\n\
\n\
===\n\
Support:\n\
diff --git a/src/i18n/es/LC_MESSAGES/es.mo b/src/i18n/es/LC_MESSAGES/es.mo
index 941ea5e..c21fe7b 100644
Binary files a/src/i18n/es/LC_MESSAGES/es.mo and b/src/i18n/es/LC_MESSAGES/es.mo differ
diff --git a/src/i18n/es/LC_MESSAGES/es.po b/src/i18n/es/LC_MESSAGES/es.po
index 9992a34..bc4b5a8 100644
--- a/src/i18n/es/LC_MESSAGES/es.po
+++ b/src/i18n/es/LC_MESSAGES/es.po
@@ -31,15 +31,8 @@ msgstr """Gracias por tu petición para %s-%s.\n\
Aquà están los links de descarga:\n\
\n\
===\n\
-Tor Browser Bundle:\n\
-===\n\
%s\n\
===\n\
-Pluggable Transport Bundle:\n\
-===\n\
-%s\n\
-Tip: Si estás en Iran, China, o bajo fuerte censura, necesitas el\n\
-Pluggable Transport Bundle.\n\
\n\
===\n\
Soporte:\n\
@@ -61,15 +54,9 @@ msgstr """Gracias por tu petición para %s-%s.\n\
Aquà están los links de descarga:\n\
\n\
===\n\
-Tor Browser Bundle:\n\
-===\n\
%s\n\
===\n\
-Pluggable Transport Bundle:\n\
-===\n\
-%s\n\
-Tip: Si estás en Iran, China, o bajo fuerte censura, necesitas el\n\
-Pluggable Transport Bundle.\n\
+Tip: Información sobre pluggable transports.\n\
\n\
===\n\
Soporte:\n\
diff --git a/src/providers/dropbox.links b/src/providers/dropbox.links
index c1e7f56..7cccca0 100644
--- a/src/providers/dropbox.links
+++ b/src/providers/dropbox.links
@@ -9,6 +9,9 @@ en = https://db.tt/ZAzgGm4Q https://db.tt/dUI80d2K 98ea6e4f216f2fb4b69fff9b3a448
es = https://db.tt/jp3Ytnzh https://db.tt/MDfUTb04 98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4
[windows]
+en = https://db.tt/ZAzgGm4Q https://db.tt/dUI80d2K 98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4
+es = https://db.tt/jp3Ytnzh https://db.tt/MDfUTb04 98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4
[osx]
-
+en = https://db.tt/ZAzgGm4Q https://db.tt/dUI80d2K 98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4
+es = https://db.tt/jp3Ytnzh https://db.tt/MDfUTb04 98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4
diff --git a/src/scripts/blacklist.py b/src/scripts/blacklist.py
new file mode 100644
index 0000000..bf812e8
--- /dev/null
+++ b/src/scripts/blacklist.py
@@ -0,0 +1,111 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# This file is part of GetTor, a Tor Browser Bundle distribution system.
+#
+# :authors: Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+import sys
+import time
+import sqlite3
+import argparse
+
+
+def main():
+ """Script for managing blacklisting of users.
+
+ See argparse usage for more details.
+
+ """
+ parser = argparse.ArgumentParser(description='Utility for GetTor'
+ ' blacklisting')
+ parser.add_argument('database', metavar='database.db', type=str,
+ help='the database file')
+ parser.add_argument('-u', '--user', default=None,
+ help='filter by user hash')
+ parser.add_argument('-s', '--service', default=None,
+ help='filter by service')
+ parser.add_argument('-b', '--blocked', default=None,
+ help='filter by blocked users')
+ parser.add_argument('-a', '--add', default=None, nargs=3,
+ metavar=('USER', 'SERVICE', 'BLOCKED'),
+ help='add user')
+ parser.add_argument('-c', '--clean', default=None, const='c', nargs='?',
+ metavar='user hash',
+ help='clean table (delete expired blacklistings)')
+ parser.add_argument('-r', '--requests', default=None,
+ help='number of requests; everyone with number of'
+ ' requests greather than this will be cleaned up')
+
+ args = parser.parse_args()
+ query = ''
+ con = sqlite3.connect(args.database)
+
+ if args.add:
+ # add new entry, useful for adding users permanently blocked
+ query = "INSERT INTO users VALUES('%s', '%s', 1, %s, %s)"\
+ % (args.add[0], args.add[1], args.add[2], time.time())
+ with con:
+ cur = con.cursor()
+ cur.execute(query)
+ print "Query execute successfully"
+ elif args.clean:
+ if args.clean == 'c':
+ if args.requests:
+ # delete by number of times
+ query = "DELETE FROM users WHERE times > %s" % args.requests
+ with con:
+ cur = con.cursor()
+ cur.execute(query)
+ print "Query executed successfully."
+ else:
+ sys.exit("Number of requests missing. See --help.")
+ else:
+ # delete by id
+ query = "DELETE FROM users WHERE id='%s'" % args.clean
+ with con:
+ cur = con.cursor()
+ cur.execute(query)
+ print "Query execute succcessfully."
+ else:
+ query = "SELECT * FROM users"
+ has_where = False
+ # filter
+ if args.service:
+ query = "%s %s" % (query, "WHERE service='%s'" % args.service)
+ has_where = True
+ if args.user:
+ if has_where:
+ query = "%s %s" % (query, "AND id='%s'" % args.user)
+ else:
+ query = "%s %s" % (query, "WHERE id='%s'" % args.user)
+ has_where = True
+ if args.blocked:
+ if has_where:
+ query = "%s %s" % (query, "AND blocked=%s" % args.blocked)
+ has_where = True
+ else:
+ query = "%s %s" % (query, "WHERE blocked=%s" % args.blocked)
+
+ with con:
+ cur = con.cursor()
+ cur.execute(query)
+ rows = cur.fetchall()
+ # show it nice
+ print "\nNumber of results: %s\n" % len(rows)
+ cns = [cn[0] for cn in cur.description]
+ print "%-70s %-10s %-10s %-10s %-s" % (cns[0], cns[1], cns[2],
+ cns[3], cns[4])
+
+ for row in rows:
+ print "%-70s %-10s %-10s %-10s %s" % (row[0], row[1], row[2],
+ row[3], row[4])
+
+if __name__ == "__main__":
+ main()
diff --git a/src/scripts/create_db.py b/src/scripts/create_db.py
new file mode 100644
index 0000000..057ae82
--- /dev/null
+++ b/src/scripts/create_db.py
@@ -0,0 +1,54 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# This file is part of GetTor, a Tor Browser Bundle distribution system.
+#
+# :authors: Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+import os
+import sqlite3
+import argparse
+
+
+def main():
+ """Create/delete GetTor database for managing stats and blacklisting.
+
+ Database file (.db) must be empty. If it doesn't exist, it will be
+ created. See argparse usage for more details.
+
+ """
+ parser = argparse.ArgumentParser(description='Utility for GetTor'
+ ' database')
+ parser.add_argument('-c', '--create', default=None,
+ metavar='path_to_database.db',
+ help='create database')
+ parser.add_argument('-d', '--delete', default=None,
+ metavar='path_to_database.db',
+ help='delete database')
+
+ args = parser.parse_args()
+ if args.create:
+ con = sqlite3.connect(args.create)
+ with con:
+ cur = con.cursor()
+ # table for handling users (i.e. blacklist)
+ cur.execute("CREATE TABLE users(id TEXT, service TEXT, times INT,"
+ " blocked INT, last_request TEXT)")
+ # table for stats
+ cur.execute("CREATE TABLE requests(service TEXT, type TEXT,"
+ " os TEXT, lc TEXT, pt INT, year INT, month INT,"
+ " day INT, status TEXT, logfile TEXT)")
+ print "Database %s created" % os.path.abspath(args.create)
+ elif args.delete:
+ os.remove(os.path.abspath(args.delete))
+ print "Database %s deleted" % os.path.abspath(args.delete)
+ else:
+ print "See --help for details on usage."
+if __name__ == "__main__":
+ main()
diff --git a/src/scripts/stats.py b/src/scripts/stats.py
new file mode 100644
index 0000000..f1d782d
--- /dev/null
+++ b/src/scripts/stats.py
@@ -0,0 +1,123 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# This file is part of GetTor, a Tor Browser Bundle distribution system.
+#
+# :authors: Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+
+import sqlite3
+import argparse
+
+
+def main():
+ """Script for showing stats.
+
+ See argparse usage for more details.
+
+ """
+ parser = argparse.ArgumentParser(description='Utility for GetTor stats')
+ parser.add_argument('database', metavar='database.db', type=str,
+ help='the database file')
+ parser.add_argument('-s', '--service', default=None,
+ help='filter by service')
+ parser.add_argument('-t', '--type', default=None,
+ help='filter by type of request')
+ parser.add_argument('-o', '--os', default=None,
+ help='filter by OS')
+ parser.add_argument('-l', '--lc', default=None,
+ help='filter by locale')
+ parser.add_argument('-p', '--pt', default=None,
+ help='filter by PT requests')
+ parser.add_argument('-y', '--year', default=None,
+ help='filter by year')
+ parser.add_argument('-m', '--month', default=None,
+ help='filter by month')
+ parser.add_argument('-d', '--day', default=None,
+ help='filter by day')
+ parser.add_argument('-u', '--status', default=None,
+ help='filter by status of the request')
+
+ args = parser.parse_args()
+ query = 'SELECT * FROM requests'
+ has_where = False
+
+ # we build the query piece by piece
+ if args.service:
+ query = "%s %s" % (query, "WHERE service = '%s'" % args.service)
+ has_where = True
+ if args.type:
+ if has_where:
+ query = "%s %s" % (query, "AND type = '%s'" % args.type)
+ else:
+ query = "%s %s" % (query, "WHERE type = '%s'" % args.type)
+ has_where = True
+ if args.os:
+ if has_where:
+ query = "%s %s" % (query, "AND os = '%s'" % args.os)
+ else:
+ query = "%s %s" % (query, "WHERE os = '%s'" % args.os)
+ has_where = True
+ if args.lc:
+ if has_where:
+ query = "%s %s" % (query, "AND lc = '%s'" % args.lc)
+ else:
+ query = "%s %s" % (query, "WHERE lc = '%s'" % args.lc)
+ has_where = True
+ if args.pt:
+ if has_where:
+ query = "%s %s" % (query, "AND pt = %s" % args.pt)
+ else:
+ query = "%s %s" % (query, "WHERE pt = %s" % args.pt)
+ has_where = True
+ if args.year:
+ if has_where:
+ query = "%s %s" % (query, "AND year = %s" % args.year)
+ else:
+ query = "%s %s" % (query, "WHERE year = %s" % args.year)
+ has_where = True
+ if args.month:
+ if has_where:
+ query = "%s %s" % (query, "AND month = %s" % args.month)
+ else:
+ query = "%s %s" % (query, "WHERE month = %s" % args.month)
+ has_where = True
+ if args.day:
+ if has_where:
+ query = "%s %s" % (query, "AND day = %s" % args.day)
+ else:
+ query = "%s %s" % (query, "WHERE day = %s" % args.day)
+ has_where = True
+ if args.status:
+ if has_where:
+ query = "%s %s" % (query, "AND status = '%s'" % args.status)
+ else:
+ query = "%s %s" % (query, "WHERE status = '%s'" % args.status)
+ has_where = True
+
+ con = sqlite3.connect(args.database)
+
+ with con:
+ cur = con.cursor()
+ cur.execute(query)
+ rows = cur.fetchall()
+ # show it nice
+ print "\nNumber of results: %s\n" % len(rows)
+ cns = [cn[0] for cn in cur.description]
+ print "%-10s %-10s %-10s %-10s %-10s %-10s %-10s %-10s"\
+ " %-15s %s"\ % (cns[0], cns[1], cns[2], cns[3], cns[4], cns[5],
+ cns[6], cns[7], cns[8], cns[9])
+
+ for row in rows:
+ print "%-10s %-10s %-10s %-10s %-10s %-10s %-10s %-10s"\
+ " %-15s %s" % (row[0], row[1], row[2], row[3], row[4],
+ row[5], row[6], row[7], row[8], row[9])
+
+if __name__ == "__main__":
+ main()
diff --git a/src/smtp.cfg b/src/smtp.cfg
index af9eb79..ddbf5fb 100644
--- a/src/smtp.cfg
+++ b/src/smtp.cfg
@@ -1,7 +1,15 @@
[general]
-basedir: smtp/
-delay: False
-our_addr: gettor at torproject.org
+basedir: /path/to/gettor/smtp
+our_domain: torproject.org
+core_cfg: /path/to/core.cfg
+
+[blacklist]
+cfg: /path/to/blacklist.cfg
+max_requests: 3
+wait_time: 20
+
+[i18n]
+dir: i18n/
[log]
level: DEBUG
diff --git a/src/xmpp.cfg b/src/xmpp.cfg
index a78ef88..20d8227 100644
--- a/src/xmpp.cfg
+++ b/src/xmpp.cfg
@@ -1,9 +1,18 @@
[account]
-user:
+user: account at domain
password:
[general]
-basedir: xmpp/
+basedir: /path/to/gettor/xmpp/
+core_cfg: /path/to/core.cfg
+
+[blacklist]
+cfg: /path/to/blacklist.cfg
+max_requests: 3
+wait_time: 20
+
+[i18n]
+dir: i18n/
[log]
level: DEBUG
diff --git a/src/xmpp/i18n/en/LC_MESSAGES/en.po b/src/xmpp/i18n/en/LC_MESSAGES/en.po
index 428b654..d242625 100644
--- a/src/xmpp/i18n/en/LC_MESSAGES/en.po
+++ b/src/xmpp/i18n/en/LC_MESSAGES/en.po
@@ -1,26 +1,22 @@
domain "en"
#: Links
-msgid "links_msg"
-msgstr "Links:\n %s"
+msgid "links"
+msgstr "Links %s-%s:\n %s"
#: Links
-msgid "links_pt_msg"
-msgstr "Links-PT:\n %s"
+msgid "links_pt"
+msgstr "Links-PT for %s-%s:\n %s"
#: Help
-msgid "help_msg"
-msgstr "help"
-
-#: Unsupported OS
-msgid "unsupported_os_msg"
-msgstr "Unsupported OS"
+msgid "help"
+msgstr "*help*"
#: Unsupported locale
-msgid "unsupported_locale_msg"
+msgid "unsupported_lc"
msgstr "Unsupported locale"
#: Internal error
-msgid "internal_error_msg"
+msgid "internal_error"
msgstr "Internal error"
diff --git a/src/xmpp/i18n/es/LC_MESSAGES/es.po b/src/xmpp/i18n/es/LC_MESSAGES/es.po
index f5cb21b..b6caac5 100644
--- a/src/xmpp/i18n/es/LC_MESSAGES/es.po
+++ b/src/xmpp/i18n/es/LC_MESSAGES/es.po
@@ -1,26 +1,22 @@
domain "es"
#: Links
-msgid "links_msg"
-msgstr "Links:\n %s"
+msgid "links"
+msgstr "Enlaces para %s-%s:\n %s"
#: Links
-msgid "links_pt_msg"
-msgstr "Links-PT:\n %s"
+msgid "links_pt"
+msgstr "Enlaces-PT para %s-%s:\n %s"
#: Help
-msgid "help_msg"
+msgid "help"
msgstr "ayuda"
-#: Unsupported OS
-msgid "unsupported_os_msg"
-msgstr "SO no soportado"
-
#: Unsupported locale
-msgid "unsupported_locale_msg"
+msgid "unsupported_lc"
msgstr "Locale no soportado"
#: Internal error
-msgid "internal_error_msg"
+msgid "internal_error"
msgstr "Error interno"
More information about the tor-commits
mailing list