[tor-commits] [gettor/master] Progress for status update August 1st. New code structure and XMPP module.

commit 143129d8381b40c373de8537dfd1ec9e11296f05 Author: ilv ilv@users.noreply.github.com Date: Fri Aug 1 20:26:14 2014 -0400

Progress for status update August 1st. New code structure and XMPP module. --- src/core_demo.py | 16 +- src/dropbox.py | 139 +++++++----- src/gettor/core.py | 402 +++++++++++++++++++++++++++++++++ src/gettor/smtp.py | 612 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/gettor/utils.py | 87 ++++++++ src/gettor/xmpp.py | 378 +++++++++++++++++++++++++++++++ src/smtp_demo.py | 14 +- src/xmpp.cfg | 11 + src/xmpp_demo.py | 7 + 9 files changed, 1598 insertions(+), 68 deletions(-)

diff --git a/src/core_demo.py b/src/core_demo.py index c783517..6298f70 100644 --- a/src/core_demo.py +++ b/src/core_demo.py @@ -1,15 +1,19 @@ #!/usr/bin/python # -# Dummy script to test GetTore's Core module progress +# Dummy script to test GetTore's Core module #

-import gettor +import gettor.core

try: - core = gettor.Core('gettor.cfg') + core = gettor.core.Core() links = core.get_links('dummy service', 'linux', 'es') print links -except ValueError as e: - print "Value error: " + str(e) -except RuntimeError as e: +except gettor.core.ConfigurationError as e: + print "Misconfiguration: " + str(e) +except gettor.core.UnsupportedOSError as e: + print "Unsupported OS: " + str(e) +except gettor.core.UnsupportedLocaleError as e: + print "Unsupported Locale: " + str(e) +except gettor.core.InternalError as e: print "Internal error: " + str(e) diff --git a/src/dropbox.py b/src/dropbox.py index 08084ef..1be1ae8 100644 --- a/src/dropbox.py +++ b/src/dropbox.py @@ -1,9 +1,13 @@ -#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# This file is part of GetTor, a Tor Browser Bundle distribution system. +# import re import os import gnupg +import hashlib import dropbox -import gettor +import gettor.core

def valid_bundle_format(file): @@ -24,12 +28,12 @@ def valid_bundle_format(file):

def get_bundle_info(file): - """ - Get the operating system and locale from a bundle string. + """Get the operating system and locale from a bundle string. + + it raises a ValueError if the bundle doesn't have a valid format + (although you should probably call valid_bundle_format first). + It returns the pair of strings operating system, locale.

- it raises a ValueError if the bundle doesn't have a valid format - (although you should probably call valid_bundle_format first). - It returns the pair of strings operating system, locale. """ m = re.search( 'tor-browser-(\w+)\d\d-\d.\d.\d_(\w\w)-\w+.tar.xz', @@ -41,17 +45,33 @@ def get_bundle_info(file): else: raise ValueError("Bundle invalid format %s" % file)

+def get_file_sha1(file): + """Get the sha1 of a file. + + Desc. + + """ + + # as seen on the internet + BLOCKSIZE = 65536 + hasher = hashlib.sha1() + with open(file, 'rb') as afile: + buf = afile.read(BLOCKSIZE) + while len(buf) > 0: + hasher.update(buf) + buf = afile.read(BLOCKSIZE) + return hasher.hexdigest()

def upload_files(basedir, client): - """ - Upload files from 'basedir' to Dropbox. + """Upload files from 'basedir' to Dropbox. + + It looks for files ending with 'tar.xz' inside 'basedir'. It + raises ValueError in case the given file doesn't have a .asc file. + It raises UploadError if something goes wrong while uploading the + files to Dropbox. All files are uploaded to '/'.

- It looks for files ending with 'tar.xz' inside 'basedir'. It - raises ValueError in case the given file doesn't have a .asc file. - It raises UploadError if something goes wrong while uploading the - files to Dropbox. All files are uploaded to '/'. + Returns a list with the names of the uploaded files.

- Returns a list with the names of the uploaded files. """ files = []

@@ -87,46 +107,53 @@ def upload_files(basedir, client):

return files

-# Test app for now -# TO DO: use config file -app_key = '' -app_secret = '' -access_token = '' -upload_dir = 'upload/' -tbb_key = 'tbb-key.asc' - -client = dropbox.client.DropboxClient(access_token) - -# Import key that signed the packages and get fingerprint -gpg = gnupg.GPG() -key_data = open(tbb_key).read() -import_result = gpg.import_keys(key_data) -fingerprint = import_result.results[0]['fingerprint'] -# Make groups of four characters to make fingerprint more readable -# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J -readable = ' '.join(fingerprint[i:i+4] for i in xrange(0, len(fingerprint), 4)) - -try: - uploaded_files = upload_files(upload_dir, client) - core = gettor.Core('gettor.cfg') - # This erases the old links file - core.create_links_file('Dropbox', readable) - - for file in uploaded_files: - # build file names - asc = file + '.asc' - abs_file = os.path.abspath(os.path.join(upload_dir, file)) - abs_asc = os.path.abspath(os.path.join(upload_dir, asc)) - - # build links - link_file = client.share(file) - link_asc = client.share(asc) - link = link_file[u'url'] + ' ' + link_asc[u'url'] +if __name__ == '__main__': + # to-do: use config file + app_key = '' + app_secret = '' + access_token = '' + upload_dir = 'upload/' + + # important: this must be the key that signed the packages + tbb_key = 'tbb-key.asc' + + client = dropbox.client.DropboxClient(access_token) + + # import key fingerprint + gpg = gnupg.GPG() + key_data = open(tbb_key).read() + import_result = gpg.import_keys(key_data) + fp = import_result.results[0]['fingerprint'] + + # make groups of four characters to make fingerprint more readable + # e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J + readable = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4)) + + try: + uploaded_files = upload_files(upload_dir, client) + # use default config + core = gettor.core.Core()

- # add links - operating_system, locale = get_bundle_info(file) - core.add_link('Dropbox', operating_system, locale, link) -except (ValueError, RuntimeError) as e: - print str(e) -except dropbox.rest.ErrorResponse as e: - print str(e) + # erase old links + core.create_links_file('Dropbox', readable) + + for file in uploaded_files: + # build file names + asc = file + '.asc' + abs_file = os.path.abspath(os.path.join(upload_dir, file)) + abs_asc = os.path.abspath(os.path.join(upload_dir, asc)) + + sha1_file = get_file_sha1(abs_file) + + # build links + link_file = client.share(file) + link_asc = client.share(asc) + link = link_file[u'url'] + ' ' + link_asc[u'url'] + ' ' + sha1_file + + # add links + operating_system, locale = get_bundle_info(file) + core.add_link('Dropbox', operating_system, locale, link) + except (ValueError, RuntimeError) as e: + print str(e) + except dropbox.rest.ErrorResponse as e: + print str(e) diff --git a/src/gettor/core.py b/src/gettor/core.py new file mode 100644 index 0000000..407e080 --- /dev/null +++ b/src/gettor/core.py @@ -0,0 +1,402 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor, a Tor Browser Bundle distribution system. +# + +import os +import re +import inspect +import logging +import tempfile +import ConfigParser + +import utils + +"""Core module for getting links from providers.""" + + +class ConfigurationError(Exception): + pass + + +class UnsupportedOSError(Exception): + pass + + +class UnsupportedLocaleError(Exception): + pass + + +class LinkFormatError(Exception): + pass + + +class LinkFileError(Exception): + pass + + +class InternalError(Exception): + pass + + +class Core(object): + """Get links from providers and deliver them to other modules. + + Public methods: + + get_links(): Get the links for the OS and locale requested. + create_links_file(): Create a file to store links of a provider. + add_link(): Add a link to a links file of a provider. + get_supported_os(): Get a list of supported operating systems. + get_supported_locale(): Get a list of supported locales. + + Exceptions: + + UnsupportedOSError: Request for an unsupported operating system. + UnsupportedLocaleError: Request for an unsupported locale. + ConfigurationError: Something's misconfigured. + LinkFormatError: The link added doesn't seem legit. + LinkFileError: Error related to the links file of a provider. + InternalError: Something went wrong internally. + + """ + + def __init__(self, cfg=None): + """Create a new core object by reading a configuration file. + + Raises: ConfigurationError if the configuration file doesn't exists + or if something goes wrong while reading options from it. + + Params: cfg - path of the configuration file. + + """ + # Define a set of default values + DEFAULT_CONFIG_FILE = 'core.cfg' + + logging.basicConfig(format='[%(levelname)s] %(asctime)s - %(message)s', + datefmt="%Y-%m-%d %H:%M:%S") + logger = logging.getLogger(__name__) + config = ConfigParser.ConfigParser() + + if cfg is None or not os.path.isfile(cfg): + cfg = DEFAULT_CONFIG_FILE + logger.info("Using default configuration") + + logger.info("Reading configuration file %s" % cfg) + config.read(cfg) + + try: + self.basedir = config.get('general', 'basedir') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'basedir' from 'general' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.linksdir = config.get('links', 'dir') + self.linksdir = os.path.join(self.basedir, self.linksdir) + except ConfigParser.Error as e: + logger.warning("Couldn't read 'links' from 'dir' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.supported_locales = config.get('links', 'locales') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'locales' from 'links' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.supported_os = config.get('links', 'os') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'os' from 'links' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.loglevel = config.get('log', 'level') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'level' from 'log' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.logdir = config.get('log', 'dir') + self.logdir = os.path.join(self.basedir, self.logdir) + except ConfigParser.Error as e: + logger.warning("Couldn't read 'dir' from 'log' %s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + # Keep log levels separated + self.logger = utils.filter_logging(logger, self.logdir, self.loglevel) + # self.logger.setLevel(logging.getLevelName(self.loglevel)) + self.logger.info('Redirecting logging to %s' % self.logdir) + + # Stop logging on stdout from now on + self.logger.propagate = False + self.logger.debug("New core object created") + + def get_links(self, service, operating_system, locale): + """Get links for OS in locale. + + This method should be called from the services modules of + GetTor (e.g. SMTP). To make it easy we let the module calling us + specify the name of the service (for stats purpose). + + Raises: UnsupportedOSError: if the operating system is not supported. + UnsupportedLocaleError: if the locale is not supported. + InternalError: if something goes wrong while internally. + + Params: service - the name of the service trying to get the links. + operating_system - the name of the operating system. + locale - two-character string representing the locale. + + Returns: String with links. + + """ + + # Which module called us and what was asking for? + self.logger.info("%s did a request for %s, %s." % + (service, operating_system, locale)) + + if locale not in self.supported_locales: + self.logger.warning("Request for unsupported locale: %s" % locale) + raise UnsupportedLocaleError("Locale %s not supported at the " + "moment" % locale) + + if operating_system not in self.supported_os: + self.logger.warning("Request for unsupported operating system: %s" + % operating_system) + raise UnsupportedOSError("Operating system %s not supported at the" + "moment" % operating_system) + + # This could change in the future, let's leave it isolated. + links = self._get_links(operating_system, locale) + + if links is None: + self.logger.error("Couldn't get the links", exc_info=True) + raise InternalError("Something went wrong internally. See logs for" + " detailed info.") + + self.logger.info("Returning the links") + return links + + def _get_links(self, operating_system, locale): + """Internal method to get the links. + + Looks for the links inside each provider file. This should only be + called from get_links() method. + + Returns: String with the links on success. + None on failure. + + Params: operating_system - name of the operating system + locale: two-character string representing the locale. + + """ + + # Read the links files using ConfigParser + # See the README for more details on the format used + links = [] + + # Look for files ending with .links + p = re.compile('.*.links$') + + for name in os.listdir(self.linksdir): + path = os.path.abspath(os.path.join(self.linksdir, name)) + if os.path.isfile(path) and p.match(path): + links.append(path) + + # Let's create a dictionary linking each provider with the links + # found for operating_system and locale. This way makes it easy + # to check if no links were found + providers = {} + + self.logger.info("Reading links from providers directory") + for name in links: + self.logger.debug("Reading %s" % name) + # We're reading files listed on linksdir, so they must exist! + config = ConfigParser.ConfigParser() + config.read(name) + + try: + pname = config.get('provider', 'name') + except ConfigParser.Error as e: + self.logger.warning("Couldn't get 'name' from 'provider' (%s)" + % name) + raise InternalError("Error while reading %s links file. See " + "log file" % name) + + self.logger.debug("Checking if %s has links for %s in %s" % + (pname, operating_system, locale)) + + try: + providers[pname] = config.get(operating_system, locale) + except ConfigParser.Error as e: + self.logger.warning("Couldn't get %s from %s (%s)" % + (locale, operating_system, name)) + raise InternalError("Error while reading %s links file. See " + "log file" % name) + + # Each provider must have a fingerprint of the key used to + # sign the uploaded packages + try: + self.logger.debug("Trying to get fingerprint from %s", pname) + fingerprint = config.get('key', 'fingerprint') + providers[pname] = providers[pname] + "\nFingerprint: " + providers[pname] = providers[pname] + fingerprint + self.logger.debug("Fingerprint added %s", fingerprint) + except ConfigParser.Error as e: + self.logger.warning("Couldn't get 'fingerprint' from 'key' " + "(%s)" % name) + raise InternalError("Error while reading %s links file. See " + "log file" % name) + + # Create the final links list with all providers + all_links = [] + + self.logger.debug("Joining all links found for %s in %s" % + (operating_system, locale)) + for key in providers.keys(): + all_links.append( + "\n%s\n%s\n" % (key, ''.join(providers[key])) + ) + + if all_links: + return "".join(all_links) + else: + self.logger.warning("Trying to get supported os and locales, but" + " no links were found") + return None + + def get_supported_os(self): + """Public method to get the list of supported operating systems. + + Returns: List of strings. + + """ + return self.supported_os.split(',') + + def get_supported_locales(self): + """Public method to get the list of supported locales. + + Returns: List of strings. + + """ + return self.supported_locales.split(',') + + def create_links_file(self, provider, fingerprint): + """Public method to create a links file for a provider. + + This should be used by all providers since it writes the links + file with the proper format. It backs up the old links file + (if exists) and creates a new one. + + Params: provider - provider's name (links file will use this + name in lower case). + + fingerprint: fingerprint of the key that signed the packages + to be uploaded to the provider. + + """ + linksfile = os.path.join(self.linksdir, provider.lower() + '.links') + linksfile_backup = "" + self.logger.info("Request to create new %s" % linksfile) + + if os.path.isfile(linksfile): + # Backup the old file in case something fails + linksfile_backup = linksfile + '.backup' + self.logger.info("Backing up %s to %s" + % (linksfile, linksfile_backup)) + os.rename(linksfile, linksfile_backup) + + try: + # This creates an empty links file (with no links) + content = ConfigParser.RawConfigParser() + content.add_section('provider') + content.set('provider', 'name', provider) + content.add_section('key') + content.set('key', 'fingerprint', fingerprint) + content.add_section('linux') + content.add_section('windows') + content.add_section('osx') + with open(linksfile, 'w+') as f: + content.write(f) + self.logger.info("New %s created" % linksfile) + except Exception as e: + if linksfile_backup: + os.rename(linksfile_backup, linksfile) + raise LinkFileError("Error while trying to create new links file.") + + def add_link(self, provider, operating_system, locale, link): + """Public method to add a link to a provider's links file. + + Use ConfigParser to add a link into the operating_system + section, under the locale option. It checks for valid format; + the provider's script should use the right format (see design). + + Raises: UnsupportedOSError: if the operating system is not supported. + UnsupportedLocaleError: if the locale is not supported. + LinkFileError: if there is no links file for the provider. + LinkFormatError: if the link format doesn't seem legit. + InternalError: if the links file doesn't have a section for the + OS requested. This *shouldn't* happen because + it means the file wasn't created correctly. + + Params: provider - name of the provider. + operating_system - name of the operating system. + locale - two-character string representing the locale. + link - string to be added. The format should be as follows: + + https://pkg_url https://asc_url + + where pkg_url is the url for the bundle and asc_url is the + url for the asc of the bundle. + + """ + linksfile = os.path.join(self.linksdir, provider.lower() + '.links') + + # Don't try to add unsupported stuff + if locale not in self.supported_locales: + self.logger.warning("Trying to add link for unsupported locale: %s" + % locale) + raise UnsupportedLocaleError("Locale %s not supported at the " + "moment" % locale) + + if operating_system not in self.supported_os: + self.logger.warning("Trying to add link for unsupported operating " + "system: %s" % operating_system) + raise UnsupportedOSError("Operating system %s not supported at the" + " moment" % operating_system) + + # Check if the link has a legit format + # e.g. https://db.tt/JjfUTb04 https://db.tt/MEfUTb04 + p = re.compile('^https://.+%5Cshttps://.+$') + + if not p.match(link): + self.logger.warning("Trying to add an invalid link: %s" + % link) + raise LinkFormatError("Link '%s' doesn't seem to have a valid " + "format" % link) + + if os.path.isfile(linksfile): + content = ConfigParser.RawConfigParser() + content.readfp(open(linksfile)) + # Check if exists and entry for locale; if not, create it + try: + links = content.get(operating_system, locale) + links = links + ",\n" + link + content.set(operating_system, locale, links) + with open(linksfile, 'w') as f: + content.write(f) + self.logger.info("Link %s added to %s %s in %s" + % (link, operating_system, locale, provider)) + except ConfigParser.NoOptionError: + content.set(operating_system, locale, link) + with open(linksfile, 'w') as f: + content.write(f) + self.logger.info("Link %s added to %s-%s in %s" + % (link, operating_system, locale, provider)) + except ConfigParser.NoSectionError: + # This shouldn't happen, but just in case + self.logger.error("Unknown section %s in links file") + raise InternalError("Unknown %s section in links file" + % operating_system) + else: + raise LinkFileError("There is no links file for %s" % provider) diff --git a/src/gettor/smtp.py b/src/gettor/smtp.py new file mode 100644 index 0000000..6e684ae --- /dev/null +++ b/src/gettor/smtp.py @@ -0,0 +1,612 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor, a Tor Browser Bundle distribution system. +# + + +import os +import re +import sys +import time +import email +import gettext +import hashlib +import logging +import ConfigParser + +import utils +import core + +"""SMTP module for processing email requests.""" + + +class ConfigurationError(Exception): + pass + + +class BlacklistError(Exception): + pass + + +class AddressError(Exception): + pass + + +class SendEmailError(Exception): + pass + + +class InternalError(Exception): + pass + + +class SMTP(object): + """Receive and reply requests by email. + + Public methods: + + process_email(): Process the email received. + + Exceptions: + + ConfigurationError: Bad configuration. + BlacklistError: Address of the sender is blacklisted. + AddressError: Address of the sender malformed. + SendEmailError: SMTP server not responding. + InternalError: Something went wrong internally. + + """ + + def __init__(self, cfg=None): + """Create new object by reading a configuration file. + + Params: cfg - path of the configuration file. + + """ + # Define a set of default values + DEFAULT_CONFIG_FILE = 'smtp.cfg' + + logging.basicConfig(format='[%(levelname)s] %(asctime)s - %(message)s', + datefmt="%Y-%m-%d %H:%M:%S") + logger = logging.getLogger(__name__) + config = ConfigParser.ConfigParser() + + if cfg is None or not os.path.isfile(cfg): + cfg = DEFAULT_CONFIG_FILE + logger.info("Using default configuration") + + logger.info("Reading configuration file %s" % cfg) + config.read(cfg) + + try: + self.basedir = config.get('general', 'basedir') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'basedir' from 'general' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.delay = config.get('general', 'delay') + # There has to be a better way for doing this... + if self.delay == 'False': + self.delay = False + + except ConfigParser.Error as e: + logger.warning("Couldn't read 'delay' from 'general' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.our_addr = config.get('general', 'our_addr') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'our_addr' from 'general' (%s)" % + cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.logdir = config.get('log', 'dir') + self.logdir = os.path.join(self.basedir, self.logdir) + except ConfigParser.Error as e: + logger.warning("Couldn't read 'dir' from 'log' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.logdir_emails = config.get('log', 'emails_dir') + self.logdir_emails = os.path.join(self.logdir, self.logdir_emails) + except ConfigParser.Error as e: + logger.warning("Couldn't read 'emails_dir' from 'log' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.loglevel = config.get('log', 'level') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'level' from 'log' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + # Use default values + self.core = core.Core() + + # Keep log levels separated + self.logger = utils.filter_logging(logger, self.logdir, self.loglevel) + self.logger.setLevel(logging.getLevelName(self.loglevel)) + logger.debug('Redirecting logging to %s' % self.logdir) + + # Stop logging on stdout from now on + logger.propagate = False + self.logger.debug("New smtp object created") + + def _get_sha1(self, string): + """Get sha1 of a string. + + Used whenever we want to do things with addresses (log, blacklist, + etc.) + + Params: The string to be sha1'ed. + + Returns: sha1 of string. + + """ + return str(hashlib.sha1(string).hexdigest()) + + def _log_request(self, addr, raw_msg): + """Log a request. + + This should be called when something goes wrong. It saves the + email content that triggered the malfunctioning. + + Raises: InternalError: if something goes wrong while trying to + save the email. + + Params: addr - The address of the sender. + content - The content of the email received. + + """ + + # to do: obtain content of msg, not the entire email + content = raw_msg + + # We store the sha1 of the original address in order to know when + # specific addresses are doing weird requests + log_addr = self._get_sha1(addr) + filename = str(time.time()) + '.log' + path = self.logdir_emails + filename + abs_path = os.path.abspath(path) + + if os.path.isfile(abs_path): + log_file = open(abs_path, 'w+') + log_file.write(content) + log_file.close() + self.logger.debug("Logging request from %s in %s" % + (log_addr, abs_path)) + else: + self.logger.warning("Couldn't open emails' log file (%s)" % + abs_path) + raise InternalError("Error while saving the email content.") + + def _check_blacklist(self, addr): + """Check if an email address is blacklisted. + + Look for the address in the file of blacklisted addresses. + + Raises: BlacklistError if the user is blacklisted. + + Params: addr - the address we want to check. + + """ + anon_addr = self._get_sha1(addr) + self.logger.debug("Checking if address %s is blacklisted" % + anon_addr) + + # if blacklisted: + # raise BlacklistError("Address %s is blacklisted!" % anon_addr) + + def _get_locale(self, addr): + """Get the locale from an email address. + + Process the email received and look for the locale in the recipient + address (e.g. gettor+en@torproject.org). If no locale found, english + by default. + + Params: The email address we want to get the locale. + + Returns: String containing the locale. + + """ + self.logger.debug("Trying to obtain locale from recipient address") + + # If no match found, english by default + locale = 'en' + + # Look for gettor+locale@torproject.org + m = re.match('gettor+(\w\w)@torproject.org', addr) + if m: + self.logger.debug("Request for locale %s" % m.groups()) + locale = "%s" % m.groups() + + return locale.lower() + + def _get_normalized_address(self, addr): + """Get normalized address. + + We look for anything inside the last '<' and '>'. Code taken from + the old GetTor (utils.py). + + Raises: AddressError: if address can't be normalized. + + Params: addr - the address we want to normalize. + + Returns: String with the normalized address on success. + + """ + if '<' in addr: + idx = addr.rindex('<') + addr = addr[idx:] + m = re.search(r'<([^>]*)>', addr) + if m is None: + raise AddressError("Couldn't extract normalized address " + "from %s" % self_get_sha1(addr)) + addr = m.group(1) + return addr + + def _parse_email(self, raw_msg, addr): + """Parse the email received. + + Get the locale and parse the text for the rest of the info. + + Params: raw_msg - content of the email to be parsed. + addr - address of the recipient (i.e. us). + + Returns: a 3-tuple with locale, os and type. + + """ + self.logger.debug("Parsing email") + + request = self._parse_text(raw_msg) + locale = self._get_locale(addr) + request['locale'] = locale + + return request + + def _parse_text(self, raw_msg): + """Parse the text part of the email received. + + Try to figure out what the user is asking, namely, the type + of request, the package and os required (if applies). + + Params: raw_msg - content of the email to be parsed. + + Returns: Tuple with the type of request and os + (None if request is for help). + """ + self.logger.debug("Parsing email text part") + + # By default we asume the request is asking for help + request = {} + request['type'] = 'help' + request['os'] = None + + # core knows what OS are supported + supported_os = self.core.get_supported_os() + + lines = raw_msg.split('\n') + found_os = False + for line in lines: + # Check for help request + if re.match('.*help.*', line, re.IGNORECASE): + self.logger.info("Request for help found") + request['type'] = 'help' + break + # Check for os + if not found_os: + for supported in supported_os: + p = '.*' + supported + '.*' + if re.match(p, line, re.IGNORECASE): + request['os'] = supported + request['type'] = 'links' + self.logger.debug("Request for links found") + found_os = True + break + # Check if the user is asking for terms related to pt + if re.match("[obfs|plugabble transport|pt]", line, re.IGNORECASE): + self.logger.info("Request for PT found") + request['pt'] = True + + return request + + def _create_email(self, from_addr, to_addr, subject, msg): + """Create an email object. + + This object will be used to construct the reply. + + Params: from_addr - address of the sender. + to_addr - address of the recipient. + subject - subject of the email. + msg - content of the email. + + Returns: The email object. + + """ + self.logger.debug("Creating email object for replying") + # try: + # email_obj = MIMEtext(msg) + # email_obj['Subject'] = subject + # email_obj['From'] = from_addr + # email_obj['To'] = to_addr + + reply = "From: " + from_addr + ", To: " + to_addr + reply = reply + ", Subject: " + subject + "\n\n" + msg + + # return email_obj + return reply + + def _send_email(self, from_addr, to_addr, subject, msg): + """Send an email. + + Take a 'from' and 'to' addresses, a subject and the content, creates + the email and send it. + + Params: from_addr - address of the sender. + to_addr - address of the recipient. + subject - subject of the email. + msg - content of the email. + + """ + email_obj = self._create_email(from_addr, to_addr, subject, msg) + # try: + # s = smtplib.SMTP("localhost") + # s.sendmail(from_addr, to_addr, msg.as_string()) + # s.quit() + # except SMTPException as e: + # self.logger.error("Couldn't send the email: %s" % str(e)) + # raise SendEmailError("Error with SMTP: %s" % str(e)) + print email_obj + self.logger.debug("Email sent") + + def _send_delay(self, locale, from_addr, to_addr): + """Send delay message. + + If the config says so, send a delay message. + + Params: locale - two-character string describing a locale. + from_addr - address of the sender. + to_addr - address of the recipient. + + """ + self.logger.debug("Delay is ON. Sending a delay message.") + + # Obtain the content in the proper language and send it + t = gettext.translation(locale, './i18n', languages=[locale]) + _ = t.ugettext + + delay_subject = _('delay_subject') + delay_msg = _('delay_msg') + + try: + self._send_email(from_addr, to_addr, delay_subject, delay_msg) + except SendEmailError as e: + self.logger.warning("Couldn't send delay message") + raise InternalError("Error while sending delay message") + + def _send_links(self, links, locale, operating_system, from_addr, to_addr, + pt): + """Send links to the user. + + Get the message in the proper language (according to the locale), + replace variables and send the email. + + Params: links - links to be sent. + locale - two-character string describing a locale. + from_addr - address of the sender. + to_addr - address of the recipient. + pt - True/False if the user did a PT request. + + """ + self.logger.debug("Request for links in %s" % locale) + + # Obtain the content in the proper language and send it + t = gettext.translation(locale, './i18n', languages=[locale]) + _ = t.ugettext + + links_subject = _('links_subject') + links_msg = _('links_msg') + links_msg = links_msg % (operating_system, locale, links, links) + + # Don't forget to check if user did a PT request + if pt: + # If so, we get the links message + info about PT included. + links_subject = _('links_pt_subject') + links_msg = _('links_pt_msg') + links_msg = links_msg % (operating_system, locale, links, links) + + try: + self._send_email(from_addr, to_addr, links_subject, links_msg) + except SendEmailError as e: + self.logger.warning("Couldn't send links message") + raise InternalError("Error while sending links message") + + def _send_help(self, locale, from_addr, to_addr): + """Send help message. + + Get the message in the proper language (according to the locale), + replace variables (if any) and send the email. + + Params: locale - two-character string describing a locale. + from_addr - address of the sender. + to_addr - address of the recipient. + + """ + self.logger.debug("Request for help in %s" % locale) + + # Obtain the content in the proper language and send it + t = gettext.translation(locale, './i18n', languages=[locale]) + _ = t.ugettext + + help_subject = _('help_subject') + help_msg = _('help_msg') + + try: + self._send_email(from_addr, to_addr, help_subject, help_msg) + except SendEmailError as e: + self.logger.warning("Couldn't send help message") + raise InternalError("Error while sending help message") + + def _send_unsupported_os(self, operating_system, locale, from_addr, + to_addr): + """Send unsupported OS message. + + Get the message for unsupported OS in the proper language + (according to the locale, or in english if the locale is + unsupported too), replace variables (if any) and send the email. + + Params: locale - two-character string describing a locale. + from_addr - address of the sender. + to_addr - address of the recipient. + + """ + # Check if the locale is unsupported too + # If so, english by default + supported_locales = self.core.get_supported_locales() + if locale not in supported_locales: + locale = 'en' + + # Obtain the content in the proper language and send it + t = gettext.translation(locale, './i18n', languages=[locale]) + _ = t.ugettext + + unsupported_os_subject = _('unsupported_os_subject') + unsupported_os_msg = _('unsupported_os_msg') + unsupported_os_msg = unsupported_os_msg % operating_system + + try: + self._send_email(from_addr, to_addr, unsupported_os_subject, + unsupported_os_msg) + except SendEmailError as e: + self.logger.warning("Couldn't send unsupported OS message") + raise InternalError("Error while sending unsupported OS message") + + def _send_unsupported_locale(self, locale, operating_system, from_addr, + to_addr): + """Send unsupported locale message. + + Get the message for unsupported locale in english replace variables + (if any) and send the email. + + Params: operating_system - name of the operating system. + from_addr - address of the sender. + to_addr - address of the recipient. + + """ + + # Obtain the content in english and send it + t = gettext.translation(locale, './i18n', languages=['en']) + _ = t.ugettext + + unsupported_locale_subject = _('unsupported_locale_subject') + unsupported_locale_msg = _('unsupported_locale_msg') + unsupported_locale_msg = unsupported_locale_msg % locale + + try: + self._send_email(from_addr, to_addr, unsupported_locale_subject, + unsupported_locale_msg) + except SendEmailError as e: + self.logger.warning("Couldn't send unsupported locale message") + raise InternalError("Error while sending unsupported locale" + "message") + + def process_email(self, raw_msg): + """Process the email received. + + Create an email object from the string received. The processing + flow is as following: + + - Check for blacklisted address. + - Parse the email. + - Check the type of request. + - Send reply. + + Raises: InternalError if something goes wrong while asking for the + links to the Core module. + + Params: raw_msg - the email received. + + """ + parsed_msg = email.message_from_string(raw_msg) + from_addr = parsed_msg['From'] + to_addr = parsed_msg['To'] + bogus_request = False + + try: + norm_from_addr = self._get_normalized_address(from_addr) + except AddressError as e: + # This shouldn't stop us from receiving other requests + self.logger.warning(str(e)) + bogus_request = True + + if norm_from_addr: + try: + self._check_blacklist(self._get_sha1(norm_from_addr)) + except BlacklistError as e: + # This shouldn't stop us from receiving other requests + self.logger.warning(str(e)) + bogus_request = True + + if not bogus_request: + # Try to figure out what the user is asking + request = self._parse_email(raw_msg, to_addr) + + # Two possible options: asking for help or for the links + self.logger.info("New request for %s" % request['type']) + if request['type'] == 'help': + # make sure we can send emails + try: + self._send_help(request['locale'], self.our_addr, + norm_from_addr) + except SendEmailError as e: + raise InternalError("Something's wrong with the SMTP " + "server: %s" % str(e)) + + elif request['type'] == 'links': + if self.delay: + # make sure we can send emails + try: + self._send_delay(request['locale'], self.our_addr, + norm_from_addr) + except SendEmailError as e: + raise InternalError("Something's wrong with the SMTP " + "server: %s" % str(e)) + + try: + self.logger.info("Asking core for links in %s for %s" % + (request['locale'], request['os'])) + + links = self.core.get_links('SMTP', request['os'], + request['locale']) + + except UnsupportedOSError as e: + self.logger.info("Request for unsupported OS: %s (%s)" % + (request['os'], str(e))) + # if we got here, the address of the sender should be valid + # so we send him/her a message about the unsupported OS + self._send_unsupported_os(request['os'], request['locale'], + self.our_addr, norm_from_addr) + + except UnsupportedLocaleError as e: + self.logger.info("Request for unsupported locale: %s (%s)" + % (request['locale'], str(e))) + # if we got here, the address of the sender should be valid + # so we send him/her a message about the unsupported locale + self._send_unsupported_locale(request['locale'], + request['os'], self.our_addr, + norm_from_addr) + + # if core fails, we fail too + except (InternalError, ConfigurationError) as e: + self.logger.error("Something's wrong with the Core module:" + " %s" % str(e)) + raise InternalError("Error obtaining the links.") + + # make sure we can send emails + try: + self._send_links(links, request['locale'], request['os'], + self.our_addr, norm_from_addr) + except SendEmailError as e: + raise SendEmailError("Something's wrong with the SMTP " + "server: %s" % str(e)) diff --git a/src/gettor/utils.py b/src/gettor/utils.py new file mode 100644 index 0000000..cc927fe --- /dev/null +++ b/src/gettor/utils.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor, a Tor Browser Bundle distribution system. +# + +import os +import logging + +"""Common utilities for GetTor modules.""" + +class SingleLevelFilter(logging.Filter): + """Filter logging levels to create separated logs. + + Public methods: + + filter(): fitler logging levels. + + """ + + def __init__(self, passlevel, reject): + """Create a new object with level to be filtered. + + If reject value is false, all but the passlevel will be filtered. + Useful for logging in separated files. + + Params: passlevel - name of a logging level. + + """ + + self.passlevel = passlevel + self.reject = reject + + def filter(self, record): + """Do the actual filtering.""" + if self.reject: + return (record.levelno != self.passlevel) + else: + return (record.levelno == self.passlevel) + +def filter_logging(logger, dir, level): + """Create separated files for each level of logging. + + Params: logger - a logging object. + dir - directory to put the log files. + level - the level of logging for the all.log file. + + Returns: logger object. + + """ + # Keep a good format + string_format = '[%(levelname)7s] %(asctime)s - %(message)s' + formatter = logging.Formatter(string_format, '%Y-%m-%d %H:%M:%S') + + # Keep logs separated (and filtered) + # all.log depends on level specified as param + all_log = logging.FileHandler(os.path.join(dir, 'all.log'), mode='a+') + all_log.setLevel(logging.getLevelName(level)) + all_log.setFormatter(formatter) + + debug_log = logging.FileHandler(os.path.join(dir, 'debug.log'), mode='a+') + debug_log.setLevel('DEBUG') + debug_log.addFilter(SingleLevelFilter(logging.DEBUG, False)) + debug_log.setFormatter(formatter) + + info_log = logging.FileHandler(os.path.join(dir, 'info.log'), mode='a+') + info_log.setLevel('INFO') + info_log.addFilter(SingleLevelFilter(logging.INFO, False)) + info_log.setFormatter(formatter) + + warn_log = logging.FileHandler(os.path.join(dir, 'warn.log'), mode='a+') + warn_log.setLevel('WARNING') + warn_log.addFilter(SingleLevelFilter(logging.WARNING, False)) + warn_log.setFormatter(formatter) + + error_log = logging.FileHandler(os.path.join(dir, 'error.log'), mode='a+') + error_log.setLevel('ERROR') + error_log.addFilter(SingleLevelFilter(logging.ERROR, False)) + error_log.setFormatter(formatter) + + logger.addHandler(all_log) + logger.addHandler(info_log) + logger.addHandler(debug_log) + logger.addHandler(warn_log) + logger.addHandler(error_log) + + return logger + diff --git a/src/gettor/xmpp.py b/src/gettor/xmpp.py new file mode 100644 index 0000000..7940359 --- /dev/null +++ b/src/gettor/xmpp.py @@ -0,0 +1,378 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor, a Tor Browser Bundle distribution system. +# + +import os +import re +import sys +import time +import gettext +import hashlib +import logging +import ConfigParser + +from sleekxmpp import ClientXMPP +from sleekxmpp.exceptions import IqError, IqTimeout + +import utils +import core + + +"""XMPP module for processing requests.""" + + +class Bot(ClientXMPP): + """XMPP bot. + + Handle messages and pass them to XMPP module for parsing. + + """ + + def __init__(self, jid, password, xmpp_obj): + ClientXMPP.__init__(self, jid, password) + + self.xmpp = xmpp_obj + self.add_event_handler("session_start", self.session_start) + self.add_event_handler("message", self.message) + + def session_start(self, event): + self.send_presence() + self.get_roster() + + try: + self.get_roster() + except IqError as err: + logging.error('There was an error getting the roster') + logging.error(err.iq['error']['condition']) + self.disconnect() + except IqTimeout: + logging.error('Server is taking too long to respond') + self.disconnect() + + def message(self, msg): + if msg['type'] in ('chat', 'normal'): + msg_to_send = self.xmpp.parse_request(msg['from'], msg['body']) + if msg_to_send: + msg.reply(msg_to_send).send() + + +class ConfigurationError(Exception): + pass + + +class BlacklistError(Exception): + pass + + +class InternalError(Exception): + pass + + +class XMPP(object): + """Receive and reply requests by XMPP. + + Public methods: + + parse_request(): parses a message and tries to figure out what the user + is asking for. + + Exceptions: + + ConfigurationError: Bad configuration. + BlacklistError: User is blacklisted. + InternalError: Something went wrong internally. + + """ + + def __init__(self, cfg=None): + """Create new object by reading a configuration file. + + Params: cfg - path of the configuration file. + + """ + # Define a set of default values + DEFAULT_CONFIG_FILE = 'xmpp.cfg' + + logging.basicConfig(format='[%(levelname)s] %(asctime)s - %(message)s', + datefmt="%Y-%m-%d %H:%M:%S") + logger = logging.getLogger(__name__) + config = ConfigParser.ConfigParser() + + if cfg is None or not os.path.isfile(cfg): + cfg = DEFAULT_CONFIG_FILE + logger.info("Using default configuration") + + logger.info("Reading configuration file %s" % cfg) + config.read(cfg) + + try: + self.user = config.get('account', 'user') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'user' from 'account' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.password = config.get('account', 'password') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'password' from 'account' (%s)" % + cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.basedir = config.get('general', 'basedir') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'basedir' from 'general' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.logdir = config.get('log', 'dir') + self.logdir = os.path.join(self.basedir, self.logdir) + except ConfigParser.Error as e: + logger.warning("Couldn't read 'dir' from 'log' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.logdir_msgs = config.get('log', 'msgs_dir') + self.logdir_msgs = os.path.join(self.logdir, self.logdir_msgs) + except ConfigParser.Error as e: + logger.warning("Couldn't read 'msgs_dir' from 'log' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + try: + self.loglevel = config.get('log', 'level') + except ConfigParser.Error as e: + logger.warning("Couldn't read 'level' from 'log' (%s)" % cfg) + raise ConfigurationError("Error with conf. See log file.") + + # Use default values + self.core = core.Core() + + # Keep log levels separated + self.logger = utils.filter_logging(logger, self.logdir, self.loglevel) + self.logger.setLevel(logging.getLevelName(self.loglevel)) + logger.debug('Redirecting logging to %s' % self.logdir) + + # Stop logging on stdout from now on + logger.propagate = False + self.logger.debug("New xmpp object created") + + def start_bot(self): + """Start the bot for handling requests. + + Start a new sleekxmpp bot. + + """ + + self.logger.debug("Calling sleekmppp bot") + xmpp = Bot(self.user, self.password, self) + xmpp.connect() + xmpp.process(block=True) + + def _get_sha1(self, string): + """Get sha1 of a string. + + Used whenever we want to do things with accounts (log, blacklist, + etc.) + + Params: The string to be sha1'ed. + + Returns: sha1 of string. + + """ + return str(hashlib.sha1(string).hexdigest()) + + def _check_blacklist(self, account): + """Check if an account is blacklisted. + + Look for the account in the file of blacklisted accounts. + + Raises: BlacklistError if the user is blacklisted. + + Params: account - the account we want to check. + + """ + anon_account = self._get_sha1(account) + self.logger.debug("Checking if address %s is blacklisted" % + anon_account) + + # if blacklisted: + # raise BlacklistError("Account %s is blacklisted!" % anon_account) + + def _get_help_msg(self, locale): + """Get help message for a given locale. + + Get the message in the proper language (according to the locale), + replace variables (if any) and return the message. + + Return: a string containing the message. + + """ + self.logger.debug("Getting help message") + # Obtain the content in the proper language + t = gettext.translation(locale, './xmpp/i18n', languages=[locale]) + _ = t.ugettext + + help_msg = _('help_msg') + return help_msg + + def _get_unsupported_locale_msg(self, locale): + """Get unsupported locale message for a given locale. + + Get the message in the proper language (according to the locale), + replace variables (if any) and return the message. + + Return: a string containing the message. + + """ + self.logger.debug("Getting unsupported locale message") + # Obtain the content in the proper language + t = gettext.translation(locale, './xmpp/i18n', languages=[locale]) + _ = t.ugettext + + unsupported_locale_msg = _('unsupported_locale_msg') + return unsupported_locale_msg + + def _get_unsupported_os_msg(self, locale): + """Get unsupported OS message for a given locale. + + Get the message in the proper language (according to the locale), + replace variables (if any) and return the message. + + Return: a string containing the message. + + """ + self.logger.debug("Getting unsupported os message") + # Obtain the content in the proper language + t = gettext.translation(locale, './xmpp/i18n', languages=[locale]) + _ = t.ugettext + + unsupported_os_msg = _('unsupported_os_msg') + return unsupported_os_msg + + def _get_internal_error_msg(self, locale): + """Get internal error message for a given locale. + + Get the message in the proper language (according to the locale), + replace variables (if any) and return the message. + + Return: a string containing the message. + + """ + self.logger.debug("Getting internal error message") + # Obtain the content in the proper language + t = gettext.translation(locale, './xmpp/i18n', languages=[locale]) + _ = t.ugettext + + unsupported_locale_msg = _('internal_error_msg') + return internal_error_msg + + def _get_links_msg(self, locale, operating_system, pt, links): + """Get links message for a given locale, operating system and PT + request. + + Get the message in the proper language (according to the locale), + replace variables (if any) and return the message. + + Return: a string containing the message. + """ + self.logger.debug("Getting links message") + # Obtain the content in the proper language + t = gettext.translation(locale, './xmpp/i18n', languages=[locale]) + _ = t.ugettext + + if pt: + links_msg = _('links_pt_msg') + else: + links_msg = _('links_msg') + + links_msg = links_msg % (locale, operating_system, links) + + return links_msg + + def _parse_text(self, msg): + """Parse the text part of a message. + + Split the message in words and look for patterns for locale, + operating system and built-in pluggable transport info. + + """ + self.logger.debug("Starting text parsing") + # core knows what OS are supported + supported_os = self.core.get_supported_os() + supported_locales = self.core.get_supported_locales() + + # default values + request = {} + request['locale'] = 'en' + request['os'] = 'windows' + request['type'] = 'help' + found_locale = False + found_os = False + + # analyze every word + # request shouldn't be more than 10 words long, so there should + # be a limit for the amount of words + for word in msg.split(' '): + # look for locale, os and pt + if not found_locale: + for locale in supported_locales: + if re.match(locale, word, re.IGNORECASE): + found_locale = True + request['locale'] = locale + self.logger.debug("Found locale: %s" % locale) + if not found_os: + for operating_system in supported_os: + if re.match(operating_system, word, re.IGNORECASE): + found_os = True + request['os'] = operating_system + request['type'] = 'links' + self.logger.debug("Found OS: %s" % operating_system) + if re.match("[obfs|plugabble transport|pt]", word, + re.IGNORECASE): + request['pt'] = True + self.logger.debug("Found PT request") + + return request + + def parse_request(self, account, msg): + """Process the request received. + + Check if the user is not blacklisted and then check the body of + the message to find out what is asking. + + Params: account - the account that did the request. + msg - the body of the message sent to us. + + """ + try: + self._check_blacklist(str(account)) + except BlacklistError as e: + return None + + # let's try to guess what the user is asking + request = self._parse_text(str(msg)) + + if request['type'] == 'help': + return_msg = self._get_help_msg(request['locale']) + elif request['type'] == 'links': + try: + links = self.core._get_links("XMPP", + request['operating_system'], + request['locale']) + + return_msg = self._get_links_msg(request['locale'], + request['operating_system'], + request['pt'], links) + + except (ConfigurationError, InternalError) as e: + return_msg = self.core._get_internal_error_msg( + request['locale']) + + except UnsupportedLocaleError as e: + self.core._get_unsupported_locale_msg(request['locale']) + + except UnsupportedOSError as e: + self.core._get_unsupported_os_msg(request['locale']) + + return return_msg diff --git a/src/smtp_demo.py b/src/smtp_demo.py index 390377c..418b3bd 100644 --- a/src/smtp_demo.py +++ b/src/smtp_demo.py @@ -1,9 +1,9 @@ #!/usr/bin/env python import sys

-import smtp +import gettor.smtp

-service = smtp.SMTP('smtp.cfg') +service = gettor.smtp.SMTP()

# For now we simulate mails reading from stdin # In linux test as follows: @@ -14,7 +14,9 @@ try: print "Email received!" service.process_email(incoming) print "Email sent!" -except ValueError as e: - print "Value error: " + str(e) -except RuntimeError as e: - print "Runtime error: " + str(e) +except gettor.smtp.ConfigurationError as e: + print "Misconfiguration: " + str(e) +except gettor.smtp.SendEmailError as e: + print "SMTP not working: " + str(e) +except gettor.smtp.InternalError as e: + print "Core module not working: " + str(e) diff --git a/src/xmpp.cfg b/src/xmpp.cfg new file mode 100644 index 0000000..a78ef88 --- /dev/null +++ b/src/xmpp.cfg @@ -0,0 +1,11 @@ +[account] +user: +password: + +[general] +basedir: xmpp/ + +[log] +level: DEBUG +dir: log/ +msgs_dir: msgs/ diff --git a/src/xmpp_demo.py b/src/xmpp_demo.py new file mode 100644 index 0000000..144f4a8 --- /dev/null +++ b/src/xmpp_demo.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +import sys + +import gettor.xmpp + +bot = gettor.xmpp.XMPP() +bot.start_bot()