[tor-commits] [gettor/master] More GetTor spring cleaning. To much little changes to count them all.

commit 254cfb9137414c60f75365b14293c2929a4cb14e Author: Christian Fromme kaner@strace.org Date: Tue Feb 22 03:03:48 2011 +0100

More GetTor spring cleaning. To much little changes to count them all. --- README.locale-spec-draft | 16 --- gettor.conf | 21 +++- lib/GetTor.py | 52 +++++---- lib/gettor/blacklist.py | 4 +- lib/gettor/config.py | 8 +- lib/gettor/requests.py | 295 ++++++++++++---------------------------------- lib/gettor/responses.py | 239 ++++++++++++++++--------------------- lib/gettor/utils.py | 30 +++-- 8 files changed, 248 insertions(+), 417 deletions(-)

diff --git a/README.locale-spec-draft b/README.locale-spec-draft index bc35557..b754f98 100644 --- a/README.locale-spec-draft +++ b/README.locale-spec-draft @@ -18,22 +18,6 @@ SUPPORTED LOCALES

All supported locales will be advertised by gettor in all emails.

-LOW TECH LOCALE PARSING ------------------------ - -If no locale is specified, GetTor will default to English in the locale 'en-US' -for all output. - -If a user wishes, they may set the locale for all responses. The locale must be -passed by the end user as a single line. The start of the line must begin with -'Lang: ' and be followed by the desired locale. The locale must be at the end -of the line. An example follows on the following line: - - Lang: de - -The first valid Locale line found in the message will designate the response -locale. - AUTOMATED LOCALE PARSING ------------------------

diff --git a/gettor.conf b/gettor.conf index 12b85d3..276e122 100644 --- a/gettor.conf +++ b/gettor.conf @@ -8,9 +8,6 @@ MAIL_FROM = "GetTor gettor@torproject.org" # Where it is all based at. Subdirs for GetTor start from here. BASEDIR = "/tmp"

-# Default locale -DEFAULT_LOCALE = "en" - # Should we send a `Your package will arrive soon, be patient' mail? DELAY_ALERT = True

@@ -23,7 +20,7 @@ LOGLEVEL = "DEBUG" # The file containing the hashed command password PASSFILE = "gettor.pass"

-# Where do we dump erronous emails? +# Where do we dump erronous emails? If this is "", no records are kept DUMPFILE = "gettor.dump"

# Do we send every mail type to every user only once before we blacklist them @@ -33,6 +30,22 @@ BLACKLIST_BY_TYPE = True # Which mirror to sync packages from RSYNC_MIRROR = "rsync.torproject.org"

+# Default locale +DEFAULT_LOCALE = "en" + +# Which languages to we support in GetTor? +SUPP_LANGS = { "en": ("english", ), + "fa": ("farsi", ), + "de": ("deutsch", ), + "ar": ("arabic", ), + "es": ("spanish", ), + "fr": ("french", ), + "it": ("italian", ), + "nl": ("dutch", ), + "pl": ("polish", ), + "ru": ("russian", ), + "zh_CN": ("chinese", "zh",) } + PACKAGES = { # "bundle name": # ("single file regex", # Path names need to be relative to diff --git a/lib/GetTor.py b/lib/GetTor.py index 68f9ddf..4c050b6 100644 --- a/lib/GetTor.py +++ b/lib/GetTor.py @@ -20,6 +20,7 @@ import gettor.config import gettor.requests import gettor.responses import gettor.utils +import gettor.filters from time import strftime

@@ -37,22 +38,18 @@ def initializeLogging(cfg): level=level, **extra)

-def processFail(conf, rawMessage, reqval, failedAction, e=None): +def processFail(conf, rawMessage, failedAction, e=None): """This routine gets called when something went wrong with the processing """ - logging.error("Failing to " + failedAction) + logging.error("Failed to " + failedAction) if e is not None: logging.error("Here is the exception I saw: %s" % sys.exc_info()[0]) logging.error("Detail: %s" %e) - # Keep a copy of the failed email for later reference - logging.info("We'll keep a record of this mail.") - dumpFile = os.path.join(conf.BASEDIR, conf.DUMPFILE) - gettor.utils.dumpMessage(dumpFile, rawMessage) - -def dumpInfo(reqval): - """Dump some info to the logging.ile - """ - logging.info("Request From: %s To: %s Package: %s Lang: %s Split: %s Signature: %s Cmdaddr: %s" % (reqval.replyTo, reqval.toField, reqval.pack, reqval.lang, reqval.split, reqval.sign, reqval.cmdAddr)) + if conf.DUMPFILE != "": + # Keep a copy of the failed email for later reference + logging.info("We'll keep a record of this mail.") + dumpFile = os.path.join(conf.BASEDIR, conf.DUMPFILE) + gettor.utils.dumpMessage(dumpFile, rawMessage)

def processMail(conf): """All mail processing happens here. Processing goes as follows: @@ -60,32 +57,45 @@ def processMail(conf): an appropriate manner. Reply address, language, package name. Also try to find out if the user wants split packages and if he has a valid signature on his mail. + - Do some filtering voodoo. - Send reply. Use all information gathered from the request and pass it on to the reply class/method to decide what to do. """ rawMessage = "" - reqval = None + reqInfo = None logging.debug("Processing mail..") # Retrieve request from stdin and parse it try: request = gettor.requests.requestMail(conf) rawMessage = request.getRawMessage() # reqval contains all important information we need from the request - reqval = request.parseMail() - dumpInfo(reqval) - except Exception, e: - processFail(conf, rawMessage, reqval, "process request", e) + reqInfo = request.parseMail() + except Exception as e: + processFail(conf, rawMessage, "process request", e) + return False + + # Do some filtering on the request + try: + reqInfo = gettor.filters.doFilter(reqInfo) + except Exception as e: + processFail(conf, rawMessage, "filter request", e) + return False + + # See if the filtering process decided if the request was valid + if reqInfo['valid'] is not True: + logging.error("Invalid request.") + processFail(conf, rawMessage, "send reply") return False

- # Ok, request information aquired. Initiate reply sequence + # Ok, request information aquired. Make it so. try: - reply = gettor.responses.Response(conf, reqval) + reply = gettor.responses.Response(conf, reqInfo) if not reply.sendReply(): - processFail(conf, rawMessage, reqval, "send reply") + processFail(conf, rawMessage, "send reply") return False return True - except Exception, e: - processFail(conf, rawMessage, reqval, "send reply (got exception)", e) + except Exception as e: + processFail(conf, rawMessage, "send reply (got exception)", e) return False

def processOptions(options, conf): diff --git a/lib/gettor/blacklist.py b/lib/gettor/blacklist.py index 3a4c87a..cbf2c47 100644 --- a/lib/gettor/blacklist.py +++ b/lib/gettor/blacklist.py @@ -46,10 +46,10 @@ class BWList: def createListEntry(self, address, blacklistName="general"): """Create a black- or whitelist entry """ - if address is None or blacklistName is None: + if address is None: logging.error("Bad args in createListEntry()") return False - if self.lookupListEntry(address) == False: + if self.lookupListEntry(address, blacklistName) == False: hashString = self.getHash(address) entry = os.path.join(self.blacklistDir, blacklistName, hashString) try: diff --git a/lib/gettor/config.py b/lib/gettor/config.py index ea8f95a..0efa1f4 100644 --- a/lib/gettor/config.py +++ b/lib/gettor/config.py @@ -9,7 +9,6 @@

MAIL_FROM: The email address we put in the `from' field in replies. BASEDIR: Where it is all based at. Subdirs for GetTor start from here. - DEFAULT_LOCALE: Default locale LOGFILE: Log messages will be written to $logFile-YYYY-MM-DD.log LOGLEVEL: The level log records are written with DELAY_ALERT: If set to True (the default), a message will be sent to any @@ -20,6 +19,8 @@ BLACKLIST_BY_TYPE: Do we send every mail type to every user only once before we blacklist them for it? RSYNC_MIRROR: Which rsync server to sync packages from + DEFAULT_LOCALE: Default locale + SUPP_LANGS: Supported languages by GetTor PACKAGES: List of packages GetTor serves

If no valid config file is provided to __init__, gettorConf will try to use @@ -35,7 +36,6 @@ __all__ = ["Config"] CONFIG_DEFAULTS = { 'MAIL_FROM': "GetTor gettor@torproject.org", 'BASEDIR': "/tmp", - 'DEFAULT_LOCALE': "en", 'DELAY_ALERT': True, 'LOGFILE': "gettorlog", 'LOGLEVEL': "DEBUG", @@ -43,10 +43,12 @@ CONFIG_DEFAULTS = { 'DUMPFILE': "./gettor.dump", 'BLACKLIST_BY_TYPE': True, 'RSYNC_MIRROR': "rsync.torproject.org", + 'DEFAULT_LOCALE': "en", + 'SUPP_LANGS': { 'en': ("english", ), }, 'PACKAGES': { "tor-browser-bundle": ("tor-browser-.*_en-US.exe$", - "tor-browser-.*_en-US_split") } + "tor-browser-.*_en-US_split"), } }

class Config: diff --git a/lib/gettor/requests.py b/lib/gettor/requests.py index 812eede..bcf461f 100644 --- a/lib/gettor/requests.py +++ b/lib/gettor/requests.py @@ -4,102 +4,58 @@

import sys import email -import dkim import re import logging

+import gettor.utils import gettor.packages

-__all__ = ["requestMail"] - -class RequestVal: - """A simple value class carrying everything that is interesting from a - parsed email - toField - Who this mail was sent to - replyTo - Who sent us the mail, this is also who gets the reply - lang - The language the user wants the reply in - split - True if the user wants us to send Tor in split files - sign - True if that mail carried a good DKIM signature - cmdAddr - Special commands from the GetTor admin - """ - def __init__(self, toField, replyTo, lang, pack, split, sign, cmdAddr): - self.toField = toField - self.replyTo = replyTo - self.lang = lang - self.pack = pack - self.split = split - self.sign = sign - self.cmdAddr = cmdAddr - class requestMail: - defaultLang = "en" - # XXX Move this to the config file - # LANG: ALIASE - supportedLangs = { "en": ("english", ), - "fa": ("farsi", ), - "de": ("deutsch", ), - "ar": ("arabic", ), - "es": ("spanish", ), - "fr": ("french", ), - "it": ("italian", ), - "nl": ("dutch", ), - "pl": ("polish", ), - "ru": ("russian", ), - "zh_CN": ("chinese", "zh",) } - def __init__(self, config): - """ Read message from stdin, initialize and do some early filtering and - sanitization + """ Read message from stdin, try to assign some values already. """ # Read email from stdin self.rawMessage = sys.stdin.read() self.parsedMessage = email.message_from_string(self.rawMessage)

- # WARNING WARNING *** This next line whitelists all addresses - # *** It exists as long as we don't want to check - # *** for DKIM properly - self.signature = True - # WARNING WARNING *** - self.config = config - self.gotPlusReq = False # Was this a gettor+lang@ request? - self.returnPackage = None - self.splitDelivery = False - self.commandAddress = None - self.replyLocale = self.defaultLang - self.replytoAddress = self.parsedMessage["Return-Path"] - self.bounce = False # Is the mail a bounce? - self.defaultFrom = self.config.MAIL_FROM - - # Filter rough edges - self.doEarlyFilter() - # We want to parse and act on the "To" field - self.sanitizeAndAssignToField(self.parsedMessage["to"]) - - logging.debug("User %s made request to %s" % \ - (self.replytoAddress, self.toAddress)) - self.gotPlusReq = self.matchPlusAddress() - - # TODO XXX: - # This should catch DNS exceptions and fail to verify if we have a - # dns timeout - # We also should catch totally malformed messages here - #try: - # if dkim.verify(self.rawMessage): - # self.signature = True - # except: - # pass - - def sanitizeAndAssignToField(self, toField): - """Do basic santization of the To: field of the mail header + self.request = {} + self.request['user'] = self.parsedMessage["Return-Path"] + self.request['ouraddr'] = self.getRealTo(self.parsedMessage["to"]) + self.request['locale'] = self.getLocaleInTo(self.request['ouraddr']) + self.request['plus'] = False # Was this a gettor+lang@ request? + self.request['package'] = None + self.request['split'] = False + self.request['forward'] = None + self.request['valid'] = False # This will get set by gettor.filters + + def getRealTo(self, toField): + """If someone wrote to `gettor+zh_CN@torproject.org', the `From:' field + in our reply should reflect that. So, use the `To:' field from the + incoming mail, but filter out everything except the gettor@ address. """ - regexGettorMail = '.*(<)?(gettor.*@torproject.org)+(?(1)>).*' - match = re.match(regexGettorMail, toField) + regexGettor = '.*(<)?(gettor.*@torproject.org)+(?(1)>).*' + match = re.match(regexGettor, toField) if match: - self.toAddress= match.group(2) + return match.group(2) else: # Fall back to default From: address - self.toAddress = self.defaultFrom + return self.config.MAIL_FROM + + def getLocaleInTo(self, address): + """See whether the user sent his email to a 'plus' address, for + instance to gettor+fa@tpo. Plus addresses are the current + mechanism to set the reply language + """ + regexPlus = '.*(<)?(\w++(\w+)@\w+(?:.\w+)+)(?(1)>)' + match = re.match(regexPlus, address) + if match: + locale = match.group(3) + logging.debug("User requested language %s" % locale) + return locale + else: + logging.debug("Not a 'plus' address") + return self.config.DEFAULT_LOCALE

def parseMail(self): """Main mail parsing routine. Returns a RequestVal value class @@ -110,64 +66,39 @@ class requestMail: # We found a text part, parse it self.parseTextPart(part.get_payload(decode=1)) else: + # Not a multipart message, just parse along what we've got self.parseTextPart(self.parsedMessage.get_payload(decode=1))

- if self.returnPackage is None: + if self.request['package'] is None: logging.debug("User didn't select any packages")

- return RequestVal(self.toAddress, - self.replytoAddress, - self.replyLocale, - self.returnPackage, - self.splitDelivery, - self.signature, - self.commandAddress) + return self.request

def parseTextPart(self, text): """If we've found a text part in a multipart email or if we just want to parse a non-multipart message, this is the routine to call with the text body as its argument """ - text = self.stripTags(text) - if not self.gotPlusReq: - self.matchLang(text) - self.checkLang() - - lines = text.split('\n') + lines = gettor.utils.stripHTMLTags(text).split('\n') for line in lines: - if self.returnPackage is None: - self.matchPackage(line) - if self.splitDelivery is False: - self.matchSplit(line) - if self.commandAddress is None: - self.matchCommand(line) - - self.torSpecialPackageExpansion() - - def matchPlusAddress(self): - """See whether the user sent his email to a 'plus' address, for - instance to gettor+fa@tpo. Plus addresses are the current - mechanism to set the reply language - """ - regexPlus = '.*(<)?(\w++(\w+)@\w+(?:.\w+)+)(?(1)>)' - match = re.match(regexPlus, self.toAddress) - if match: - self.replyLocale = match.group(3) - logging.debug("User requested language %s" % self.replyLocale) - return True - else: - logging.debug("Not a 'plus' address") - return False + if self.request['package'] is None: + self.request['package'] = self.matchPackage(line) + if self.request['split'] is False: + self.request['split'] = self.matchSplit(line) + if self.request['forward'] is None: + self.request['forward'] = self.matchForwardCommand(line)

def matchPackage(self, line): - """Look up which package the user wants to have""" - for package in self.config.PACKAGES.keys(): - matchme = ".*" + package + ".*" + """Look up which package the user is requesting. + """ + for p in self.config.PACKAGES.keys(): + matchme = ".*" + p + ".*" match = re.match(matchme, line, re.DOTALL) if match: - self.returnPackage = package - logging.debug("User requested package %s" % self.returnPackage) - return + logging.debug("User requested package %s" % p) + return p + + return None

def matchSplit(self, line): """If we find 'split' somewhere we assume that the user wants a split @@ -175,22 +106,12 @@ class requestMail: """ match = re.match(".*split.*", line, re.DOTALL) if match: - self.splitDelivery = True logging.debug("User requested a split delivery") + return True + else: + return False

- def matchLang(self, line): - """See if we have a "Lang: <lang>" directive in the mail. If so, - set the reply language appropriatly. - Note that setting the language in this way is somewhat deprecated. - Currently, replay language is chosen by the user with "plus" email - addresses (e.g. gettor+fa@tpo) - """ - match = re.match(".*[Ll]ang:\s+(.*)$", line, re.DOTALL) - if match: - self.replyLocale = match.group(1) - logging.debug("User requested locale %s" % self.replyLocale) - - def matchCommand(self, line): + def matchForwardCommand(self, line): """Check if we have a command from the GetTor admin in this email. Command lines always consists of the following syntax: 'Command: <password> <command part 1> <command part 2>' @@ -202,7 +123,7 @@ class requestMail: """ match = re.match(".*[Cc]ommand:\s+(.*)$", line, re.DOTALL) if match: - logging.debug("Command received from %s" % self.replytoAddress) + logging.debug("Command received from %s" % self.request['user']) cmd = match.group(1).split() length = len(cmd) assert length == 3, "Wrong command syntax" @@ -213,97 +134,31 @@ class requestMail: verified = gettor.utils.verifyPassword(self.config, auth) assert verified == True, \ "Unauthorized attempt to command from: %s" \ - % self.replytoAddress - self.commandAddress = address - - def torSpecialPackageExpansion(self): - """If someone wants one of the localizable packages, add language - suffix. This isn't nice because we're hard-coding package names here - Attention: This needs to correspond to the packages in packages.py - """ - if self.returnPackage == "tor-browser-bundle" \ - or self.returnPackage == "tor-im-browser-bundle" \ - or self.returnPackage == "linux-browser-bundle-i386" \ - or self.returnPackage == "linux-browser-bundle-x86_64": - # "tor-browser-bundle" => "tor-browser-bundle_de" - self.returnPackage = self.returnPackage + "_" + self.replyLocale - - def stripTags(self, string): - """Simple HTML stripper - """ - return re.sub(r'<[^>]*?>', '', string) - - def getRawMessage(self): - return self.rawMessage - - def hasVerifiedSignature(self): - return self.signature - - def getParsedMessage(self): - return self.parsedMessage - - def getReplyTo(self): - return self.replytoAddress - - def getPackage(self): - return self.returnPackage - - def getLocale(self): - return self.replyLocale - - def getSplitDelivery(self): - return self.splitDelivery - - def getAll(self): - return (self.replytoAddress, self.replyLocale, \ - self.returnPackage, self.splitDelivery, self.signature) + % self.request['user'] + return address + else: + return None

- def checkLang(self): + def checkAndGetLocale(self, locale): """Look through our aliases list for languages and check if the user requested an alias rather than an 'official' language name. If he does, map back to that official name. Also, if the user didn't request a language we support, fall back to default. """ - for (lang, aliases) in self.supportedLangs.items(): - if lang == self.replyLocale: + for (lang, aliases) in self.config.SUPP_LANGS.items(): + if lang == locale: logging.debug("User requested lang %s" % lang) - return + return locale if aliases is not None: - for alias in aliases: - if alias == self.replyLocale: - logging.debug("Request for %s via alias %s" % (lang, alias)) - # Set it back to the 'official' name - self.replyLocale = lang - return + if locale in aliases: + logging.debug("Request for %s via alias %s" % (lang, alias)) + # Return the 'official' name + return lang else: - logging.debug("Requested language %s not supported. Falling back to %s" \ - % (self.replyLocale, self.defaultLang)) - self.replyLocale = self.defaultLang + logging.debug("Requested language %s not supported. Fallback: %s" \ + % (self.replyLocale, self.config.DEFAULT_LOCALE)) + self.replyLocale = self.config.DEFAULT_LOCALE return

- def checkInternalEarlyBlacklist(self): - """This is called by doEarlyFilter() and makes it possible to add - hardcoded blacklist entries - XXX: This should merge somehow with the GetTor blacklisting - mechanism at some point - """ - if re.compile(".*@.*torproject.org.*").match(self.replytoAddress): - return True - else: - return False - - def doEarlyFilter(self): - """This exists to by able to drop mails as early as possible to avoid - mail loops and other terrible things. - Calls checkInternalEarlyBlacklist() to filter unwanted sender - addresses - """ - # Make sure we drop bounce mails - if self.replytoAddress == "<>": - logging.debug("We've received a bounce") - self.bounce = True - assert self.bounce is not True, "We've got a bounce. Bye." - - # Make sure we drop stupid from addresses - badMail = "Mail from address: %s" % self.replytoAddress - assert self.checkInternalEarlyBlacklist() is False, badMail + def getRawMessage(self): + return self.rawMessage diff --git a/lib/gettor/responses.py b/lib/gettor/responses.py index ba2d888..c741da6 100644 --- a/lib/gettor/responses.py +++ b/lib/gettor/responses.py @@ -18,49 +18,19 @@ import gettor.blacklist

trans = None

-def sendNotification(config, sendFr, sendTo): - """Send notification to user - """ - response = Response(config, sendFr, sendTo, None, "", False, True, "") - message = gettor.constants.mailfailmsg - return response.sendGenericMessage(message) - class Response: - def __init__(self, config, reqval): + def __init__(self, config, reqInfo): """Intialize the reply class. The most important values are passed in - via the 'reqval' RequestValue class. Initialize the locale subsystem - and do early blacklist checks of reply-to addresses + via the 'reqInfo' dict. """ self.config = config - self.reqval = reqval - # Set default From: field for reply. Defaults to gettor@torproject.org - if reqval.toField is None: - self.srcEmail = self.config.MAIL_FROM - else: - self.srcEmail = reqval.toField - self.replyTo = reqval.replyTo - # Make sure we know who to reply to. If not, we can't continue - assert self.replyTo is not None, "Empty reply address." - # Set default lang in none is set. Defaults to 'en' - if reqval.lang is None: - reqval.lang = self.config.LOCALE - self.mailLang = reqval.lang - self.package = reqval.pack - self.splitsend = reqval.split - self.signature = reqval.sign - self.cmdAddr = reqval.cmdAddr - # If cmdAddr is set, we are forwarding mail rather than sending a - # reply to someone - if self.cmdAddr is not None: - self.sendTo = self.cmdAddr - else: - self.sendTo = self.replyTo + self.reqInfo = reqInfo

# Initialize the reply language usage try: localeDir = os.path.join(self.config.BASEDIR, "i18n") - trans = gettext.translation("gettor", localeDir, [reqval.lang]) - trans.install() + t = gettext.translation("gettor", localeDir, [reqInfo['locale']]) + t.install() # OMG TEH HACK!! Constants need to be imported *after* we've # initialized the locale/gettext subsystem import gettor.constants @@ -71,13 +41,13 @@ class Response: # Init black & whitelists wlStateDir = os.path.join(self.config.BASEDIR, "wl") blStateDir = os.path.join(self.config.BASEDIR, "bl") - self.whiteList = gettor.blacklist.BWList(wlStateDir) - self.blackList = gettor.blacklist.BWList(blStateDir) + self.wList = gettor.blacklist.BWList(wlStateDir) + self.bList = gettor.blacklist.BWList(blStateDir) # Check blacklist section 'general' list & Drop if necessary # XXX: This should learn wildcards - blacklisted = self.blackList.lookupListEntry(self.replyTo, "general") - assert blacklisted is not True, \ - "Mail from blacklisted user %s" % self.replyTo + bListed = self.bList.lookupListEntry(self.reqInfo['user'], "general") + assert bListed is not True, \ + "Mail from blacklisted user %s" % self.reqInfo['user']

def sendReply(self): """All routing decisions take place here. Sending of mails takes place @@ -85,23 +55,18 @@ class Response: """ if self.isAllowed(): # Ok, see what we've got here. - # Was this a GetTor control command wanting us to forward a package? - if self.cmdAddr is not None: - success = self.sendPackage() - if not success: - logging.error("Failed to forward mail to '%s'" % self.cmdAddr) - return self.sendForwardReply(success) - + # Should we forward a certain package? + if self.reqInfo['forward'] is not None: + return self.forwardPackage() # Did the user choose a package? - if self.package is None: + if self.reqInfo['package'] is None: return self.sendPackageHelp() - delayAlert = self.config.DELAY_ALERT # Be a polite bot and send message that mail is on the way - if delayAlert: + if self.config.DELAY_ALERT: if not self.sendDelayAlert(): logging.error("Failed to sent delay alert.") # Did the user request a split or normal package download? - if self.splitsend: + if self.reqInfo['split']: return self.sendSplitPackage() else: return self.sendPackage() @@ -110,25 +75,7 @@ class Response: """Do all checks necessary to decide whether the reply-to user is allowed to get a reply by us. """ - - # Check we're happy with sending this user a package - # XXX This is currently useless since we set self.signature = True - if not self.signature and not self.cmdAddr \ - and not self.whiteList.lookupListEntry(self.replyTo) \ - and not re.compile(".*@yahoo.com.cn").match(self.replyTo) \ - and not re.compile(".*@yahoo.cn").match(self.replyTo) \ - and not re.compile(".*@gmail.com").match(self.replyTo): - blackListed = self.blackList.lookupListEntry(self.replyTo) - if blackListed: - logging.info("Unsigned messaged to gettor by blacklisted user dropped.") - return False - else: - # Reply with some help and bail out - self.blackList.createListEntry(self.replyTo) - logging.info("Unsigned messaged to gettor. We will issue help.") - return self.sendHelp() - else: - return True + return True # *g*

def isBlacklistedForMessageType(self, fname): """This routine takes care that for each function fname, a given user @@ -136,33 +83,34 @@ class Response: type name we're looking for """ # First of all, check if user is whitelisted: Whitelist beats Blacklist - if self.whiteList.lookupListEntry(self.replyTo, "general"): - logging.info("Whitelisted user " + self.replyTo) + if self.wList.lookupListEntry(self.reqInfo['user'], "general"): + logging.info("Whitelisted user " + self.reqInfo['user']) return False # Create a unique dir name for the requested routine - blStateDir = os.path.join(self.config.BASEDIR, "bl") - blackList = gettor.blacklist.BWList(blStateDir) - blackList.createSublist(fname) - if blackList.lookupListEntry(self.replyTo, fname): - logging.info("User " + self.replyTo + " is blacklisted for " + fname) + self.bList.createSublist(fname) + if self.bList.lookupListEntry(self.reqInfo['user'], fname): + logging.info("User %s is blacklisted for %s" \ + % (self.reqInfo['user'], fname)) return True else: - blackList.createListEntry(self.replyTo, fname) + self.bList.createListEntry(self.reqInfo['user'], fname) return False

def sendPackage(self): """ Send a message with an attachment to the user. The attachment is - chosen automatically from the selected self.package + chosen automatically from the selected self.reqInfo['package'] """ + pack = self.reqInfo['package'] + to = self.reqInfo['user'] if self.isBlacklistedForMessageType("sendPackage"): # Don't send anything return False - logging.info("Sending out %s to %s." % (self.package, self.sendTo)) - filename = os.path.join(self.config.BASEDIR, "packages", self.package + ".z") - message = gettor.constants.packagemsg - package = self.constructMessage(message, fileName=filename) + logging.info("Sending out %s to %s." % (pack, to)) + f = os.path.join(self.config.BASEDIR, "packages", pack + ".z") + txt = gettor.constants.packagemsg + msg = self.makeMsg(txt, to, fileName=f) try: - status = self.sendMessage(package) + status = self.sendEmail(to, msg) except: logging.error("Could not send package to user") status = False @@ -170,6 +118,33 @@ class Response: logging.debug("Send status: %s" % status) return status

+ def forwardPackage(self): + """ Forward a certain package to a user. Also send a message to the + one sending in the forward command. + """ + pack = self.reqInfo['package'] + fwd = self.reqInfo['forward'] + to = self.reqInfo['user'] + logging.info("Sending out %s to %s." % (pack, fwd)) + f = os.path.join(self.config.BASEDIR, "packages", pack + ".z") + text = gettor.constants.packagemsg + msg = self.makeMsg(text, fwd, fileName=f) + try: + status = self.sendEmail(fwd, msg) + except: + logging.error("Could not forward package to user") + status = False + + logging.info("Sending reply to forwarder '%s'" % to) + text = "Forwarding mail to '%s' status: %s" % (fwd, status) + msg = self.makeMsg(text, to) + try: + status = self.sendEmail(to, msg) + except: + logging.error("Could not send information to forward admin") + + return status + def sendSplitPackage(self): """Send a number of messages with attachments to the user. The number depends on the number of parts of the package. @@ -177,24 +152,29 @@ class Response: if self.isBlacklistedForMessageType("sendSplitPackage"): # Don't send anything return False - splitpack = self.package + ".split" - splitdir = os.path.join(self.config.BASEDIR, "packages", splitpack) - files = os.listdir(splitdir) + + # XXX + # Danger, Will Robinson: We assume that the split package is named + # `package.split' -- this is stupid because we let the user configure + # split package names in gettor.conf. + splitpack = self.reqInfo['package'] + ".split" + splitDir = os.path.join(self.config.BASEDIR, "packages", splitpack) + fileList = os.listdir(splitDir) # Sort the files, so we can send 01 before 02 and so on.. - files.sort() - nFiles = len(files) + fileList.sort() + nFiles = len(fileList) num = 0 # For each available split file, send a mail - for filename in files: - fullPath = os.path.join(splitdir, filename) + for filename in fileList: + path = os.path.join(splitDir, filename) num = num + 1 - subj = "[GetTor] Split package [%02d / %02d] " % (num, nFiles) - message = gettor.constants.splitpackagemsg - package = self.constructMessage(message, subj, fullPath) + sub = "[GetTor] Split package [%02d / %02d] " % (num, nFiles) + txt = gettor.constants.splitpackagemsg + msg = self.makeMsg(txt, sub, self.reqInfo['user'], fileName=path) try: - status = self.sendMessage(package) - logging.info("Sent out split package [%02d / %02d]. Status: %s" \ - % (num, nFiles, status)) + status = self.sendEmail(self.reqInfo['user'], msg) + logging.info("Package [%02d / %02d] sent. Status: %s" \ + % (num, nFiles, status)) except: logging.error("Could not send package %s to user" % filename) # XXX What now? Keep on sending? Bail out? Use might have @@ -209,8 +189,8 @@ class Response: if self.isBlacklistedForMessageType("sendDelayAlert"): # Don't send anything return False - logging.info("Sending delay alert to %s" % self.sendTo) - return self.sendGenericMessage(gettor.constants.delayalertmsg) + logging.info("Sending delay alert to %s" % self.reqInfo['user']) + return self.sendTextEmail(gettor.constants.delayalertmsg)

def sendHelp(self): """Send a help mail. This happens when a user sent us a request we @@ -219,15 +199,8 @@ class Response: if self.isBlacklistedForMessageType("sendHelp"): # Don't send anything return False - logging.info("Sending out help message to %s" % self.sendTo) - return self.sendGenericMessage(gettor.constants.helpmsg) - -## XXX the following line was used below to automatically list the names -## of available packages. But they were being named in an arbitrary -## order, which caused people to always pick the first one. I listed -## tor-browser-bundle first since that's the one they should want. -## Somebody should figure out how to automate yet sort. -RD -## """ + "".join([ "\t%s\n" % key for key in packageList.keys()]) + """ + logging.info("Sending out help message to %s" % self.reqInfo['user']) + return self.sendTextEmail(gettor.constants.helpmsg)

def sendPackageHelp(self): """Send a helpful message to the user interacting with us about @@ -236,45 +209,36 @@ class Response: if self.isBlacklistedForMessageType("sendPackageHelp"): # Don't send anything return False - logging.info("Sending package help to %s" % self.sendTo) - return self.sendGenericMessage(gettor.constants.multilangpackagehelpmsg) + logging.info("Sending package help to %s" % self.reqInfo['user']) + return self.sendTextEmail(gettor.constants.multilangpackagehelpmsg)

- def sendForwardReply(self, status): - """Send a message to the user that issued the forward command + def sendTextEmail(self, text): + """Generic text message sending routine. """ - logging.info("Sending reply to forwarder '%s'" % self.replyTo) - message = "Forwarding mail to '%s' status: %s" % (self.sendTo, status) - # Magic: We're now returning to the original issuer - self.sendTo = self.replyTo - return self.sendGenericMessage(message) - - def sendGenericMessage(self, text): - """Generic message sending routine. All mails that are being sent out - go through this function. - """ - message = self.constructMessage(text) + message = self.makeMsg(text, self.reqInfo['user']) try: - status = self.sendMessage(message) + status = self.sendEmail(self.reqInfo['user'], message) except: - logging.error("Could not send message to user %s" % self.sendTo) + logging.error("Could not send message to user %s" \ + % self.reqInfo['user']) status = False

logging.debug("Send status: %s" % status) return status

- def constructMessage(self, messageText, subj="[GetTor] Your Request", fileName=None): + def makeMsg(self, txt, to, sub="[GetTor] Your Request", fileName=None): """Construct a multi-part mime message, including only the first part with plaintext. """ - + # Build message, add header message = MIMEMultipart() - message['Subject'] = subj - message['To'] = self.sendTo - message['From'] = self.srcEmail + message['Subject'] = sub + message['To'] = to + message['From'] = self.reqInfo['ouraddr']

- text = MIMEText(messageText, _subtype="plain", _charset="utf-8") # Add text part - message.attach(text) + mText = MIMEText(txt, _subtype="plain", _charset="utf-8") + message.attach(mText)

# Add a file part only if we have one if fileName: @@ -284,18 +248,19 @@ class Response: fp.close() encoders.encode_base64(filePart) # Add file part - filePart.add_header('Content-Disposition', 'attachment', filename=os.path.basename(fileName)) + f = os.path.basename(fileName) + filePart.add_header('Content-Disposition', 'attachment', filename=f) message.attach(filePart)

return message

- def sendMessage(self, message, smtpserver="localhost:25"): + def sendEmail(self, sendTo, message, smtpserver="localhost:25"): """Send out message via STMP. If an error happens, be verbose about the reason """ try: smtp = smtplib.SMTP(smtpserver) - smtp.sendmail(self.srcEmail, self.sendTo, message.as_string()) + smtp.sendmail(self.reqInfo['ouraddr'], sendTo, message.as_string()) smtp.quit() status = True except smtplib.SMTPAuthenticationError: @@ -325,8 +290,8 @@ class Response: except smtplib.SMTPException: logging.error("General SMTP error caught") return False - except Exception, e: - logging.error("Unknown SMTP error while trying to send via local MTA") + except Exception as e: + logging.error("Unknown SMTP error while sending via local MTA") logging.error("Here is the exception I saw: %s" % sys.exc_info()[0]) logging.error("Detail: %s" %e)

diff --git a/lib/gettor/utils.py b/lib/gettor/utils.py index a91a160..c103f9a 100644 --- a/lib/gettor/utils.py +++ b/lib/gettor/utils.py @@ -55,7 +55,7 @@ def dumpMessage(dumpFile, message): fd.write(message) fd.close return True - except Exception, e: + except Exception as e: logging.error("Creating dump entry failed: %s" % e) return False

@@ -111,7 +111,7 @@ def addWhitelistEntry(conf, address): wlStateDir = conf.BASEDIR + "/wl" logging.info("Adding address to whitelist: %s" % address) try: - whiteList = gettor.blacklist.BWList(conf.wlStateDir) + whiteList = gettor.blacklist.BWList(wlStateDir) except IOError, e: logging.error("Whitelist error: %s" % e) return False @@ -200,26 +200,22 @@ def clearBlacklist(conf, olderThanDays): logging.info("Deleting blacklist done.") return True

-def setCmdPassword(cmdPassFile, password): +def setCmdPassword(conf, password): """Write the password for the admin commands in the configured file Hash routine used: SHA1 """ logging.info("Setting command password") passwordHash = str(hashlib.sha1(password).hexdigest()) # Be nice: Create dir if it's not there - passDir = os.path.dirname(cmdPassFile) - if not os.access(passDir, os.W_OK): - if not createDir(passDir): - logging.error("Could not create pass dir") - return False + passFile = os.path.join(conf.BASEDIR, conf.PASSFILE) try: - fd = open(cmdPassFile, 'w') + fd = open(passFile, 'w') fd.write(passwordHash) fd.close - # Be secretive - os.chmod(cmdPassFile, 0400) + # Sekrit + os.chmod(passFile, 0400) return True - except Exception, e: + except Exception as e: logging.error("Creating pass file failed: %s" % e) return False

@@ -229,7 +225,8 @@ def verifyPassword(conf, password): """ candidateHash = str(hashlib.sha1(password).hexdigest()) try: - fd = open(conf.PASSFILE, 'r') + passFile = os.path.join(conf.BASEDIR, conf.PASSFILE) + fd = open(passFile, 'r') passwordHash = fd.read() fd.close if candidateHash == passwordHash: @@ -238,7 +235,7 @@ def verifyPassword(conf, password): else: logging.info("Verification failed") return False - except Exception, e: + except Exception as e: logging.error("Verifying password failed: %s" % e) return False

@@ -359,3 +356,8 @@ def stripEmail(address): # Make sure to return the address in the form of 'bla@foo.de' return normalizeAddress(address)

+def stripHTMLTags(string): + """Simple HTML stripper + """ + return re.sub(r'<[^>]*?>', '', string) +