June 2014 - tor-commits - lists.torproject.org

[bridgedb/develop] Move email autoresponder code to separate module.
by isis＠torproject.org 07 Jun '14

07 Jun '14

commit 0a0af4b89f37bbaae40bb34654fdf854858e837a Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Jun 2 20:44:19 2014 +0000 Move email autoresponder code to separate module. * ADD new bridgedb.email.autoresponder module. * MOVE b.e.server.createResponseBody() → b.e.autoresponder.createResponseBody() * MOVE b.e.server.generateResponse() → b.e.autoresponder.generateResponse() * MOVE b.e.server.MailResponse → b.e.autoresponder.EmailResponse * ADD new b.e.autoresponder.SMTPAutoresponder class, created mostly from pieces of b.e.server.MailMessage, but made more compatible with Twisted's existing SMTP server/client architecture. --- lib/bridgedb/email/autoresponder.py | 639 +++++++++++++++++++++++++++++++++++ lib/bridgedb/email/server.py | 409 ---------------------- 2 files changed, 639 insertions(+), 409 deletions(-) diff --git a/lib/bridgedb/email/autoresponder.py b/lib/bridgedb/email/autoresponder.py new file mode 100644 index 0000000..d5dadae --- /dev/null +++ b/lib/bridgedb/email/autoresponder.py @@ -0,0 +1,639 @@ +# -*- coding: utf-8; test-case-name: bridgedb.test.test_email_autoresponder -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson <nickm(a)torproject.org> +# Isis Lovecruft <isis(a)torproject.org> 0xA3ADB67A2CDB8B35 +# Matthew Finkel <sysrqb(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2007-2014, The Tor Project, Inc. +# (c) 2013-2014, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +"""Functionality for autoresponding to incoming emails.""" + +from __future__ import unicode_literals +from __future__ import print_function + +import io +import logging +import time + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.mail import smtp +from twisted.python import failure + +from bridgedb import safelog +from bridgedb.crypto import gpgSignMessage +from bridgedb.crypto import NEW_BUFFER_INTERFACE +from bridgedb.Dist import EmailRequestedHelp +from bridgedb.Dist import EmailRequestedKey +from bridgedb.Dist import TooSoonEmail +from bridgedb.Dist import IgnoreEmail +from bridgedb.email import dkim +from bridgedb.email import request +from bridgedb.email import templates +from bridgedb.parse import addr +from bridgedb import translations + + +def createResponseBody(lines, context, client, lang='en'): + """Parse the **lines** from an incoming email request and determine how to + respond. + + :param list lines: The list of lines from the original request sent by the + client. + :type context: class:`MailServerContext` + :param context: The context which contains settings for the email server. + :type client: :api:`twisted.mail.smtp.Address` + :param client: The client's email address which should be in the + :header:`To:` header of the response email. + :param str lang: The 2-5 character locale code to use for translating the + email. This is obtained from a client sending a email to a valid plus + address which includes the translation desired, i.e. by sending an + email to ``bridges+fa(a)torproject.org``, the client should receive a + response in Farsi. + :rtype: None or str + :returns: None if we shouldn't respond to the client (i.e., if they have + already received a rate-limiting warning email). Otherwise, returns a + string containing the (optionally translated) body for the email + response which we should send out. + """ + translator = translations.installTranslations(lang) + bridges = None + try: + bridgeRequest = request.determineBridgeRequestOptions(lines) + + # The request was invalid, respond with a help email which explains + # valid email commands: + if not bridgeRequest.isValid(): + raise EmailRequestedHelp("Email request from '%s' was invalid." + % str(client)) + + # Otherwise they must have requested bridges: + interval = context.schedule.getInterval(time.time()) + bridges = context.distributor.getBridgesForEmail( + str(client), + interval, + context.nBridges, + countryCode=None, + bridgeFilterRules=bridgeRequest.filters) + except EmailRequestedHelp as error: + logging.info(error) + return templates.buildWelcomeText(translator, client) + except EmailRequestedKey as error: + logging.info(error) + return templates.buildKeyMessage(translator, client) + except TooSoonEmail as error: + logging.info("Got a mail too frequently: %s." % error) + return templates.buildSpamWarning(translator, client) + except (IgnoreEmail, addr.BadEmail) as error: + logging.info(error) + # Don't generate a response if their email address is unparsable or + # invalid, or if we've already warned them about rate-limiting: + return None + else: + answer = "(no bridges currently available)\r\n" + if bridges: + transport = bridgeRequest.justOnePTType() + answer = "".join(" %s\r\n" % b.getConfigLine( + includeFingerprint=context.includeFingerprints, + addressClass=bridgeRequest.addressClass, + transport=transport, + request=str(client)) for b in bridges) + return templates.buildAnswerMessage(translator, client, answer) + +def generateResponse(fromAddress, client, body, subject=None, + messageID=None, gpgContext=None): + """Create a :class:`EmailResponse`, which acts like an in-memory + ``io.StringIO`` file, by creating and writing all headers and the email + body into the file-like ``EmailResponse.mailfile``. + + :param str fromAddress: The rfc:`2821` email address which should be in + the :header:`From:` header. + :type client: :api:`twisted.mail.smtp.Address` + :param client: The client's email address which should be in the + :header:`To:` header of the response email. + :param str subject: The string to write to the :header:`subject` header. + :param str body: The body of the email. If a **gpgContext** is also given, + and that ``Context`` has email signing configured, then + :meth:`EmailResponse.writeBody` will generate and include any + ascii-armored OpenPGP signatures in the **body**. + :type messageID: None or str + :param messageID: The :rfc:`2822` specifier for the :header:`Message-ID:` + header, if including one is desirable. + :type gpgContext: None or ``gpgme.Context``. + :param gpgContext: A pre-configured GPGME context. See + :func:`~crypto.getGPGContext`. + :rtype: :class:`EmailResponse` + :returns: A ``EmailResponse`` which contains the entire email. To obtain + the contents of the email, including all headers, simply use + :meth:`EmailResponse.readContents`. + """ + response = EmailResponse(gpgContext) + response.to = client + response.writeHeaders(fromAddress, str(client), subject, + inReplyTo=messageID) + response.writeBody(body) + + # Only log the email text (including all headers) if SAFE_LOGGING is + # disabled: + if not safelog.safe_logging: + contents = response.readContents() + logging.debug("Email contents:\n%s" % str(contents)) + else: + logging.debug("Email text for %r created." % str(client)) + + response.rewind() + return response + + +class EmailResponse(object): + """Holds information for generating a response email for a request. + + .. todo:: At some point, we may want to change this class to optionally + handle creating Multipart MIME encoding messages, so that we can + include attachments. (This would be useful for attaching our GnuPG + keyfile, for example, rather than simply pasting it into the body of + the email.) + + :type _buff: unicode or buffer + :cvar _buff: Used internally to write lines for the response email into + the ``_mailfile``. The reason why both of these attributes have two + possible types is for the same Python-buggy reasons which require + :data:`~bridgedb.crypto.NEW_BUFFER_INTERFACE`. + :type mailfile: :class:`io.StringIO` or :class:`io.BytesIO`. + :cvar mailfile: An in-memory file for storing the formatted headers and + body of the response email. + + :ivar str delimiter: Delimiter between lines written to the + :cvar:`mailfile`. + :ivar bool closed: ``True`` if :meth:`close` has been called. + :type to: :api:`twisted.mail.smtp.Address` + :ivar to: The client's email address which this response should be sent to. + """ + _buff = buffer if NEW_BUFFER_INTERFACE else unicode + mailfile = io.BytesIO if NEW_BUFFER_INTERFACE else io.StringIO + + def __init__(self, gpgContext=None): + """Create a response to an email we have recieved. + + This class deals with correctly formatting text for the response email + headers and the response body into an instance of :cvar:`mailfile`. + + :type gpgContext: None or ``gpgme.Context`` + :param gpgContext: A pre-configured GPGME context. See + :meth:`bridgedb.crypto.getGPGContext` for obtaining a + pre-configured **gpgContext**. If given, and the ``Context`` has + been configured to sign emails, then a response email body string + given to :meth:`writeBody` will be signed before being written + into the ``mailfile``. + """ + self.gpgContext = gpgContext + self.mailfile = self.mailfile() + self.delimiter = '\n' + self.closed = False + self.to = None + + def close(self): + """Close our :ivar:`mailfile` and set :ivar:`closed` to ``True``.""" + logging.debug("Closing %s.mailfile..." % (self.__class__.__name__)) + self.mailfile.close() + self.closed = True + + def read(self, size=None): + """Read, at most, **size** bytes from our :ivar:`mailfile`. + + .. note:: This method is required by Twisted's SMTP system. + + :param int size: The number of bytes to read. Defaults to ``None``, + which reads until EOF. + :rtype: str + :returns: The bytes read from the :ivar:`mailfile`. + """ + contents = '' + logging.debug("Reading%s from %s.mailfile..." + % ((' {0} bytes'.format(size) if size else ''), + self.__class__.__name__)) + try: + if size is not None: + contents = self.mailfile.read(int(size)) + else: + contents = self.mailfile.read() + except Exception as error: # pragma: no cover + logging.exception(error) + + return contents + + def readContents(self): + """Read the all the contents written thus far to the :cvar:`mailfile`, + and then :meth:`seek` to return to the original pointer position we + were at before this method was called. + + :rtype: str + :returns: The entire contents of the :cvar:`mailfile`. + """ + pointer = self.mailfile.tell() + self.mailfile.seek(0) + contents = self.mailfile.read() + self.mailfile.seek(pointer) + return contents + + def rewind(self): + """Rewind to the very beginning of the :cvar:`mailfile`.""" + logging.debug("Rewinding %s.mailfile..." % self.__class__.__name__) + self.mailfile.seek(0) + + def write(self, line): + """Write the **line** to the :ivar:`mailfile`. + + Any **line** written to me will have :ivar:`delimiter` appended to it + beforehand. + + :param str line: Something to append into the :ivar:`mailfile`. + """ + if line.find('\r\n') != -1: + # If **line** contains newlines, send it to :meth:`writelines` to + # break it up so that we can replace them: + logging.debug("Found newlines in %r. Calling writelines()." % line) + self.writelines(line) + else: + line += self.delimiter + self.mailfile.write(self._buff(line.encode('utf8'))) + self.mailfile.flush() + + def writelines(self, lines): + """Calls :meth:`write` for each line in **lines**. + + Line endings of ``'\r\n'`` will be replaced with :ivar:`delimiter` + (i.e. ``'\n'``). See :api:`twisted.mail.smtp.SMTPClient.getMailData` + for the reason. + + :type lines: basestring or list + :param lines: The lines to write to the :ivar:`mailfile`. + """ + if isinstance(lines, basestring): + lines = lines.replace('\r\n', '\n') + for ln in lines.split('\n'): + self.write(ln) + elif isinstance(lines, (list, tuple,)): + for ln in lines: + self.write(ln) + + def writeHeaders(self, fromAddress, toAddress, subject=None, + inReplyTo=None, includeMessageID=True, + contentType='text/plain; charset="utf-8"', **kwargs): + """Write all headers into the response email. + + :param str fromAddress: The email address for the ``From:`` header. + :param str toAddress: The email address for the ``To:`` header. + :type subject: None or str + :param subject: The ``Subject:`` header. + :type inReplyTo: None or str + :param inReplyTo: If set, an ``In-Reply-To:`` header will be + generated. This should be set to the ``Message-ID:`` header from + the client's original request email. + :param bool includeMessageID: If ``True``, generate and include a + ``Message-ID:`` header for the response. + :param str contentType: The ``Content-Type:`` header. + :kwargs: If given, the key will become the name of the header, and the + value will become the Contents of that header. + """ + self.write("From: %s" % fromAddress) + self.write("To: %s" % toAddress) + if includeMessageID: + self.write("Message-ID: %s" % smtp.messageid()) + if inReplyTo: + self.write("In-Reply-To: %s" % inReplyTo) + self.write("Content-Type: %s" % contentType) + self.write("Date: %s" % smtp.rfc822date()) + + if not subject: + subject = '[no subject]' + if not subject.lower().startswith('re'): + subject = "Re: " + subject + self.write("Subject: %s" % subject) + + if kwargs: + for headerName, headerValue in kwargs.items(): + headerName = headerName.capitalize() + headerName = headerName.replace(' ', '-') + headerName = headerName.replace('_', '-') + self.write("%s: %s" % (headerName, headerValue)) + + # The first blank line designates that the headers have ended: + self.write(self.delimiter) + + def writeBody(self, body): + """Write the response body into the :cvar:`mailfile`. + + If ``EmailResponse.gpgContext`` is set, and signing is configured, the + **body** will be automatically signed before writing its contents into + the ``mailfile``. + + :param str body: The body of the response email. + """ + logging.info("Writing email body...") + if self.gpgContext: + logging.info("Attempting to sign email...") + body, _ = gpgSignMessage(self.gpgContext, body) + self.writelines(body) + + +class SMTPAutoresponder(smtp.SMTPClient): + """An :api:`twisted.mail.smtp.SMTPClient` for responding to incoming mail. + + The main worker in this class is the :meth:`reply` method, which functions + to dissect an incoming email from an incoming :class:`SMTPMessage` and + create a :class:`EmailResponse` email message in reply to it, and then, + finally, send it out. + + :ivar log: A :api:`twisted.python.util.LineLog` cache of messages. + :ivar debug: If ``True``, enable logging (accessible via :ivar:`log`). + :ivar str identity: Our FQDN which will be sent during client ``HELO``. + :ivar incoming: An incoming + :api:`Message <twisted.mail.smtp.rfc822.Message>`, i.e. as returned + from :meth:`SMTPMessage.getIncomingMessage`. + :ivar deferred: A :api:`Deferred <twisted.internet.defer.Deferred>` with + registered callbacks, :meth:`sentMail` and :meth:`sendError`, which + will be given to the reactor in order to process the sending of the + outgoing response email. + """ + debug = True + identity = smtp.DNSNAME + + def __init__(self): + """Handle responding (or not) to an incoming email.""" + smtp.SMTPClient.__init__(self, self.identity) + self.incoming = None + self.deferred = defer.Deferred() + self.deferred.addCallback(self.sentMail) + self.deferred.addErrback(self.sendError) + + def getMailData(self): + """Gather all the data for building the response to the client. + + This method must return a file-like object containing the data of the + message to be sent. Lines in the file should be delimited by '\n'. + + :rtype: ``None`` or :class:`EmailResponse` + :returns: An ``EmailResponse``, if we have a response to send in reply + to the incoming email, otherwise, returns ``None``. + """ + clients = self.getMailTo() + if not clients: return + client = clients[0] # There should have been only one anyway + if not self.runChecks(client): return + + recipient = self.getMailFrom() + # Look up the locale part in the 'To:' address, if there is one, and + # get the appropriate Translation object: + lang = translations.getLocaleFromPlusAddr(recipient) + logging.info("Client requested email translation: %s" % lang) + + body = createResponseBody(self.incoming.lines, + self.incoming.context, + client, lang) + if not body: return # The client was already warned. + + messageID = self.incoming.message.getheader("Message-ID", None) + subject = self.incoming.message.getheader("Subject", None) + response = generateResponse(recipient, client, + body, subject, messageID, + self.incoming.context.gpgContext) + return response + + def getMailTo(self): + """Attempt to get the client's email address from an incoming email. + + :rtype: list + :returns: A list containing the client's + :func:`normalized <addr.normalizeEmail>` email + :api:`Address <twisted.mail.smtp.Address>`, if it originated from + a domain that we accept and the address was well-formed. Otherwise, + returns ``None``. Even though we're likely to respond to only one + client at a time, the return value of this method must be a list + in order to hook into the rest of + :api:`twisted.mail.smtp.SMTPClient` correctly. + """ + clients = [] + addrHeader = None + try: fromAddr = self.incoming.message.getaddr("From")[1] + except (IndexError, TypeError, AttributeError): pass + else: addrHeader = fromAddr + + if not addrHeader: + logging.warn("No From header on incoming mail.") + try: senderHeader = self.incoming.message.getaddr("Sender")[1] + except (IndexError, TypeError, AttributeError): pass + else: addrHeader = senderHeader + if not addrHeader: + logging.warn("No Sender header on incoming mail.") + else: + try: + client = smtp.Address(addr.normalizeEmail( + addrHeader, + self.incoming.context.domainMap, + self.incoming.context.domainRules)) + except (addr.UnsupportedDomain, addr.BadEmail, + smtp.AddressError) as error: + logging.warn(error) + else: + clients.append(client) + return clients + + def getMailFrom(self): + """Find our address in the recipients list of the **incoming** message. + + :rtype: str + :return: Our address from the recipients list. If we can't find it + return our default ``EMAIL_FROM_ADDRESS`` from the config file. + """ + logging.debug("Searching for our email address in 'To:' header...") + + ours = None + + try: + ourAddress = smtp.Address(self.incoming.context.fromAddr) + allRecipients = self.incoming.message.getaddrlist("To") + + for _, addr in allRecipients: + recipient = smtp.Address(addr) + if not ourAddress.domain in recipient.domain: + logging.debug(("Not our domain (%s) or subdomain, skipping" + " email address: %s") + % (ourAddress.domain, str(recipient))) + continue + # The recipient's username should at least start with ours, + # but it still might be a '+' address. + if not recipient.local.startswith(ourAddress.local): + logging.debug(("Username doesn't begin with ours, skipping" + " email address: %s") % str(recipient)) + continue + # Only check the username before the first '+': + beforePlus = recipient.local.split('+', 1)[0] + if beforePlus == ourAddress.local: + ours = str(recipient) + if not ours: + raise addr.BadEmail(allRecipients) + + except Exception as error: + logging.error(("Couldn't find our email address in incoming email " + "headers: %r" % error)) + # Just return the email address that we're configured to use: + ours = self.incoming.context.fromAddr + + logging.debug("Found our email address: %s." % ours) + return ours + + def sentMail(self, success): + """Callback for a :api:`twisted.mail.smtp.SMTPSenderFactory`, + called when an attempt to send an email is completed. + + If some addresses were accepted, code and resp are the response + to the DATA command. If no addresses were accepted, code is -1 + and resp is an informative message. + + :param int code: The code returned by the SMTP Server. + :param str resp: The string response returned from the SMTP Server. + :param int numOK: The number of addresses accepted by the remote host. + :param list addresses: A list of tuples (address, code, resp) listing + the response to each ``RCPT TO`` command. + :param log: The SMTP session log. (We don't use this, but it is sent + by :api:`twisted.mail.smtp.SMTPSenderFactory` nonetheless.) + """ + numOk, addresses = success + + for (address, code, resp) in addresses: + logging.info("Sent reply to %s" % address) + logging.debug("SMTP server response: %d %s" % (code, resp)) + + if self.debug: + for line in self.log.log: + if line: + logging.debug(line) + + def sendError(self, fail): + """Errback for a :api:`twisted.mail.smtp.SMTPSenderFactory`. + + :param fail: A :api:`twisted.python.failure.Failure` or a + :api:`twisted.mail.smtp.SMTPClientError` which occurred during the + transaction to send the outgoing email. + """ + logging.debug("called with %r" % fail) + + if isinstance(fail, failure.Failure): + error = fail.getTraceback() or "Unknown" + elif isinstance(fail, Exception): + error = fail + logging.error(error) + + # This handles QUIT commands, disconnecting, and closing the transport: + smtp.SMTPClient.sendError(self, fail) + + def reply(self): + """Reply to an incoming email. Maybe. + + If nothing is returned from either :func:`createResponseBody` or + :func:`generateResponse`, then the incoming email will not be + responded to at all. This can happen for several reasons, for example: + if the DKIM signature was invalid or missing, or if the incoming email + came from an unacceptable domain, or if there have been too many + emails from this client in the allotted time period. + + :rtype: :api:`twisted.internet.defer.Deferred` + :returns: A ``Deferred`` which will callback when the response has + been successfully sent, or errback if an error occurred while + sending the email. + """ + logging.info("Got an email; deciding whether to reply.") + + response = self.getMailData() + if not response: + return self.deferred + + return self.send(response) + + def runChecks(self, client): + """Run checks on the incoming message, and only reply if they pass. + + 1. Check that the domain names, taken from the SMTP ``MAIL FROM:`` + command and the email ``'From:'`` header, can be + :func:`canonicalized <addr.canonicalizeEmailDomain>`. + + 2. Check that those canonical domains match, + + 3. If the incoming message is from a domain which supports DKIM + signing, then run :func:`bridgedb.email.dkim.checkDKIM` as well. + + .. note:: Calling this method sets the ``canonicalFromEmail`` and + :ivar:``canonicalDomainRules`` attributes of the :ivar:`incoming` + message. + + :param client: An :api:`twisted.mail.smtp.Address`, which contains + the client's email address, extracted from the ``'From:'`` header + from the incoming email. + :rtype: bool + :returns: ``False`` if the checks didn't pass, ``True`` otherwise. + """ + # If the SMTP ``RCPT TO:`` domain name couldn't be canonicalized, then + # we *should* have bailed at the SMTP layer, but we'll reject this + # email again nonetheless: + if not self.incoming.canonicalFromSMTP: + logging.warn(("SMTP 'MAIL FROM' wasn't from a canonical domain " + "for email from %s") % str(client)) + return False + + logging.debug("Canonicalizing client email domain...") + # The client's address was already checked to see if it came from a + # supported domain and is a valid email address in :meth:`getMailTo`, + # so we should just be able to re-extract the canonical domain safely + # here: + canonicalFromEmail = addr.canonicalizeEmailDomain( + client.domain, self.incoming.canon) + logging.debug("Canonical email domain: %s" % canonicalFromEmail) + + # The canonical domains from the SMTP ``MAIL FROM:`` and the email + # ``From:`` header should match: + if self.incoming.canonicalFromSMTP != canonicalFromEmail: + logging.error("SMTP/Email canonical domain mismatch!") + return False + + domainRules = self.incoming.context.domainRules.get( + canonicalFromEmail, list()) + + # If the domain's ``domainRules`` say to check DKIM verification + # results, and those results look bad, reject this email: + if not dkim.checkDKIM(self.incoming.message, domainRules): + return False + + self.incoming.canonicalDomainRules = domainRules + self.incoming.canonicalFromEmail = canonicalFromEmail + return True + + def send(self, response, retries=0, timeout=30, reaktor=reactor): + """Send our **response** in reply to :ivar:`incoming`. + + :type client: :api:`twisted.mail.smtp.Address` + :param client: The email address of the client. + :param response: A :class:`EmailResponse`. + :param int retries: Try resending this many times. (default: ``0``) + :param int timeout: Timeout after this many seconds. (default: ``30``) + :rtype: :api:`Deferred <twisted.internet.defer.Deferred>` + :returns: Our :ivar:`deferred`. + """ + logging.info("Sending reply to %s ..." % str(response.to)) + + factory = smtp.SMTPSenderFactory(self.incoming.context.smtpFromAddr, + str(response.to), + response, + self.deferred, + retries=retries, + timeout=timeout) + reaktor.connectTCP(self.incoming.context.smtpServerIP, + self.incoming.context.smtpServerPort, + factory) + return self.deferred diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py index f526b47..c526852 100644 --- a/lib/bridgedb/email/server.py +++ b/lib/bridgedb/email/server.py @@ -46,114 +46,6 @@ from bridgedb.parse.addr import UnsupportedDomain from bridgedb.parse.addr import canonicalizeEmailDomain -def createResponseBody(lines, context, client, lang='en'): - """Parse the **lines** from an incoming email request and determine how to - respond. - - :param list lines: The list of lines from the original request sent by the - client. - :type context: class:`MailContext` - :param context: The context which contains settings for the email server. - :type client: :api:`twisted.mail.smtp.Address` - :param client: The client's email address which should be in the - :header:`To:` header of the response email. - :param str lang: The 2-5 character locale code to use for translating the - email. This is obtained from a client sending a email to a valid plus - address which includes the translation desired, i.e. by sending an - email to ``bridges+fa(a)torproject.org``, the client should receive a - response in Farsi. - :rtype: None or str - :returns: None if we shouldn't respond to the client (i.e., if they have - already received a rate-limiting warning email). Otherwise, returns a - string containing the (optionally translated) body for the email - response which we should send out. - """ - t = translations.installTranslations(lang) - bridges = None - try: - bridgeRequest = request.determineBridgeRequestOptions(lines) - - # The request was invalid, respond with a help email which explains - # valid email commands: - if not bridgeRequest.isValid(): - raise EmailRequestedHelp("Email request from '%s' was invalid." - % str(client)) - - # Otherwise they must have requested bridges: - interval = context.schedule.getInterval(time.time()) - bridges = context.distributor.getBridgesForEmail( - str(client), - interval, - context.nBridges, - countryCode=None, - bridgeFilterRules=bridgeRequest.filters) - except EmailRequestedHelp as error: - logging.info(error) - return templates.buildWelcomeText(t, client) - except EmailRequestedKey as error: - logging.info(error) - return templates.buildKeyMessage(t, client) - except TooSoonEmail as error: - logging.info("Got a mail too frequently: %s." % error) - return templates.buildSpamWarning(t, client) - except (IgnoreEmail, BadEmail) as error: - logging.info(error) - # Don't generate a response if their email address is unparsable or - # invalid, or if we've already warned them about rate-limiting: - return None - else: - answer = "(no bridges currently available)\r\n" - if bridges: - transport = bridgeRequest.justOnePTType() - answer = "".join(" %s\r\n" % b.getConfigLine( - includeFingerprint=context.includeFingerprints, - addressClass=bridgeRequest.addressClass, - transport=transport, - request=str(client)) for b in bridges) - return templates.buildAnswerMessage(t, client, answer) - -def generateResponse(fromAddress, clientAddress, body, subject=None, - messageID=None, gpgContext=None): - """Create a :class:`MailResponse`, which acts like an in-memory - ``io.StringIO`` file, by creating and writing all headers and the email - body into the file-like ``MailResponse.mailfile``. - - :param str fromAddress: The rfc:`2821` email address which should be in - the :header:`From:` header. - :param str clientAddress: The rfc:`2821` email address which should be in - the :header:`To:` header. - :param str subject: The string to write to the :header:`subject` header. - :param str body: The body of the email. If a **gpgContext** is also given, - and that ``Context`` has email signing configured, then - :meth:`MailResponse.writeBody` will generate and include any - ascii-armored OpenPGP signatures in the **body**. - :type messageID: None or str - :param messageID: The :rfc:`2822` specifier for the :header:`Message-ID:` - header, if including one is desirable. - :type gpgContext: None or ``gpgme.Context``. - :param gpgContext: A pre-configured GPGME context. See - :meth:`~crypto.getGPGContext`. - :rtype: :class:`MailResponse` - :returns: A ``MailResponse`` which contains the entire email. To obtain - the contents of the email, including all headers, simply use - :meth:`MailResponse.readContents`. - """ - response = MailResponse(gpgContext) - response.writeHeaders(fromAddress, clientAddress, subject, - inReplyTo=messageID) - response.writeBody(body) - - # Only log the email text (including all headers) if SAFE_LOGGING is - # disabled: - if not safelog.safe_logging: - contents = response.readContents() - logging.debug("Email contents:\n%s" % contents) - else: - logging.debug("Email text for %r created." % clientAddress) - response.rewind() - return response - - class MailContext(object): """Helper object that holds information used by email subsystem. @@ -231,155 +123,6 @@ class MailContext(object): return canon -class MailResponse(object): - """Holds information for generating a response email for a request. - - .. todo:: At some point, we may want to change this class to optionally - handle creating Multipart MIME encoding messages, so that we can - include attachments. (This would be useful for attaching our GnuPG - keyfile, for example, rather than simply pasting it into the body of - the email.) - - :type _buff: unicode or buffer - :cvar _buff: Used internally to write lines for the response email into - the ``_mailfile``. The reason why both of these attributes have two - possible types is for the same Python-buggy reasons which require - :data:`~bridgedb.crypto.NEW_BUFFER_INTERFACE`. - :type mailfile: :class:`io.StringIO` or :class:`io.BytesIO`. - :cvar mailfile: An in-memory file for storing the formatted headers and - body of the response email. - """ - _buff = buffer if NEW_BUFFER_INTERFACE else unicode - mailfile = io.BytesIO if NEW_BUFFER_INTERFACE else io.StringIO - - def __init__(self, gpgContext=None): - """Create a response to an email we have recieved. - - This class deals with correctly formatting text for the response email - headers and the response body into an instance of :cvar:`mailfile`. - - :type gpgContext: None or ``gpgme.Context`` - :param gpgContext: A pre-configured GPGME context. See - :meth:`bridgedb.crypto.getGPGContext` for obtaining a - pre-configured **gpgContext**. If given, and the ``Context`` has - been configured to sign emails, then a response email body string - given to :meth:`writeBody` will be signed before being written - into the ``mailfile``. - """ - self.gpgContext = gpgContext - self.mailfile = self.mailfile() - self.closed = False - - # These are methods and attributes for controlling I/O operations on our - # underlying ``mailfile``. - - def close(self): - self.mailfile.close() - self.closed = True - close.__doc__ = mailfile.close.__doc__ - - # The following are custom methods to control reading and writing to the - # underlying ``mailfile``. - - def readContents(self): - """Read the all the contents written thus far to the :cvar:`mailfile`, - and then :meth:`seek` to return to the original pointer position we - were at before this method was called. - - :rtype: str - :returns: The entire contents of the :cvar:`mailfile`. - """ - pointer = self.mailfile.tell() - self.mailfile.seek(0) - contents = self.mailfile.read() - self.mailfile.seek(pointer) - return contents - - def rewind(self): - """Rewind to the very beginning of the :cvar:`mailfile`.""" - logging.debug("Rewinding %s.mailfile..." % self.__class__.__name__) - self.mailfile.seek(0) - - def write(self, line): - """Any **line** written to me will have ``'\r\n'`` appended to it.""" - if line.find('\n') != -1: - # If **line** contains newlines, send it to :meth:`writelines` to - # break it up so that we can replace them: - logging.debug("Found newlines in %r. Calling writelines()." % line) - self.writelines(line) - else: - self.mailfile.write(self._buff(line + '\r\n')) - self.mailfile.flush() - - def writelines(self, lines): - """Calls :meth:`write` for each line in **lines**.""" - if isinstance(lines, basestring): - for ln in lines.split('\n'): - self.write(ln) - elif isinstance(lines, (list, tuple,)): - for ln in lines: - self.write(ln) - - def writeHeaders(self, fromAddress, toAddress, subject=None, - inReplyTo=None, includeMessageID=True, - contentType='text/plain; charset="utf-8"', **kwargs): - """Write all headers into the response email. - - :param str fromAddress: The email address for the ``From:`` header. - :param str toAddress: The email address for the ``To:`` header. - :type subject: None or str - :param subject: The ``Subject:`` header. - :type inReplyTo: None or str - :param inReplyTo: If set, an ``In-Reply-To:`` header will be - generated. This should be set to the ``Message-ID:`` header from - the client's original request email. - :param bool includeMessageID: If ``True``, generate and include a - ``Message-ID:`` header for the response. - :param str contentType: The ``Content-Type:`` header. - :kwargs: If given, the key will become the name of the header, and the - value will become the Contents of that header. - """ - self.write("From: %s" % fromAddress) - self.write("To: %s" % toAddress) - if includeMessageID: - self.write("Message-ID: %s" % smtp.messageid()) - if inReplyTo: - self.write("In-Reply-To: %s" % inReplyTo) - self.write("Content-Type: %s" % contentType) - self.write("Date: %s" % smtp.rfc822date()) - - if not subject: - subject = '[no subject]' - if not subject.lower().startswith('re'): - subject = "Re: " + subject - self.write("Subject: %s" % subject) - - if kwargs: - for headerName, headerValue in kwargs.items(): - headerName = headerName.capitalize() - headerName = headerName.replace(' ', '-') - headerName = headerName.replace('_', '-') - self.write("%s: %s" % (headerName, headerValue)) - - # The first blank line designates that the headers have ended: - self.write("\r\n") - - def writeBody(self, body): - """Write the response body into the :cvar:`mailfile`. - - If ``MailResponse.gpgContext`` is set, and signing is configured, the - **body** will be automatically signed before writing its contents into - the ``mailfile``. - - :param str body: The body of the response email. - """ - logging.info("Writing email body...") - if self.gpgContext: - logging.info("Attempting to sign email...") - body, _ = gpgSignMessage(self.gpgContext, body) - self.writelines(body) - - class MailMessage(object): """Plugs into the Twisted Mail and receives an incoming message. @@ -441,161 +184,9 @@ class MailMessage(object): rawMessage.seek(0) return smtp.rfc822.Message(rawMessage) - def getClientAddress(self, incoming): - """Attempt to get the client's email address from an incoming email. - - :type incoming: :api:`twisted.mail.smtp.rfc822.Message` - :param incoming: An incoming ``Message``, i.e. as returned from - :meth:`getIncomingMessage`. - :rtype: ``None`` or :api:`twisted.mail.smtp.Address` - :returns: The client's email ``Address``, if it originated from a - domain that we accept and the address was well-formed. Otherwise, - returns ``None``. - """ - addrHeader = None - try: fromAddr = incoming.getaddr("From")[1] - except (IndexError, TypeError, AttributeError): pass - else: addrHeader = fromAddr - - if not addrHeader: - logging.warn("No From header on incoming mail.") - try: senderHeader = incoming.getaddr("Sender")[1] - except (IndexError, TypeError, AttributeError): pass - else: addrHeader = senderHeader - if not addrHeader: - logging.warn("No Sender header on incoming mail.") - else: - try: - client = smtp.Address(addr.normalizeEmail( - addrHeader, - self.context.domainMap, - self.context.domainRules)) - except (UnsupportedDomain, BadEmail, smtp.AddressError) as error: - logging.warn(error) - else: - return client - - def getMailFrom(self, incoming): - """Find our address in the recipients list of the **incoming** message. - - :type incoming: :api:`twisted.mail.smtp.rfc822.Message` - :param incoming: An incoming ``Message``, i.e. as returned from - :meth:`getIncomingMessage`. - :rtype: str - :return: Our address from the recipients list. If we can't find it - return our default ``SMTP_FROM_ADDRESS`` from the config file. - """ - logging.debug("Searching for our email address in 'To:' header...") - - ours = None - - try: - ourAddress = smtp.Address(self.context.fromAddr) - allRecipients = incoming.getaddrlist("To") - - for _, addr in allRecipients: - recipient = smtp.Address(addr) - if not (ourAddress.domain in recipient.domain): - logging.debug(("Not our domain (%s) or subdomain, skipping" - " email address: %s") - % (ourAddress.domain, str(recipient))) - continue - # The recipient's username should at least start with ours, - # but it still might be a '+' address. - if not recipient.local.startswith(ourAddress.local): - logging.debug(("Username doesn't begin with ours, skipping" - " email address: %s") % str(recipient)) - continue - # Ignore everything after the first '+', if there is one. - beforePlus = recipient.local.split('+', 1)[0] - if beforePlus == ourAddress.local: - ours = str(recipient) - if not ours: - raise BadEmail(allRecipients) - except Exception as error: - logging.error(("Couldn't find our email address in incoming email " - "headers: %r" % error)) - # Just return the email address that we're configured to use: - ours = self.context.fromAddr - logging.debug("Found our email address: %s." % ours) - return ours - def getCanonicalDomain(self, domain): - try: - canonical = canonicalizeEmailDomain(domain, self.context.canon) - except (UnsupportedDomain, BadEmail) as error: - logging.warn(error) - else: - return canonical - - def reply(self): - """Reply to an incoming email. Maybe. - - If nothing is returned from either :func:`createResponseBody` or - :func:`generateResponse`, then the incoming email will not be - responded to at all. This can happen for several reasons, for example: - if the DKIM signature was invalid or missing, or if the incoming email - came from an unacceptable domain, or if there have been too many - emails from this client in the allotted time period. - - :rtype: :api:`twisted.internet.defer.Deferred` - :returns: A ``Deferred`` which will callback when the response has - been successfully sent, or errback if an error occurred while - sending the email. - """ - logging.info("Got an email; deciding whether to reply.") - - def _replyEB(fail): # pragma: no cover - """Errback for a :api:`twisted.mail.smtp.SMTPSenderFactory`. - - :param fail: A :api:`twisted.python.failure.Failure` which occurred during - the transaction. - """ - logging.debug("_replyToMailEB() called with %r" % fail) - error = fail.getTraceback() or "Unknown" - logging.error(error) - - d = defer.Deferred() - d.addErrback(_replyEB) - - incoming = self.getIncomingMessage() - recipient = self.getMailFrom(incoming) - client = self.getClientAddress(incoming) - - if not client: - return d - - if not self.fromCanonical: - self.fromCanonical = self.getCanonicalDomain(client.domain) - rules = self.context.domainRules.get(self.fromCanonical, []) - if not checkDKIM(incoming, rules): - return d - - clientAddr = '@'.join([client.local, client.domain]) - messageID = incoming.getheader("Message-ID", None) - subject = incoming.getheader("Subject", None) - - # Look up the locale part in the 'To:' address, if there is one and - # get the appropriate Translation object: - lang = translations.getLocaleFromPlusAddr(recipient) - logging.info("Client requested email translation: %s" % lang) - - body = createResponseBody(self.lines, self.context, client, lang) - if not body: return d # The client was already warned. - - response = generateResponse(self.context.fromAddr, clientAddr, body, - subject, messageID, self.context.gpgContext) - if not response: return d - - logging.info("Sending reply to %s" % client) - factory = smtp.SMTPSenderFactory(self.context.smtpFromAddr, clientAddr, - response, d, retries=0, timeout=30) - reactor.connectTCP(self.context.smtpServerIP, - self.context.smtpServerPort, - factory) - return d class MailDelivery(object):

1 0

[bridgedb/develop] Refactor b.e.server.MailMessage class.
by isis＠torproject.org 07 Jun '14

07 Jun '14

commit 9a1dbf55924ad71c1257d86caf6ad6cc65c78510 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Jun 2 20:58:23 2014 +0000 Refactor b.e.server.MailMessage class. * ADD attributes so that the information about the incoming SMTP message is still accessible to the autoresponder while we're creating the reply. * FIXUP docstrings. --- lib/bridgedb/email/server.py | 49 ++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py index 249b7c7..a493be0 100644 --- a/lib/bridgedb/email/server.py +++ b/lib/bridgedb/email/server.py @@ -122,34 +122,54 @@ class MailServerContext(object): return canon -class MailMessage(object): +class SMTPMessage(object): """Plugs into the Twisted Mail and receives an incoming message. :ivar list lines: A list of lines from an incoming email message. :ivar int nBytes: The number of bytes received thus far. :ivar bool ignoring: If ``True``, we're ignoring the rest of this message - because it exceeded :ivar:`MailContext.maximumSize`. + because it exceeded :ivar:`MailServerContext.maximumSize`. + :ivar canonicalFromSMTP: See :meth:`SMTPAutoresponder.runChecks`. + :ivar canonicalFromEmail: See :meth:`SMTPAutoresponder.runChecks`. + :ivar canonicalDomainRules: See :meth:`SMTPAutoresponder.runChecks`. + :type message: :api:`twisted.mail.smtp.rfc822.Message` or ``None`` + :ivar message: The incoming email message. + :type responder: :class:`autoresponder.SMTPAutoresponder` + :ivar responder: A parser and checker for the incoming :ivar:`message`. If + it decides to do so, it will build a + :meth:`~autoresponder.SMTPAutoresponder.reply` email and + :meth:`~autoresponder.SMTPAutoresponder.send` it. """ implements(smtp.IMessage) - def __init__(self, context, fromCanonical=None): - """Create a new MailMessage from a MailContext. + def __init__(self, context, canonicalFromSMTP=None): + """Create a new SMTPMessage. - :type context: :class:`MailContext` - :param context: The configured context for the email server. - :type canonicalFrom: str or None - :param canonicalFrom: The canonical domain which this message was + These are created automatically via + :class:`SMTPIncomingDelivery`. + + :param context: The configured :class:`MailServerContext`. + :type canonicalFromSMTP: str or None + :param canonicalFromSMTP: The canonical domain which this message was received from. For example, if ``'gmail.com'`` is the configured canonical domain for ``'googlemail.com'`` and a message is received from the latter domain, then this would be set to the former. """ self.context = context - self.fromCanonical = fromCanonical + self.canon = context.canon + self.canonicalFromSMTP = canonicalFromSMTP + self.canonicalFromEmail = None + self.canonicalDomainRules = None + self.lines = [] self.nBytes = 0 self.ignoring = False + self.message = None + self.responder = autoresponder.SMTPAutoresponder() + self.responder.incoming = self + def lineReceived(self, line): """Called when we get another line of an incoming message.""" self.nBytes += len(line) @@ -161,9 +181,10 @@ class MailMessage(object): logging.debug("> %s", line.rstrip("\r\n")) def eomReceived(self): - """Called when we receive the end of a message.""" + """Tell the :ivar:`responder` to reply when we receive an EOM.""" if not self.ignoring: - self.reply() + self.message = self.getIncomingMessage() + self.responder.reply() return defer.succeed(None) def connectionLost(self): @@ -171,15 +192,15 @@ class MailMessage(object): pass def getIncomingMessage(self): - """Create and parse an :rfc:`2822` message object for all ``lines`` + """Create and parse an :rfc:`2822` message object for all :ivar:`lines` received thus far. :rtype: :api:`twisted.mail.smtp.rfc822.Message` :returns: A ``Message`` comprised of all lines received thus far. """ rawMessage = io.StringIO() - for ln in self.lines: - rawMessage.writelines(unicode(ln) + unicode('\n')) + for line in self.lines: + rawMessage.writelines(unicode(line) + unicode('\n')) rawMessage.seek(0) return smtp.rfc822.Message(rawMessage)

1 0

[bridgedb/develop] Split up Sphinx source files pertaining to email documentation.
by isis＠torproject.org 07 Jun '14

07 Jun '14

commit 1f1981743ffbf01edefe1f1a4c46e46903b57101 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Jun 2 20:28:45 2014 +0000 Split up Sphinx source files pertaining to email documentation. * CHANGE Sphinx source files to mirrow layout of the new email module. --- doc/sphinx/source/bridgedb.EmailServer.rst | 8 -------- doc/sphinx/source/bridgedb.email.dkim.rst | 8 ++++++++ doc/sphinx/source/bridgedb.email.request.rst | 8 ++++++++ doc/sphinx/source/bridgedb.email.server.rst | 8 ++++++++ doc/sphinx/source/bridgedb.email.templates.rst | 8 ++++++++ doc/sphinx/source/bridgedb.rst | 6 +++++- doc/sphinx/source/conf.py | 6 +++++- 7 files changed, 42 insertions(+), 10 deletions(-) diff --git a/doc/sphinx/source/bridgedb.EmailServer.rst b/doc/sphinx/source/bridgedb.EmailServer.rst deleted file mode 100644 index 8b8be9d..0000000 --- a/doc/sphinx/source/bridgedb.EmailServer.rst +++ /dev/null @@ -1,8 +0,0 @@ -bridgedb.EmailServer --------------------- - -.. automodule:: bridgedb.EmailServer - :members: - :undoc-members: - :private-members: - :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.email.dkim.rst b/doc/sphinx/source/bridgedb.email.dkim.rst new file mode 100644 index 0000000..6c1b5c4 --- /dev/null +++ b/doc/sphinx/source/bridgedb.email.dkim.rst @@ -0,0 +1,8 @@ +bridgedb.email.dkim +------------------- + +.. automodule:: bridgedb.email.dkim + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.email.request.rst b/doc/sphinx/source/bridgedb.email.request.rst new file mode 100644 index 0000000..48638bc --- /dev/null +++ b/doc/sphinx/source/bridgedb.email.request.rst @@ -0,0 +1,8 @@ +bridgedb.email.request +---------------------- + +.. automodule:: bridgedb.email.request + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.email.server.rst b/doc/sphinx/source/bridgedb.email.server.rst new file mode 100644 index 0000000..b2d59fb --- /dev/null +++ b/doc/sphinx/source/bridgedb.email.server.rst @@ -0,0 +1,8 @@ +bridgedb.email.server +--------------------- + +.. automodule:: bridgedb.email.server + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.email.templates.rst b/doc/sphinx/source/bridgedb.email.templates.rst new file mode 100644 index 0000000..e7f2a58 --- /dev/null +++ b/doc/sphinx/source/bridgedb.email.templates.rst @@ -0,0 +1,8 @@ +bridgedb.email.templates +------------------------ + +.. automodule:: bridgedb.email.templates + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.rst b/doc/sphinx/source/bridgedb.rst index e52c041..73e822b 100644 --- a/doc/sphinx/source/bridgedb.rst +++ b/doc/sphinx/source/bridgedb.rst @@ -11,7 +11,11 @@ BridgeDB Package and Module Documentation bridgedb.captcha bridgedb.crypto bridgedb.Dist - bridgedb.EmailServer + bridgedb.email + bridgedb.email.dkim + bridgedb.email.request + bridgedb.email.server + bridgedb.email.templates bridgedb.Filters bridgedb.HTTPServer bridgedb.Main diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index 161a471..3e320d9 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -32,7 +32,11 @@ import bridgedb.Bridges import bridgedb.Bucket import bridgedb.crypto import bridgedb.Dist -import bridgedb.EmailServer +import bridgedb.email +import bridgedb.email.dkim +import bridgedb.email.request +import bridgedb.email.server +import bridgedb.email.templates import bridgedb.Filters import bridgedb.HTTPServer import bridgedb.Main

1 0

[bridgedb/develop] Rename b.e.server.MailContext to b.e.server.MailServerContext.
by isis＠torproject.org 07 Jun '14

07 Jun '14

commit 2e5f94f4aaf4187f5bc566d3d9028fc10749fe05 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Jun 2 20:53:37 2014 +0000 Rename b.e.server.MailContext to b.e.server.MailServerContext. --- lib/bridgedb/email/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py index c526852..d27b525 100644 --- a/lib/bridgedb/email/server.py +++ b/lib/bridgedb/email/server.py @@ -46,7 +46,7 @@ from bridgedb.parse.addr import UnsupportedDomain from bridgedb.parse.addr import canonicalizeEmailDomain -class MailContext(object): +class MailServerContext(object): """Helper object that holds information used by email subsystem. :ivar str username: Reject any RCPT TO lines that aren't to this @@ -103,12 +103,12 @@ class MailContext(object): """Build a map for all email provider domains from which we will accept emails to their canonical domain name. - .. note:: Be sure that ``MailContext.domainRules`` and - ``MailContext.domainMap`` are set appropriately before calling + .. note:: Be sure that ``MailServerContext.domainRules`` and + ``MailServerContext.domainMap`` are set appropriately before calling this method. This method is automatically called during initialisation, and the - resulting domain map is stored as ``MailContext.canon``. + resulting domain map is stored as ``MailServerContext.canon``. :rtype: dict :returns: A dictionary which maps all domains and subdomains which we

1 0

[bridgedb/develop] Remove local hostaddr domain pass for SMTP checks.
by isis＠torproject.org 07 Jun '14

07 Jun '14

commit aca6dcdc0ab926d6ad353c781962a19fa392c890 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Jun 2 20:54:59 2014 +0000 Remove local hostaddr domain pass for SMTP checks. * CHANGE b.e.server.MailDelivery class to check incoming DNS names, never addresses. --- lib/bridgedb/email/server.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py index d27b525..249b7c7 100644 --- a/lib/bridgedb/email/server.py +++ b/lib/bridgedb/email/server.py @@ -87,7 +87,6 @@ class MailServerContext(object): self.username = (config.EMAIL_USERNAME or "bridges") self.hostname = socket.gethostname() - self.hostaddr = socket.gethostbyname(self.hostname) self.fromAddr = (config.EMAIL_FROM_ADDR or "bridges(a)torproject.org") self.smtpFromAddr = (config.EMAIL_SMTP_FROM_ADDR or self.fromAddr) self.smtpServerPort = (config.EMAIL_SMTP_PORT or 25) @@ -242,20 +241,22 @@ class MailDelivery(object): """ try: if ((origin.domain == self.context.hostname) or - (origin.domain == self.context.hostaddr)): - return origin + (origin.domain == smtp.DNSNAME)): + self.fromCanonicalSMTP = origin.domain else: - logging.debug("ORIGIN DOMAIN: %r" % origin.domain) + logging.debug("Canonicalizing client SMTP domain...") canonical = canonicalizeEmailDomain(origin.domain, self.context.canon) - logging.debug("Got canonical domain: %r" % canonical) - self.fromCanonical = canonical + logging.debug("Canonical SMTP domain: %r" % canonical) + self.fromCanonicalSMTP = canonical except UnsupportedDomain as error: logging.info(error) raise smtp.SMTPBadSender(origin.domain) except Exception as error: logging.exception(error) - return origin # This method *cannot* return None, or it'll cause a 503. + + # This method **cannot** return None, or it'll cause a 503 error. + return origin def validateTo(self, user): """Validate the SMTP ``RCPT TO:`` address for the incoming connection.

1 0

[bridgedb/develop] Move email unittest helpers to new bridgedb.test.email_helpers module.
by isis＠torproject.org 07 Jun '14

07 Jun '14

commit 7840735aca65d6ed1221e36dfb3bb4174f7f3c3b Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Jun 2 20:23:55 2014 +0000 Move email unittest helpers to new bridgedb.test.email_helpers module. * ADD a new mocked EmailBasedDistributor, called DummyEmailDistributorWithState, which keeps state (for sending out rate limiting email and such) to bridgedb.test.email_helpers. --- lib/bridgedb/test/email_helpers.py | 160 ++++++++++++++++++++++++++++++++ lib/bridgedb/test/test_email_server.py | 68 -------------- 2 files changed, 160 insertions(+), 68 deletions(-) diff --git a/lib/bridgedb/test/email_helpers.py b/lib/bridgedb/test/email_helpers.py new file mode 100644 index 0000000..59b3aa8 --- /dev/null +++ b/lib/bridgedb/test/email_helpers.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013, Isis Lovecruft +# (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: see LICENSE for licensing information + + +"""Helpers for testing the email distributor and its servers.""" + + +import io + +from bridgedb.Dist import IgnoreEmail +from bridgedb.Dist import TooSoonEmail +from bridgedb.persistent import Conf +from bridgedb.email.server import MailServerContext +from bridgedb.schedule import Unscheduled +from bridgedb.test.test_HTTPServer import DummyBridge + + +EMAIL_DIST = True +EMAIL_INCLUDE_FINGERPRINTS = True +EMAIL_GPG_SIGNING_ENABLED = True +EMAIL_GPG_SIGNING_KEY = 'TESTING.subkeys.sec' +EMAIL_DOMAIN_MAP = { + 'googlemail.com': 'gmail.com', + 'mail.google.com': 'gmail.com', +} +EMAIL_DOMAIN_RULES = { + 'gmail.com': ["ignore_dots", "dkim"], + 'example.com': [], + 'localhost': [], +} +EMAIL_DOMAINS = ["gmail.com", "example.com", "localhost"] +EMAIL_USERNAME = "bridges" +EMAIL_SMTP_HOST = "127.0.0.1" +EMAIL_SMTP_PORT = 25 +EMAIL_SMTP_FROM_ADDR = "bridges@localhost" +EMAIL_N_BRIDGES_PER_ANSWER = 3 +EMAIL_FROM_ADDR = "bridges@localhost" +EMAIL_BIND_IP = "127.0.0.1" +EMAIL_PORT = 5225 + +TEST_CONFIG_FILE = io.StringIO(unicode("""\ +EMAIL_DIST = %s +EMAIL_INCLUDE_FINGERPRINTS = %s +EMAIL_GPG_SIGNING_ENABLED = %s +EMAIL_GPG_SIGNING_KEY = %s +EMAIL_DOMAIN_MAP = %s +EMAIL_DOMAIN_RULES = %s +EMAIL_DOMAINS = %s +EMAIL_USERNAME = %s +EMAIL_SMTP_HOST = %s +EMAIL_SMTP_PORT = %s +EMAIL_SMTP_FROM_ADDR = %s +EMAIL_N_BRIDGES_PER_ANSWER = %s +EMAIL_FROM_ADDR = %s +EMAIL_BIND_IP = %s +EMAIL_PORT = %s +""" % (repr(EMAIL_DIST), + repr(EMAIL_INCLUDE_FINGERPRINTS), + repr(EMAIL_GPG_SIGNING_ENABLED), + repr(EMAIL_GPG_SIGNING_KEY), + repr(EMAIL_DOMAIN_MAP), + repr(EMAIL_DOMAIN_RULES), + repr(EMAIL_DOMAINS), + repr(EMAIL_USERNAME), + repr(EMAIL_SMTP_HOST), + repr(EMAIL_SMTP_PORT), + repr(EMAIL_SMTP_FROM_ADDR), + repr(EMAIL_N_BRIDGES_PER_ANSWER), + repr(EMAIL_FROM_ADDR), + repr(EMAIL_BIND_IP), + repr(EMAIL_PORT)))) + + +def _createConfig(configFile=TEST_CONFIG_FILE): + configuration = {} + TEST_CONFIG_FILE.seek(0) + compiled = compile(configFile.read(), '<string>', 'exec') + exec compiled in configuration + config = Conf(**configuration) + return config + +def _createMailServerContext(config=None, distributor=None): + if not config: + config = _createConfig() + + if not distributor: + distributor = DummyEmailDistributor( + domainmap=config.EMAIL_DOMAIN_MAP, + domainrules=config.EMAIL_DOMAIN_RULES) + + context = MailServerContext(config, distributor, Unscheduled()) + return context + + +class DummyEmailDistributor(object): + """A mocked :class:`bridgedb.Dist.EmailBasedDistributor` which is used to + test :class:`bridgedb.EmailServer`. + """ + + def __init__(self, key=None, domainmap=None, domainrules=None, + answerParameters=None): + """None of the parameters are really used, ― they are just there to retain an + identical method signature. + """ + self.key = self.__class__.__name__ + self.domainmap = domainmap + self.domainrules = domainrules + self.answerParameters = answerParameters + + def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None, + countryCode=None, bridgeFilterRules=None): + return [DummyBridge() for _ in xrange(N)] + + def cleanDatabase(self): + pass + + +class DummyEmailDistributorWithState(DummyEmailDistributor): + """A mocked :class:`bridgedb.Dist.EmailBasedDistributor` which raises + :exc:`bridgedb.Dist.TooSoonEmail` on the second email and + :exc:`bridgedb.Dist.IgnoreEmail` on the third. + + Note that the state tracking is done in a really dumb way. For example, we + currently don't consider requests for help text or GnuPG keys to be a + "real" request, so in the real email distributor they won't trigger either + a TooSoonEmail or IgnoreEmail. Here we only track the total number of + *any* type of request per client. + """ + + def __init__(self, *args, **kwargs): + super(DummyEmailDistributorWithState, self).__init__() + self.alreadySeen = {} + + def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None, + countryCode=None, bridgeFilterRules=None): + # Keep track of the number of times we've seen a client. + if not emailaddress in self.alreadySeen.keys(): + self.alreadySeen[emailaddress] = 0 + self.alreadySeen[emailaddress] += 1 + + if self.alreadySeen[emailaddress] <= 1: + return [DummyBridge() for _ in xrange(N)] + elif self.alreadySeen[emailaddress] == 2: + raise TooSoonEmail( + "Seen client '%s' %d times" + % (emailaddress, self.alreadySeen[emailaddress]), + emailaddress) + else: + raise IgnoreEmail( + "Seen client '%s' %d times" + % (emailaddress, self.alreadySeen[emailaddress]), + emailaddress) diff --git a/lib/bridgedb/test/test_email_server.py b/lib/bridgedb/test/test_email_server.py index 7aff08e..198cb27 100644 --- a/lib/bridgedb/test/test_email_server.py +++ b/lib/bridgedb/test/test_email_server.py @@ -40,74 +40,6 @@ from twisted.test import proto_helpers from twisted.trial import unittest -TEST_CONFIG_FILE = io.StringIO(unicode("""\ -EMAIL_DIST = True -EMAIL_INCLUDE_FINGERPRINTS = True -EMAIL_GPG_SIGNING_ENABLED = True -EMAIL_GPG_SIGNING_KEY = 'TESTING.subkeys.sec' -EMAIL_DOMAIN_MAP = { - 'googlemail.com': 'gmail.com', - 'mail.google.com': 'gmail.com', -} -EMAIL_DOMAIN_RULES = { - 'gmail.com': ["ignore_dots", "dkim"], - 'example.com': [], - 'localhost': [], -} -EMAIL_DOMAINS = ["gmail.com", "example.com", "localhost"] -EMAIL_USERNAME = "bridges" -EMAIL_SMTP_HOST = "127.0.0.1" -EMAIL_SMTP_PORT = 25 -EMAIL_SMTP_FROM_ADDR = "bridges@localhost" -EMAIL_N_BRIDGES_PER_ANSWER = 3 -EMAIL_FROM_ADDR = "bridges@localhost" -EMAIL_BIND_IP = "127.0.0.1" -EMAIL_PORT = 5225 -""")) - - -def _createConfig(configFile=TEST_CONFIG_FILE): - configuration = {} - TEST_CONFIG_FILE.seek(0) - compiled = compile(configFile.read(), '<string>', 'exec') - exec compiled in configuration - config = Conf(**configuration) - return config - -def _createMailContext(config=None, distributor=None): - if not config: - config = _createConfig() - - if not distributor: - distributor = DummyEmailDistributor( - domainmap=config.EMAIL_DOMAIN_MAP, - domainrules=config.EMAIL_DOMAIN_RULES) - - context = server.MailContext(config, distributor, Unscheduled()) - return context - - -class DummyEmailDistributor(object): - """A mocked :class:`bridgedb.Dist.EmailBasedDistributor` which is used to - test :class:`bridgedb.EmailServer`. - """ - - def __init__(self, key=None, domainmap=None, domainrules=None, - answerParameters=None): - """None of the parameters are really used, ― they are just there to retain an - identical method signature. - """ - self.key = self.__class__.__name__ - self.domainmap = domainmap - self.domainrules = domainrules - self.answerParameters = answerParameters - - def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None, - countryCode=None, bridgeFilterRules=None): - return [DummyBridge() for _ in xrange(N)] - - def cleanDatabase(self): - pass class CheckDKIMTests(unittest.TestCase):

1 0

[bridgedb/develop] Rename b.e.server.MailDelivery → b.e.server.SMTPIncomingDelivery.
by isis＠torproject.org 07 Jun '14

07 Jun '14

commit f176af73c1e50394d8282ab9676eb8c86b28715c Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Jun 2 21:03:34 2014 +0000 Rename b.e.server.MailDelivery → b.e.server.SMTPIncomingDelivery. * ADD more documentation to b.e.server.SMTPIncomingDelivery. --- lib/bridgedb/email/server.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py index a493be0..a23bbe8 100644 --- a/lib/bridgedb/email/server.py +++ b/lib/bridgedb/email/server.py @@ -205,17 +205,30 @@ class SMTPMessage(object): return smtp.rfc822.Message(rawMessage) - - - - -class MailDelivery(object): - """Plugs into Twisted Mail and handles SMTP commands.""" +class SMTPIncomingDelivery(smtp.SMTP): + """Plugs into :class:`SMTPIncomingServerFactory` and handles SMTP commands + for incoming connections. + + :type context: :class:`MailServerContext` + :ivar context: A context containing SMTP/Email configuration settings. + :ivar deferred: A :api:`deferred <twisted.internet.defer.Deferred>` which + will be returned when :meth:`reply` is called. Additional callbacks + may be set on this deferred in order to schedule additional actions + when the response is being sent. + :type fromCanonicalSMTP: str or ``None`` + :ivar fromCanonicalSMTP: If set, this is the canonicalized domain name of + the address we received from incoming connection's ``MAIL FROM:``. + """ implements(smtp.IMessageDelivery) - def setBridgeDBContext(self, context): - self.context = context - self.fromCanonical = None + context = None + deferred = defer.Deferred() + fromCanonicalSMTP = None + + @classmethod + def setContext(cls, context): + """Set our :ivar:`context` to a new :class:`MailServerContext.""" + cls.context = context def receivedHeader(self, helo, origin, recipients): """Create the ``Received:`` header for an incoming email. @@ -244,7 +257,7 @@ class MailDelivery(object): :func:`socket.gethostname`) or our own FQDN, allow the connection. Otherwise, if the ``MAIL FROM:`` domain has a canonical domain in our - mapping (taken from :ivar:`context.canon <MailContext.canon>`, which + mapping (taken from :ivar:`context.canon <MailServerContext.canon>`, which is taken in turn from the ``EMAIL_DOMAIN_MAP``), then our :ivar:`fromCanonicalSMTP` is set to that domain. @@ -317,7 +330,7 @@ class MailDelivery(object): if beforePlus != ourAddress.local: raise smtp.SMTPBadRcpt(str(recipient)) - return lambda: MailMessage(self.context, self.fromCanonical) + return lambda: SMTPMessage(self.context, self.fromCanonicalSMTP) class MailFactory(smtp.SMTPFactory):

1 0

[bridgedb/develop] Rewrite 'Received:' header generation function to make more standard headers.
by isis＠torproject.org 07 Jun '14

07 Jun '14

commit ede14d2af4b72caaad46a6f94bde7eb2f6be4a60 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Jun 2 21:04:38 2014 +0000 Rewrite 'Received:' header generation function to make more standard headers. --- lib/bridgedb/email/server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py index a23bbe8..e575eab 100644 --- a/lib/bridgedb/email/server.py +++ b/lib/bridgedb/email/server.py @@ -240,11 +240,11 @@ class SMTPIncomingDelivery(smtp.SMTP): :type recipients: list :param recipients: A list of :api:`twisted.mail.smtp.User` instances. """ - cameFrom = "%s (%s [%s])" % (helo[0] or origin, helo[0], helo[1]) - cameFor = ', '.join(["<{0}>".format(recp.dest) for recp in recipients]) - hdr = str("Received: from %s for %s; %s" - % (cameFrom, cameFor, smtp.rfc822date())) - return hdr + helo_ = ' helo={0}'.format(helo[0]) if helo[0] else '' + from_ = 'from %s ([%s]%s)' % (helo[0], helo[1], helo_) + by_ = 'by %s with BridgeDB (%s)' % (smtp.DNSNAME, __version__) + for_ = 'for %s; %s ' % (' '.join(map(str, recipients)), rfc822date()) + return str('Received: %s\n\t%s\n\t%s' % (from_, by_, for_)) def validateFrom(self, helo, origin): """Validate the ``MAIL FROM:`` address on the incoming SMTP connection.

1 0

[bridgedb/develop] Add unittests for bridgedb.email.autoresponder module.
by isis＠torproject.org 07 Jun '14

07 Jun '14

commit 778ae586fd635902a2f3813566f4c9d282ae308f Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Jun 2 21:09:22 2014 +0000 Add unittests for bridgedb.email.autoresponder module. --- lib/bridgedb/test/test_email_autoresponder.py | 547 +++++++++++++++++++++++++ 1 file changed, 547 insertions(+) diff --git a/lib/bridgedb/test/test_email_autoresponder.py b/lib/bridgedb/test/test_email_autoresponder.py new file mode 100644 index 0000000..ed22a72 --- /dev/null +++ b/lib/bridgedb/test/test_email_autoresponder.py @@ -0,0 +1,547 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013, Isis Lovecruft +# (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: 3-Clause BSD, see LICENSE for licensing information + +"""Unittests for the :mod:`bridgedb.email.autoresponder` module.""" + +from __future__ import print_function + +import io +import os +import shutil + +from twisted.internet import defer +from twisted.mail.smtp import Address +from twisted.python.failure import Failure +from twisted.trial import unittest +from twisted.test import proto_helpers + +from bridgedb.email import autoresponder +from bridgedb.email.server import SMTPMessage +from bridgedb.Dist import TooSoonEmail +from bridgedb.test.email_helpers import _createConfig +from bridgedb.test.email_helpers import _createMailServerContext +from bridgedb.test.email_helpers import DummyEmailDistributorWithState + + +class CreateResponseBodyTests(unittest.TestCase): + """Tests for :func:`bridgedb.email.autoresponder.createResponseBody`.""" + + def _moveGPGTestKeyfile(self): + here = os.getcwd() + topDir = here.rstrip('_trial_temp') + self.gpgFile = os.path.join(topDir, 'gnupghome', 'TESTING.subkeys.sec') + self.gpgMoved = os.path.join(here, 'TESTING.subkeys.sec') + shutil.copy(self.gpgFile, self.gpgMoved) + + def setUp(self): + """Create fake email, distributor, and associated context data.""" + self._moveGPGTestKeyfile() + self.toAddress = "user(a)example.com" + self.config = _createConfig() + self.ctx = _createMailServerContext(self.config) + self.distributor = self.ctx.distributor + + def _getIncomingLines(self, clientAddress="user(a)example.com"): + """Generate the lines of an incoming email from **clientAddress**.""" + self.toAddress = Address(clientAddress) + lines = [ + "From: %s" % clientAddress, + "To: bridges@localhost", + "Subject: testing", + "", + "get bridges", + ] + return lines + + def test_createResponseBody_getKey(self): + """A request for 'get key' should receive our GPG key.""" + lines = self._getIncomingLines() + lines[4] = "get key" + ret = autoresponder.createResponseBody(lines, self.ctx, self.toAddress) + self.assertSubstring('-----BEGIN PGP PUBLIC KEY BLOCK-----', ret) + + def test_createResponseBody_bridges_invalid(self): + """An invalid request for 'transport obfs3' should get help text.""" + lines = self._getIncomingLines("testing@localhost") + lines[4] = "transport obfs3" + ret = autoresponder.createResponseBody(lines, self.ctx, self.toAddress) + self.assertSubstring("COMMANDs", ret) + + def test_createResponseBody_bridges_obfs3(self): + """A request for 'get transport obfs3' should receive a response.""" + lines = self._getIncomingLines("testing@localhost") + lines[4] = "get transport obfs3" + ret = autoresponder.createResponseBody(lines, self.ctx, self.toAddress) + self.assertSubstring("Here are your bridges", ret) + self.assertSubstring("obfs3", ret) + + def test_createResponseBody_bridges_obfsobfswebz(self): + """We should only pay attention to the *last* in a crazy request.""" + lines = self._getIncomingLines("testing@localhost") + lines[4] = "get unblocked webz" + lines.append("get transport obfs2") + lines.append("get transport obfs3") + ret = autoresponder.createResponseBody(lines, self.ctx, self.toAddress) + self.assertSubstring("Here are your bridges", ret) + self.assertSubstring("obfs3", ret) + + def test_createResponseBody_bridges_obfsobfswebzipv6(self): + """We should *still* only pay attention to the *last* request.""" + lines = self._getIncomingLines("testing@localhost") + lines[4] = "transport obfs3" + lines.append("get unblocked webz") + lines.append("get ipv6") + lines.append("get transport obfs2") + ret = autoresponder.createResponseBody(lines, self.ctx, self.toAddress) + self.assertSubstring("Here are your bridges", ret) + self.assertSubstring("obfs2", ret) + + def test_createResponseBody_two_requests_TooSoonEmail(self): + """The same client making two requests in a row should receive a + rate-limit warning for the second response. + """ + # Set up a mock distributor which keeps state: + dist = DummyEmailDistributorWithState() + ctx = _createMailServerContext(self.config, dist) + + lines = self._getIncomingLines("testing@localhost") + first = autoresponder.createResponseBody(lines, ctx, self.toAddress) + self.assertSubstring("Here are your bridges", first) + second = autoresponder.createResponseBody(lines, ctx, self.toAddress) + self.assertSubstring("Please slow down", second) + + def test_createResponseBody_three_requests_TooSoonEmail(self): + """Alice making a request, next Bob making a request, and then Alice again, + should result in both of their first requests getting them bridges, + and then Alice's second request gets her a rate-limit warning email. + """ + # Set up a mock distributor which keeps state: + dist = DummyEmailDistributorWithState() + ctx = _createMailServerContext(self.config, dist) + + aliceLines = self._getIncomingLines("alice@localhost") + aliceFirst = autoresponder.createResponseBody(aliceLines, ctx, + self.toAddress) + self.assertSubstring("Here are your bridges", aliceFirst) + + bobLines = self._getIncomingLines("bob@localhost") + bobFirst = autoresponder.createResponseBody(bobLines, ctx, + self.toAddress) + self.assertSubstring("Here are your bridges", bobFirst) + + aliceSecond = autoresponder.createResponseBody(aliceLines, ctx, + self.toAddress) + self.assertSubstring("Please slow down", aliceSecond) + + def test_createResponseBody_three_requests_IgnoreEmail(self): + """The same client making three requests in a row should receive a + rate-limit warning for the second response, and then nothing for every + request thereafter. + """ + # Set up a mock distributor which keeps state: + dist = DummyEmailDistributorWithState() + ctx = _createMailServerContext(self.config, dist) + + lines = self._getIncomingLines("testing@localhost") + first = autoresponder.createResponseBody(lines, ctx, self.toAddress) + self.assertSubstring("Here are your bridges", first) + second = autoresponder.createResponseBody(lines, ctx, self.toAddress) + self.assertSubstring("Please slow down", second) + third = autoresponder.createResponseBody(lines, ctx, self.toAddress) + self.assertIsNone(third) + fourth = autoresponder.createResponseBody(lines, ctx, self.toAddress) + self.assertIsNone(fourth) + + +class EmailResponseTests(unittest.TestCase): + """Tests for ``generateResponse()`` and ``EmailResponse``.""" + + def setUp(self): + self.fromAddr = "bridges(a)torproject.org" + self.clientAddr = "user(a)example.com" + self.body = """\ +People think that time is strictly linear, but, in reality, it's actually just +a ball of timey-wimey, wibbly-warbly... stuff.""" + + def tearDown(self): + autoresponder.safelog.safe_logging = True + + def test_EmailResponse_generateResponse(self): + response = autoresponder.generateResponse(self.fromAddr, + self.clientAddr, + self.body) + self.assertIsInstance(response, autoresponder.EmailResponse) + + def test_EmailResponse_generateResponse_noSafelog(self): + autoresponder.safelog.safe_logging = False + response = autoresponder.generateResponse(self.fromAddr, + self.clientAddr, + self.body) + self.assertIsInstance(response, autoresponder.EmailResponse) + + def test_EmailResponse_generateResponse_mailfile(self): + response = autoresponder.generateResponse(self.fromAddr, + self.clientAddr, + self.body) + self.assertIsInstance(response.mailfile, (io.BytesIO, io.StringIO)) + + def test_EmailResponse_generateResponse_withInReplyTo(self): + response = autoresponder.generateResponse(self.fromAddr, + self.clientAddr, + self.body, + messageID="NSA") + contents = str(response.readContents()).replace('\x00', '') + self.assertIsInstance(response.mailfile, (io.BytesIO, io.StringIO)) + self.assertSubstring("In-Reply-To: NSA", contents) + + def test_EmailResponse_generateResponse_readContents(self): + response = autoresponder.generateResponse(self.fromAddr, + self.clientAddr, + self.body) + contents = str(response.readContents()).replace('\x00', '') + self.assertSubstring('timey-wimey, wibbly-warbly... stuff.', contents) + + def test_EmailResponse_additionalHeaders(self): + response = autoresponder.EmailResponse() + response.writeHeaders(self.fromAddr, self.clientAddr, + subject="Re: echelon", inReplyTo="NSA", + X_been_there="They were so 2004") + contents = str(response.readContents()).replace('\x00', '') + self.assertIsInstance(response.mailfile, (io.BytesIO, io.StringIO)) + self.assertSubstring("In-Reply-To: NSA", contents) + self.assertSubstring("X-been-there: They were so 2004", contents) + + def test_EmailResponse_close(self): + """Calling EmailResponse.close() should close the ``mailfile`` and set + ``closed=True``. + """ + response = autoresponder.EmailResponse() + self.assertEqual(response.closed, False) + response.close() + self.assertEqual(response.closed, True) + self.assertRaises(ValueError, response.write, self.body) + + def test_EmailResponse_read(self): + """Calling EmailResponse.read() should read bytes from the file.""" + response = autoresponder.EmailResponse() + response.write(self.body) + response.rewind() + contents = str(response.read()).replace('\x00', '') + # The newlines in the email body should have been replaced with + # ``EmailResponse.delimiter``. + delimited = self.body.replace('\n', response.delimiter) \ + + response.delimiter + self.assertEqual(delimited, contents) + + def test_EmailResponse_read_three_bytes(self): + """EmailResponse.read(3) should read three bytes from the file.""" + response = autoresponder.EmailResponse() + response.write(self.body) + response.rewind() + contents = str(response.read(3)).replace('\x00', '') + self.assertEqual(contents, self.body[:3]) + + def test_EmailResponse_write(self): + """Calling EmailResponse.write() should write to the mailfile.""" + response = autoresponder.EmailResponse() + response.write(self.body) + contents = str(response.readContents()).replace('\x00', '') + # The newlines in the email body should have been replaced with + # ``EmailResponse.delimiter``. + delimited = self.body.replace('\n', response.delimiter) \ + + response.delimiter + self.assertEqual(delimited, contents) + + def test_EmailResponse_write_withRetNewlines(self): + """Calling EmailResponse.write() with '\r\n' in the lines should call + writelines(), which splits up the lines and then calls write() again. + """ + response = autoresponder.EmailResponse() + response.write(self.body.replace('\n', '\r\n')) + contents = str(response.readContents()).replace('\x00', '') + # The newlines in the email body should have been replaced with + # ``EmailResponse.delimiter``. + delimited = self.body.replace('\n', response.delimiter) \ + + response.delimiter + self.assertEqual(delimited, contents) + + def test_EmailResponse_writelines_list(self): + """Calling EmailResponse.writelines() with a list should write the + concatenated contents of the list into the mailfile. + """ + response = autoresponder.EmailResponse() + response.writelines(self.body.split('\n')) + contents = str(response.readContents()).replace('\x00', '') + # The newlines in the email body should have been replaced with + # ``EmailResponse.delimiter``. + delimited = self.body.replace('\n', response.delimiter) \ + + response.delimiter + self.assertEqual(delimited, contents) + + +class SMTPAutoresponderTests(unittest.TestCase): + """Unittests for :class:`bridgedb.email.autoresponder.SMTPAutoresponder`.""" + + def setUp(self): + self.config = _createConfig() + self.context = _createMailServerContext(self.config) + self.message = SMTPMessage(self.context) + + def _getIncomingLines(self, clientAddress="user(a)example.com"): + """Generate the lines of an incoming email from **clientAddress**.""" + lines = [ + "From: %s" % clientAddress, + "To: bridges@localhost", + "Subject: testing", + "", + "get bridges", + ] + self.message.lines = lines + + def _setUpResponder(self): + """Set up the incoming message of our autoresponder. + + This is necessary because normally our SMTP server acts as a line + protocol, waiting for an EOM which sets off a chain of deferreds + resulting in the autoresponder sending out the response. This should + be called after :meth:`_getIncomingLines` so that we can hook into the + SMTP protocol without actually triggering all the deferreds. + """ + self.message.message = self.message.getIncomingMessage() + self.responder = self.message.responder + # The following are needed to provide client disconnection methods for + # the call to ``twisted.mail.smtp.SMTPClient.sendError`` in + # ``bridgedb.email.autoresponder.SMTPAutoresponder.sendError``: + #protocol = proto_helpers.AccumulatingProtocol() + #transport = proto_helpers.StringTransportWithDisconnection() + self.tr = proto_helpers.StringTransportWithDisconnection() + # Set the transport's protocol, because + # StringTransportWithDisconnection is a bit janky: + self.tr.protocol = self.responder + self.responder.makeConnection(self.tr) + + def test_SMTPAutoresponder_getMailFrom_notbridgedb_at_yikezors_dot_net(self): + """SMTPAutoresponder.getMailFrom() for an incoming email sent to any email + address other than the one we're listening for should return our + configured address, not the one in the incoming email. + """ + self._getIncomingLines() + self.message.lines[1] = 'To: notbridgedb(a)yikezors.net' + self._setUpResponder() + recipient = str(self.responder.getMailFrom()) + self.assertEqual(recipient, self.context.fromAddr) + + def test_SMTPAutoresponder_getMailFrom_givemebridges_at_seriously(self): + """SMTPAutoresponder.getMailFrom() for an incoming email sent to any email + address other than the one we're listening for should return our + configured address, not the one in the incoming email. + """ + self._getIncomingLines() + self.message.lines[1] = 'To: givemebridges(a)serious.ly' + self._setUpResponder() + recipient = str(self.responder.getMailFrom()) + self.assertEqual(recipient, self.context.fromAddr) + + def test_SMTPAutoresponder_getMailFrom_bad_address(self): + """SMTPAutoresponder.getMailFrom() for an incoming email sent to a malformed + email address should log an smtp.AddressError and then return our + configured email address. + """ + self._getIncomingLines() + self.message.lines[1] = 'To: ><@><<<>>.foo' + self._setUpResponder() + recipient = str(self.responder.getMailFrom()) + self.assertEqual(recipient, self.context.fromAddr) + + def test_SMTPAutoresponder_getMailFrom_plus_address(self): + """SMTPAutoresponder.getMailFrom() for an incoming email sent with a valid + plus address should respond. + """ + self._getIncomingLines() + ours = Address(self.context.fromAddr) + plus = '@'.join([ours.local + '+zh_cn', ours.domain]) + self.message.lines[1] = 'To: {0}'.format(plus) + self._setUpResponder() + recipient = str(self.responder.getMailFrom()) + self.assertEqual(recipient, plus) + + def test_SMTPAutoresponder_getMailFrom_getbridges_at_localhost(self): + """SMTPAutoresponder.getMailFrom() for an incoming email sent with + 'getbridges+zh_cn@localhost' should be responded to from the default + address. + """ + self._getIncomingLines() + ours = Address(self.context.fromAddr) + plus = '@'.join(['get' + ours.local + '+zh_cn', ours.domain]) + self.message.lines[1] = 'To: {0}'.format(plus) + self._setUpResponder() + recipient = str(self.responder.getMailFrom()) + self.assertEqual(recipient, self.context.fromAddr) + + def test_SMTPAutoresponder_getMailTo_UnsupportedDomain(self): + """getMailTo() should catch emails from UnsupportedDomains.""" + emailFrom = 'some.dude(a)un.support.ed' + self._getIncomingLines(emailFrom) + self._setUpResponder() + clients = self.responder.getMailTo() + self.assertIsInstance(clients, list, ( + "Returned value of SMTPAutoresponder.getMailTo() isn't a list! " + "Type: %s" % type(clients))) + # The client was from an unsupported domain; they shouldn't be in the + # clients list: + self.assertEqual(len(clients), 0) + + def test_SMTPAutoresponder_reply_noFrom(self): + """A received email without a "From:" or "Sender:" header shouldn't + receive a response. + """ + self._getIncomingLines() + self.message.lines[0] = "" + self._setUpResponder() + ret = self.responder.reply() + self.assertIsInstance(ret, defer.Deferred) + + def test_SMTPAutoresponder_reply_badAddress(self): + """Don't respond to RFC2822 malformed source addresses.""" + self._getIncomingLines("testing*.?\"@example.com") + self._setUpResponder() + ret = self.responder.reply() + # This will call ``self.responder.reply()``: + #ret = self.responder.incoming.eomReceived() + self.assertIsInstance(ret, defer.Deferred) + + def test_SMTPAutoresponder_reply_anotherBadAddress(self): + """Don't respond to RFC2822 malformed source addresses.""" + self._getIncomingLines("Mallory <>>@example.com") + self._setUpResponder() + ret = self.responder.reply() + self.assertIsInstance(ret, defer.Deferred) + + def test_SMTPAutoresponder_reply_invalidDomain(self): + """Don't respond to RFC2822 malformed source addresses.""" + self._getIncomingLines("testing(a)exa#mple.com") + self._setUpResponder() + ret = self.responder.reply() + self.assertIsInstance(ret, defer.Deferred) + + def test_SMTPAutoresponder_reply_anotherInvalidDomain(self): + """Don't respond to RFC2822 malformed source addresses.""" + self._getIncomingLines("testing(a)exam+ple.com") + self._setUpResponder() + ret = self.responder.reply() + self.assertIsInstance(ret, defer.Deferred) + + def test_SMTPAutoresponder_reply_DKIM_badDKIMheader(self): + """An email with an 'X-DKIM-Authentication-Result:' header appended + after the body should not receive a response. + """ + self._getIncomingLines("testing(a)gmail.com") + self.message.lines.append("X-DKIM-Authentication-Result: ") + self._setUpResponder() + ret = self.responder.reply() + self.assertIsInstance(ret, defer.Deferred) + + def test_SMTPAutoresponder_reply_goodDKIMheader(self): + """An email with a good DKIM header should be responded to.""" + self._getIncomingLines("testing(a)gmail.com") + self.message.lines.insert(3, "X-DKIM-Authentication-Result: pass") + self._setUpResponder() + ret = self.responder.reply() + self.assertIsInstance(ret, defer.Deferred) + + def test_SMTPAutoresponder_reply_transport_invalid(self): + """An invalid request for 'transport obfs3' should get help text.""" + #self.skip = True + #raise unittest.SkipTest("We need to fake the reactor for this one") + + def cb(success): + pass + self._getIncomingLines("testing(a)example.com") + self.message.lines[4] = "transport obfs3" + self._setUpResponder() + ret = self.responder.reply() + self.assertIsInstance(ret, defer.Deferred) + #self.assertSubstring("COMMANDs", ret) + print(self.tr.value()) + return ret + + def test_SMTPAutoresponder_reply_transport_valid(self): + """An valid request for 'get transport obfs3' should get obfs3.""" + #self.skip = True + #raise unittest.SkipTest("We need to fake the reactor for this one") + + self._getIncomingLines("testing(a)example.com") + self.message.lines[4] = "transport obfs3" + self._setUpResponder() + ret = self.responder.reply() + self.assertIsInstance(ret, defer.Deferred) + #self.assertSubstring("obfs3", ret) + print(self.tr.value()) + return ret + + def test_SMTPAutoresponder_sentMail(self): + """``SMTPAutoresponder.sendMail()`` should handle successes from an + :api:`twisted.mail.smtp.SMTPSenderFactory`. + """ + success = (1, [('me(a)myaddress.com', 250, 'OK',)]) + self._getIncomingLines() + self._setUpResponder() + self.responder.sentMail(success) + + def test_SMTPAutoresponder_sendError_fail(self): + """``SMTPAutoresponder.sendError()`` should handle failures.""" + fail = Failure(ValueError('This failure was sent on purpose.')) + self._getIncomingLines() + self._setUpResponder() + self.responder.sendError(fail) + + def test_SMTPAutoresponder_sendError_exception(self): + """``SMTPAutoresponder.sendError()`` should handle exceptions.""" + error = ValueError('This error was sent on purpose.') + self._getIncomingLines() + self._setUpResponder() + self.responder.sendError(error) + + def test_SMTPAutoresponder_runChecks_RCPTTO_From_mismatched_domain(self): + """runChecks() should catch emails where the SMTP 'MAIL FROM:' command + reported being from an email address at one supported domain and the + email's 'From:' header reported another domain. + """ + smtpFrom = 'not.an.evil.bot(a)yahoo.com' + emailFrom = Address('not.an.evil.bot(a)gmail.com') + self._getIncomingLines(str(emailFrom)) + self._setUpResponder() + self.responder.incoming.canonicalFromSMTP = smtpFrom + self.assertFalse(self.responder.runChecks(emailFrom)) + + def test_SMTPAutoresponder_runChecks_RCPTTO_From_mismatched_username(self): + """runChecks() should catch emails where the SMTP 'MAIL FROM:' command + reported being from an email address and the email's 'From:' header + reported another email address, even if the only the username part is + mismatched. + """ + smtpFrom = 'feidanchaoren0001(a)gmail.com' + emailFrom = Address('feidanchaoren0038(a)gmail.com') + self._getIncomingLines(str(emailFrom)) + self._setUpResponder() + self.responder.incoming.canonicalFromSMTP = smtpFrom + self.assertFalse(self.responder.runChecks(emailFrom)) + + def test_SMTPAutoresponder_runChecks_badDKIM(self): + """runChecks() should catch emails with bad DKIM headers for canonical + domains which we've configured to check DKIM verification results for. + """ + emailFrom = Address('dkimlikewat(a)gmail.com') + header = "X-DKIM-Authentication-Results: dunno" + self._getIncomingLines(str(emailFrom)) + self.message.lines.insert(3, header) + self._setUpResponder() + self.assertFalse(self.responder.runChecks(emailFrom))

1 0

[bridgedb/develop] Refactor b.e.server.MailFactory.
by isis＠torproject.org 07 Jun '14

07 Jun '14

commit 3ae9de3621fb2cc158482dcf0c3d5a4d37073a86 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Jun 2 21:05:29 2014 +0000 Refactor b.e.server.MailFactory. * CHANGE b.e.server.MailFactory into two separate classes, b.e.server.SMTPIncomingDeliveryFactory and b.e.server.SMTPIncomingServerFactory. This will enable us to be able to keep state for multiple emails coming in over a single connection, if necessary. * CHANGE b.e.server.addServer to use the new classes. --- lib/bridgedb/email/server.py | 81 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py index e575eab..6bc7300 100644 --- a/lib/bridgedb/email/server.py +++ b/lib/bridgedb/email/server.py @@ -333,23 +333,75 @@ class SMTPIncomingDelivery(smtp.SMTP): return lambda: SMTPMessage(self.context, self.fromCanonicalSMTP) -class MailFactory(smtp.SMTPFactory): - """Plugs into Twisted Mail; creates a new MailDelivery whenever we get - a connection on the SMTP port.""" +class SMTPIncomingDeliveryFactory(object): + """Factory for :class:`SMTPIncomingDelivery`s. + + This class is used to distinguish between different messages delivered + over the same connection. This can be used to optimize delivery of a + single message to multiple recipients, something which cannot be done by + :api:`IMessageDelivery <twisted.mail.smtp.IMessageDelivery>` implementors + due to their lack of information. + + :ivar context: A :class:`MailServerContext` for storing configuration settings. + :ivar delivery: A :class:`SMTPIncomingDelivery` to deliver incoming + SMTP messages to. + """ + implements(smtp.IMessageDeliveryFactory) + + context = None + delivery = SMTPIncomingDelivery - def __init__(self, context=None, **kw): - smtp.SMTPFactory.__init__(self, **kw) - self.delivery = MailDelivery() - if context: - self.setBridgeDBContext(context) + def __init__(self): + logging.debug("%s created." % self.__class__.__name__) - def setBridgeDBContext(self, context): - self.context = context - self.delivery.setBridgeDBContext(context) + @classmethod + def setContext(cls, context): + """Set our :ivar:`context` and the context for our :ivar:`delivery`.""" + cls.context = context + cls.delivery.setContext(cls.context) + + def getMessageDelivery(self): + """Get a new :class:`SMTPIncomingDelivery` instance.""" + return self.delivery() + + +class SMTPIncomingServerFactory(smtp.SMTPFactory): + """Plugs into :api:`twisted.mail.smtp.SMTPFactory`; creates a new + :class:`SMTPMessageDelivery`, which handles response email automation, + whenever we get a incoming connection on the SMTP port. + + .. warning:: My :ivar:`context` isn't an OpenSSL context, as is used for + the :api:`twisted.mail.smtp.ESMTPSender` + + :ivar context: A :class:`MailServerContext` for storing configuration settings. + :ivar deliveryFactory: A :class:`SMTPIncomingDeliveryFactory` for + producing :class:`SMTPIncomingDelivery`s. + :ivar domain: :api:`Our FQDN <twisted.mail.smtp.DNSNAME>`. + :ivar int timeout: The number of seconds to wait, after the last chunk of + data was received, before raising a + :api:`SMTPTimeoutError <twisted.mail.smtp.SMTPTimeoutError>` for an + incoming connection. + :ivar protocol: :api:`SMTP <twisted.mail.smtp.SMTP>` + """ + + context = None + deliveryFactory = SMTPIncomingDeliveryFactory + + def __init__(self, **kwargs): + smtp.SMTPFactory.__init__(self, **kwargs) + self.deliveryFactory = self.deliveryFactory() + + @classmethod + def setContext(cls, context): + """Set :ivar:`context` and :ivar:`deliveryFactory`.context.""" + cls.context = context + cls.deliveryFactory.setContext(cls.context) def buildProtocol(self, addr): p = smtp.SMTPFactory.buildProtocol(self, addr) - p.delivery = self.delivery + self.deliveryFactory.transport = p.transport # XXX is this set yet? + p.factory = self + p.deliveryFactory = self.deliveryFactory return p @@ -364,8 +416,9 @@ def addServer(config, distributor, schedule): :type schedule: :class:`bridgedb.schedule.ScheduledInterval` :param schedule: The schedule. XXX: Is this even used? """ - context = MailContext(config, distributor, schedule) - factory = MailFactory(context) + context = MailServerContext(config, distributor, schedule) + factory = SMTPIncomingServerFactory() + factory.setContext(context) addr = config.EMAIL_BIND_IP or "" port = config.EMAIL_PORT

1 0