May 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Add new email handling Exception classes to bridgedb.Dist.
by isis＠torproject.org 16 May '14

16 May '14

commit a5f709890f7da820aee8299efb99f00b396e980b Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 5 18:45:15 2014 +0000 Add new email handling Exception classes to bridgedb.Dist. --- lib/bridgedb/Dist.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 1596fd1..934c0b0 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -41,6 +41,12 @@ class IgnoreEmail(addr.BadEmail): class TooSoonEmail(addr.BadEmail): """Raised when we got a request from this address too recently.""" +class EmailRequestedHelp(Exception): + """Raised when a client has emailed requesting help.""" + +class EmailRequestedKey(Exception): + """Raised when an incoming email requested a copy of our GnuPG keys.""" + def uniformMap(ip): """Map an IP to an arbitrary 'area' string, such that any two /24 addresses

1 0

[bridgedb/master] Raise a BadEmail if we couldn't parse an email address in Dist.
by isis＠torproject.org 16 May '14

16 May '14

commit 91a8a68798e745a764294cafe2013f29710b4305 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 5 19:07:10 2014 +0000 Raise a BadEmail if we couldn't parse an email address in Dist. --- lib/bridgedb/Dist.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 934c0b0..580aa5d 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -399,16 +399,19 @@ class EmailBasedDistributor(Distributor): if not bridgeFilterRules: bridgeFilterRules=[] now = time.time() + + emailaddr = None try: - emailaddress = addr.normalizeEmail(emailaddress, self.domainmap, - self.domainrules) - except addr.BadEmail as err: - logging.warn(err) + emailaddr = addr.normalizeEmail(emailaddress, + self.domainmap, + self.domainrules) + if not emailaddr: + raise addr.BadEmail("Couldn't normalize email address: %r" + % emailaddress) + except addr.BadEmail as error: + logging.warn(error) return [] - if not emailaddress: - return [] #XXXX raise an exception. - with bridgedb.Storage.getDB() as db: wasWarned = db.getWarnedEmail(emailaddress) lastSaw = db.getEmailTime(emailaddress)

1 0

[bridgedb/master] Rewrite HTML templates to pass template namespace parameters.
by isis＠torproject.org 16 May '14

16 May '14

commit 0380ffe042a1e9490a151771403062867bf5929b Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 5 18:58:25 2014 +0000 Rewrite HTML templates to pass template namespace parameters. In order to use translated strings from the bridgedb.strings module, we need to pass the namespace parameters (i.e., the arguments in HTTPServer.py to `template.render()`) from the parent template (base.html) to the inheriting templates. While we're at it, we should pass information on the requested lang, so that translations don't suddenly turn off if you go to a different page, and info on whether the page should be rendered RTL. This fixes a slight bug where the pages deriving from `bridgedb.HTTPServer.CaptchaProtectedResource` weren't being correctly translated or RTL-rendered when they should be. And also while we're at it, we should probably not do *anything* inside of `bridgedb.HTTPServer.*Resource.render_*` methods outside of the try/except block, because any error will be rendered as a traceback to the client. Additionally, we should set the content type and encoding in the headers of every response before returning it. --- lib/bridgedb/HTTPServer.py | 68 +++++++++++++++++++---------------- lib/bridgedb/templates/base.html | 8 +++-- lib/bridgedb/templates/bridges.html | 3 +- lib/bridgedb/templates/captcha.html | 2 ++ lib/bridgedb/templates/howto.html | 1 + lib/bridgedb/templates/index.html | 2 ++ lib/bridgedb/templates/options.html | 2 ++ 7 files changed, 52 insertions(+), 34 deletions(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index e11073e..7c4689a 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -206,17 +206,24 @@ class CaptchaProtectedResource(resource.Resource): :returns: A rendered HTML page containing a ReCaptcha challenge image for the client to solve. """ + rtl = False image, challenge = self.getCaptchaImage(request) try: + langs = translations.getLocaleFromHTTPRequest(request) + rtl = translations.usingRTLLang(langs) # TODO: this does not work for versions of IE < 8.0 imgstr = 'data:image/jpeg;base64,%s' % base64.b64encode(image) template = lookup.get_template('captcha.html') - rendered = template.render(imgstr=imgstr, + rendered = template.render(strings, + rtl=rtl, + lang=langs[0], + imgstr=imgstr, challenge_field=challenge) except Exception as err: rendered = replaceErrorPage(err, 'captcha.html') + request.setHeader("Content-Type", "text/html; charset=utf-8") return rendered def render_POST(self, request): @@ -236,6 +243,8 @@ class CaptchaProtectedResource(resource.Resource): :returns: A rendered HTML page containing a ReCaptcha challenge image for the client to solve. """ + request.setHeader("Content-Type", "text/html; charset=utf-8") + if self.checkSolution(request) is True: try: rendered = self.resource.render(request) @@ -559,15 +568,15 @@ class WebResourceOptions(resource.Resource): def render_GET(self, request): rtl = False - langs = translations.getLocaleFromHTTPRequest(request) - try: + langs = translations.getLocaleFromHTTPRequest(request) rtl = translations.usingRTLLang(langs) + template = lookup.get_template('options.html') + rendered = template.render(strings, rtl=rtl, lang=langs[0]) except Exception as err: # pragma: no cover - logging.exception(err) - + rendered = replaceErrorPage(err) request.setHeader("Content-Type", "text/html; charset=utf-8") - return lookup.get_template('options.html').render(rtl=rtl) + return rendered render_POST = render_GET @@ -584,15 +593,15 @@ class WebResourceHowto(resource.Resource): def render_GET(self, request): rtl = False - langs = translations.getLocaleFromHTTPRequest(request) - try: + langs = translations.getLocaleFromHTTPRequest(request) rtl = translations.usingRTLLang(langs) + template = lookup.get_template('howto.html') + rendered = template.render(strings, rtl=rtl, lang=langs[0]) except Exception as err: # pragma: no cover - logging.exception(err) - + rendered = replaceErrorPage(err) request.setHeader("Content-Type", "text/html; charset=utf-8") - return lookup.get_template('howto.html').render(rtl=rtl) + return rendered render_POST = render_GET @@ -683,11 +692,6 @@ class WebResourceBridges(resource.Resource): if countryCode: logging.debug("Client request from GeoIP CC: %s" % countryCode) - langs = translations.getLocaleFromHTTPRequest(request) - rtl = translations.usingRTLLang(langs) - if rtl: - logging.debug("Rendering RTL response.") - # XXX separate function again format = request.args.get("format", None) if format and len(format): format = format[0] # choose the first arg @@ -749,10 +753,10 @@ class WebResourceBridges(resource.Resource): request=bridgedb.Dist.uniformMap(ip) ) for b in bridges) - answer = self.renderAnswer(request, bridgeLines, rtl, format) + answer = self.renderAnswer(request, bridgeLines, format) return answer - def renderAnswer(self, request, bridgeLines=None, rtl=False, format=None): + def renderAnswer(self, request, bridgeLines=None, format=None): """Generate a response for a client which includes **bridges**. The generated response can be plaintext or HTML. @@ -763,8 +767,6 @@ class WebResourceBridges(resource.Resource): :type bridgeLines: list or None :param bridgeLines: A list of strings used to configure a Tor client to use a bridge. - :param bool rtl: If ``True``, the language used for the response to - the client should be rendered right-to-left. :type format: str or None :param format: If ``'plain'``, return a plaintext response. Otherwise, use the :file:`bridgedb/templates/bridges.html` template to render @@ -772,17 +774,21 @@ class WebResourceBridges(resource.Resource): :rtype: str :returns: A plaintext or HTML response to serve. """ + rtl = False + if format == 'plain': request.setHeader("Content-Type", "text/plain") rendered = bridgeLines else: request.setHeader("Content-Type", "text/html; charset=utf-8") try: - # XXX FIXME the returned page from - # ``WebResourceBridgesTests.test_render_GET_RTLlang`` - # is in Arabic and has `<html lang="en">`! Doh. + langs = translations.getLocaleFromHTTPRequest(request) + rtl = translations.usingRTLLang(langs) template = lookup.get_template('bridges.html') - rendered = template.render(answer=bridgeLines, rtl=rtl) + rendered = template.render(strings, + rtl=rtl, + lang=langs[0], + answer=bridgeLines) except Exception as err: rendered = replaceErrorPage(err) @@ -804,17 +810,17 @@ class WebRoot(resource.Resource): :param request: An incoming request. """ rtl = False - langs = translations.getLocaleFromHTTPRequest(request) - try: + langs = translations.getLocaleFromHTTPRequest(request) rtl = translations.usingRTLLang(langs) + template = lookup.get_template('index.html') + rendered = template.render(strings, + rtl=rtl, + lang=langs[0]) except Exception as err: - logging.exception(err) - logging.error("The gettext files were not properly installed.") - logging.info("To install translations, try doing `python " \ - "setup.py compile_catalog`.") + rendered = replaceErrorPage(err) - return lookup.get_template('index.html').render(rtl=rtl) + return rendered def addWebServer(cfg, dist, sched): diff --git a/lib/bridgedb/templates/base.html b/lib/bridgedb/templates/base.html index 4228f9f..1ec49a3 100644 --- a/lib/bridgedb/templates/base.html +++ b/lib/bridgedb/templates/base.html @@ -1,6 +1,10 @@ ## -*- coding: utf-8 -*- + +<%namespace name="base" file="base.html" inheritable="True"/> +<%page args="strings, rtl=False, lang='en', **kwargs"/> + <!DOCTYPE html> -<html lang="en"> +<html lang="${lang}"> <head> <meta charset="utf-8"> <title>BridgeDB</title> @@ -50,7 +54,7 @@ span { </div> -${self.body()} +${next.body(strings, rtl=rtl, lang=lang, **kwargs)} <div class="faq"> diff --git a/lib/bridgedb/templates/bridges.html b/lib/bridgedb/templates/bridges.html index a118404..4058d54 100644 --- a/lib/bridgedb/templates/bridges.html +++ b/lib/bridgedb/templates/bridges.html @@ -1,5 +1,7 @@ ## -*- coding: utf-8 -*- + <%inherit file="base.html"/> +<%page args="strings, rtl=False, lang='en', answer=0, **kwargs"/> <div class="container-fluid" style="width: 98%; align: center; margin: auto;"> @@ -78,7 +80,6 @@ ${_(""" Select "Yes" and click "Next" in order to configure your""" \ </div> </div> <br /> - % endif <hr /> diff --git a/lib/bridgedb/templates/captcha.html b/lib/bridgedb/templates/captcha.html index 23b9772..9f62258 100644 --- a/lib/bridgedb/templates/captcha.html +++ b/lib/bridgedb/templates/captcha.html @@ -1,5 +1,7 @@ ## -*- coding: utf-8 -*- + <%inherit file="base.html"/> +<%page args="strings, rtl=False, lang='en', imgstr=0, captcha_challenge=0, **kwargs"/> <div class="container-narrow" id="captchaSubmissionContainer" diff --git a/lib/bridgedb/templates/howto.html b/lib/bridgedb/templates/howto.html index 42e14ca..b143741 100644 --- a/lib/bridgedb/templates/howto.html +++ b/lib/bridgedb/templates/howto.html @@ -1,6 +1,7 @@ ## -*- coding: utf-8 -*- <%inherit file="base.html"/> +<%page args="strings, rtl=False, lang='en', **kwargs"/> <div class="container-fluid" style="width: 98%; align: center; margin: auto;"> diff --git a/lib/bridgedb/templates/index.html b/lib/bridgedb/templates/index.html index 243ba10..21a5281 100644 --- a/lib/bridgedb/templates/index.html +++ b/lib/bridgedb/templates/index.html @@ -1,5 +1,7 @@ ## -*- coding: utf-8 -*- + <%inherit file="base.html"/> +<%page args="strings, rtl=False, lang='en', **kwargs"/> <div class="main-steps"> <div class="step row"> diff --git a/lib/bridgedb/templates/options.html b/lib/bridgedb/templates/options.html index d786ac1..c25e418 100644 --- a/lib/bridgedb/templates/options.html +++ b/lib/bridgedb/templates/options.html @@ -1,5 +1,7 @@ ## -*- coding: utf-8 -*- + <%inherit file="base.html"/> +<%page args="strings, rtl=False, lang='en', **kwargs"/> <div class="container-fluid" style="width: 96%; align: center; margin: 2%">

1 0

[bridgedb/master] Add bridgerequest module for abstracting info about bridge types requested.
by isis＠torproject.org 16 May '14

16 May '14

commit 92ca7cbefe7eec0fe30452e47e752bd4fb820839 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 5 19:13:39 2014 +0000 Add bridgerequest module for abstracting info about bridge types requested. --- lib/bridgedb/bridgerequest.py | 128 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py new file mode 100644 index 0000000..4f32f67 --- /dev/null +++ b/lib/bridgedb/bridgerequest.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_bridgerequest ; -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2007-2014, The Tor Project, Inc. +# (c) 2014, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + + +import logging + +import ipaddr + +from zope.interface import implements +from zope.interface import Attribute +from zope.interface import Interface + +from bridgedb import Filters + + +class IBridgeRequest(Interface): + """Interface specification of client options for requested bridges.""" + + addressClass = Attribute( + "The IP version of bridges addresses to distribute to the client.") + filters = Attribute( + "A list of callables used to filter bridges from a hashring.") + transports = Attribute( + "A list of strings of Pluggable Transport types requested.") + notBlockedIn = Attribute( + "A list of 2-4 letter country codes. The distributed bridges should " + "not be blocked in these countries.") + valid = Attribute( + "A boolean. Should be ``True`` if the client's request was valid.") + + def addFilter(): + """Add a filter to the list of ``filters``.""" + + def clearFilters(): + """Clear the list of ``filters``.""" + + def generateFilters(): + """Build the list of callables, ``filters``, according to the current + contents of the lists of ``transports``, ``notBlockedIn``, and the + ``addressClass``. + """ + + def isValid(): + """Determine if the request is ``valid`` according to some parameters.""" + + def withIPv4(): + """Set the ``addressClass`` to IPv4.""" + + def withIPv6(): + """Set the ``addressClass`` to IPv6.""" + + def withPluggableTransportType(typeOfPT): + """Add this **typeOfPT** to the list of requested ``transports``.""" + + def withoutBlockInCountry(countryCode): + """Add this **countryCode** to the list of countries which distributed + bridges should not be blocked in (``notBlockedIn``). + """ + + +class BridgeRequestBase(object): + """A generic base class for storing options of a client bridge request.""" + implements(IBridgeRequest) + + def __init__(self, addressClass=None): + self.addressClass = addressClass + if not isinstance(self.addressClass, + (ipaddr.IPv4Address, ipaddr.IPv6Address)): + self.addressClass = ipaddr.IPv4Address + self.filters = list() + self.transports = list() + self.notBlockedIn = list() + self.valid = False + + def isValid(self): + pass + + def withIPv4(self): + self.addressClass = ipaddr.IPv4Address + + def withIPv6(self): + self.addressClass = ipaddr.IPv6Address + + def withoutBlockInCountry(self, country): + self.notBlockedIn.append(country) + + def withPluggableTransportType(self, pt): + self.transports.append(pt) + + def addFilter(self, filtre): + self.filters.append(filtre) + + def clearFilters(self): + self.filters = [] + + def justOnePTType(self): + """Get just one bridge PT type at a time!""" + ptType = None + try: + ptType = self.transports[-1] # Use the last PT requested + except IndexError: + logging.debug("No pluggable transports were requested.") + return ptType + + def generateFilters(self): + if self.addressClass is ipaddr.IPv6Address: + self.addFilter(Filters.filterBridgesByIP6) + else: + self.addFilter(Filters.filterBridgesByIP4) + + transport = self.justOnePTType() + if transport: + self.clearFilters() + self.addFilter(Filters.filterBridgesByTransport(transport, + self.addressClass)) + for country in self.notBlockedIn: + self.addFilter(Filters.filterBridgesByNotBlockedIn(country, + self.addressClass, + transport))

1 0

[bridgedb/master] Add TRANSLATORS note on which strings should never be translated.
by isis＠torproject.org 16 May '14

16 May '14

commit b8d627c44fd7e3b25afb47977f490bd17b198c14 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 5 18:54:38 2014 +0000 Add TRANSLATORS note on which strings should never be translated. I put it here because this is the first translated string which ends up in the gettext .pot template file, and thus it ends up being included before any of the strings to translate. --- lib/bridgedb/HTTPServer.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index dc38082..e11073e 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -103,6 +103,22 @@ def replaceErrorPage(error, template_name=None): % (template_name or 'template', mako.exceptions.text_error_template().render())) + # TRANSLATORS: Please DO NOT translate the following words and/or phrases in + # any string (regardless of capitalization and/or punctuation): + # + # "bridge" + # "bridges" + # "BridgeDB" + # "pluggable transport" + # "pluggable transports" + # "obfs2" + # "obfs3" + # "scramblesuit" + # "fte" + # "Tor" + # "Tor Browser" + # "TBB" + # errmsg = _("Sorry! Something went wrong with your request.") rendered = """<html> <head>

1 0

[bridgedb/master] Fix unittest fail due to Arabic string that no longer exists.
by isis＠torproject.org 16 May '14

16 May '14

commit 4c4379e24213e6ca3e3412f9aebc9c900244b84a Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 5 19:51:52 2014 +0000 Fix unittest fail due to Arabic string that no longer exists. --- lib/bridgedb/test/test_HTTPServer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index 0de233d..570fe9c 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -596,7 +596,8 @@ class WebResourceBridgesTests(unittest.TestCase): page = self.bridgesResource.render(request) self.assertSubstring("direction: rtl", page) self.assertSubstring( - "إذا لم يعمل تور بنجاح معك، يجب عليك ارسال بريد إلكتروني إلي", page) + # "I need an alternative way to get bridges!" + "انا بحاجة إلى وسيلة بديلة للحصول على الجسور!", page) for bridgeLine in self.parseBridgesFromHTMLPage(page): # Check that each bridge line had the expected number of fields:

1 0

[bridgedb/master] Completely rewrite email servers.
by isis＠torproject.org 16 May '14

16 May '14

commit b11d1c513af7431f473d4443d37e40e1a407aee2 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 5 19:15:43 2014 +0000 Completely rewrite email servers. The old bridgedb.EmailServer module has now been divided into several modules in the bridgedb.email package. * FIXES #5463 by adding the last touches for email signing. * FIXES #7547, #7550, and #8241 by adding a welcome email which can be received by sending an invalid request, or by saying "get help" in the body of the email. * FIXES #11475 by using the same "How To Use Your Bridge Lines" text for TBB/TorLauncher which is used for the HTTP distributor (on the website). * FIXES #11753 by making email responses translatable. A translated response can be requested, for example, for Farsi, by emailing mailto:bridges+fa@torproject.org. --- lib/bridgedb/EmailServer.py | 483 ------------------------- lib/bridgedb/Main.py | 6 +- lib/bridgedb/email/request.py | 153 ++++++++ lib/bridgedb/email/server.py | 746 +++++++++++++++++++++++++++++++++++++++ lib/bridgedb/email/templates.py | 123 +++++++ setup.py | 1 + 6 files changed, 1026 insertions(+), 486 deletions(-) diff --git a/lib/bridgedb/EmailServer.py b/lib/bridgedb/EmailServer.py deleted file mode 100644 index 29787ea..0000000 --- a/lib/bridgedb/EmailServer.py +++ /dev/null @@ -1,483 +0,0 @@ -# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_EmailServer -*- -# BridgeDB by Nick Mathewson. -# Copyright (c) 2007-2013, The Tor Project, Inc. -# See LICENSE for licensing information - -"""This module implements the email interface to the bridge database.""" - -from __future__ import unicode_literals - -from email import message -import gettext -import gpgme -import io -import logging -import re -import time - -from ipaddr import IPv4Address -from ipaddr import IPv6Address - -from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet.task import LoopingCall -from twisted.mail import smtp - -from zope.interface import implements - -from bridgedb import Dist -from bridgedb import I18n -from bridgedb import safelog -from bridgedb import translations -from bridgedb.crypto import getGPGContext -from bridgedb.crypto import gpgSignMessage -from bridgedb.crypto import NEW_BUFFER_INTERFACE -from bridgedb.Filters import filterBridgesByIP6 -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByTransport -from bridgedb.Filters import filterBridgesByNotBlockedIn -from bridgedb.parse import addr -from bridgedb.parse.addr import BadEmail -from bridgedb.parse.addr import UnsupportedDomain -from bridgedb.parse.addr import canonicalizeEmailDomain - - -def getBridgeDBEmailAddrFromList(ctx, address_list): - """Loop through a list of (full name, email address) pairs and look up our - mail address. If our address isn't found (which can't happen), return - the default ctx from address so we can keep on working. - """ - email = ctx.fromAddr - for _, address in address_list: - # Strip the @torproject.org part from the address - idx = address.find('@') - if idx != -1: - username = address[:idx] - # See if the user looks familiar. We do a 'find' instead - # of compare because we might have a '+' address here - if username.find(ctx.username) != -1: - email = address - return email - -def getMailResponse(lines, ctx): - """Given a list of lines from an incoming email message, and a - MailContext object, parse the email and decide what to do in response. - If we want to answer, return a 2-tuple containing the address that - will receive the response, and a readable filelike object containing - the response. Return None,None if we shouldn't answer. - """ - raw = io.StringIO() - raw.writelines([unicode('{0}\n'.format(line)) for line in lines]) - raw.seek(0) - - msg = smtp.rfc822.Message(raw) - # Extract data from the headers. - msgID = msg.getheader("Message-ID", None) - subject = msg.getheader("Subject", None) or "[no subject]" - - fromHeader = msg.getaddr("From") - senderHeader = msg.getaddr("Sender") - - clientAddrHeader = None - try: - clientAddrHeader = fromHeader[1] - except (IndexError, TypeError, AttributeError): - pass - - if not clientAddrHeader: - logging.warn("No From header on incoming mail.") - try: - clientAddrHeader = senderHeader[1] - except (IndexError, TypeError, AttributeError): - pass - - if not clientAddrHeader: - logging.warn("No Sender header on incoming mail.") - return None, None - - try: - clientAddr = addr.normalizeEmail(clientAddrHeader, - ctx.cfg.EMAIL_DOMAIN_MAP, - ctx.cfg.EMAIL_DOMAIN_RULES) - except (UnsupportedDomain, BadEmail) as error: - logging.warn(error) - return None, None - - # RFC822 requires at least one 'To' address - clientToList = msg.getaddrlist("To") - clientToAddr = getBridgeDBEmailAddrFromList(ctx, clientToList) - - # Look up the locale part in the 'To:' address, if there is one and get - # the appropriate Translation object - lang = translations.getLocaleFromPlusAddr(clientToAddr) - t = translations.installTranslations(lang) - - canon = ctx.cfg.EMAIL_DOMAIN_MAP - for domain, rule in ctx.cfg.EMAIL_DOMAIN_RULES.items(): - if domain not in canon.keys(): - canon[domain] = domain - for domain in ctx.cfg.EMAIL_DOMAINS: - canon[domain] = domain - - try: - _, clientDomain = addr.extractEmailAddress(clientAddr.lower()) - canonical = canonicalizeEmailDomain(clientDomain, canon) - except (UnsupportedDomain, BadEmail) as error: - logging.warn(error) - return None, None - - rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(canonical, []) - - if 'dkim' in rules: - # getheader() returns the last of a given kind of header; we want - # to get the first, so we use getheaders() instead. - dkimHeaders = msg.getheaders("X-DKIM-Authentication-Results") - dkimHeader = "<no header>" - if dkimHeaders: - dkimHeader = dkimHeaders[0] - if not dkimHeader.startswith("pass"): - logging.info("Rejecting bad DKIM header on incoming email: %r " - % dkimHeader) - return None, None - - # Was the magic string included - #for ln in lines: - # if ln.strip().lower() in ("get bridges", "subject: get bridges"): - # break - #else: - # logging.info("Got a mail from %r with no bridge request; dropping", - # clientAddr) - # return None,None - - # Figure out which bridges to send - unblocked = transport = ipv6 = skippedheaders = False - bridgeFilterRules = [] - addressClass = None - for ln in lines: - # ignore all lines before the subject header - if "subject" in ln.strip().lower(): - skippedheaders = True - if not skippedheaders: - continue - - if "ipv6" in ln.strip().lower(): - ipv6 = True - if "transport" in ln.strip().lower(): - try: - transport = re.search("transport ([_a-zA-Z][_a-zA-Z0-9]*)", - ln).group(1).strip() - except (TypeError, AttributeError): - transport = None - logging.debug("Got request for transport: %s" % transport) - if "unblocked" in ln.strip().lower(): - try: - unblocked = re.search("unblocked ([a-zA-Z]{2,4})", - ln).group(1).strip() - except (TypeError, AttributeError): - transport = None - - if ipv6: - bridgeFilterRules.append(filterBridgesByIP6) - addressClass = IPv6Address - else: - bridgeFilterRules.append(filterBridgesByIP4) - addressClass = IPv4Address - - if transport: - bridgeFilterRules = [filterBridgesByTransport(transport, addressClass)] - - if unblocked: - rules.append(filterBridgesByNotBlockedIn(unblocked, - addressClass, - transport)) - - try: - interval = ctx.schedule.getInterval(time.time()) - bridges = ctx.distributor.getBridgesForEmail(clientAddr, - interval, ctx.N, - countryCode=None, - bridgeFilterRules=bridgeFilterRules) - - # Handle rate limited email - except Dist.TooSoonEmail as err: - logging.info("Got a mail too frequently; warning '%s': %s." - % (clientAddr, err)) - # MAX_EMAIL_RATE is in seconds, convert to hours - body = buildSpamWarningTemplate(t) % (Dist.MAX_EMAIL_RATE / 3600) - return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID, - gpgContext=ctx.gpgContext) - except Dist.IgnoreEmail as err: - logging.info("Got a mail too frequently; ignoring '%s': %s." - % (clientAddr, err)) - return None, None - except BadEmail as err: - logging.info("Got a mail from a bad email address '%s': %s." - % (clientAddr, err)) - return None, None - - answer = "(no bridges currently available)\n" - if bridges: - with_fp = ctx.cfg.EMAIL_INCLUDE_FINGERPRINTS - answer = "".join(" %s\n" % b.getConfigLine( - includeFingerprint=with_fp, - addressClass=addressClass, - transport=transport, - request=clientAddr) for b in bridges) - - body = buildMessageTemplate(t) % answer - # Generate the message. - return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID, - gpgContext=ctx.gpgContext) - -def buildMessageTemplate(t): - msg_template = t.gettext(I18n.BRIDGEDB_TEXT[5]) + "\n\n" \ - + t.gettext(I18n.BRIDGEDB_TEXT[0]) + "\n\n" \ - + "%s\n" \ - + t.gettext(I18n.BRIDGEDB_TEXT[1]) + "\n\n" \ - + t.gettext(I18n.BRIDGEDB_TEXT[2]) + "\n\n" \ - + t.gettext(I18n.BRIDGEDB_TEXT[3]) + "\n\n" \ - + t.gettext(I18n.BRIDGEDB_TEXT[17])+ "\n\n" - # list supported commands, e.g. ipv6, transport - msg_template = msg_template \ - + " " + t.gettext(I18n.BRIDGEDB_TEXT[18])+ "\n" \ - + " " + t.gettext(I18n.BRIDGEDB_TEXT[19])+ "\n\n" \ - + t.gettext(I18n.BRIDGEDB_TEXT[6]) + "\n\n" - return msg_template - -def buildSpamWarningTemplate(t): - msg_template = t.gettext(I18n.BRIDGEDB_TEXT[5]) + "\n\n" \ - + t.gettext(I18n.BRIDGEDB_TEXT[10]) + "\n\n" \ - + "%s " \ - + t.gettext(I18n.BRIDGEDB_TEXT[11]) + "\n\n" \ - + t.gettext(I18n.BRIDGEDB_TEXT[12]) + "\n\n" - return msg_template - -def _ebReplyToMailFailure(fail): - """Errback for a :api:`twisted.mail.smtp.SMTPSenderFactory`. - - :param fail: A :api:`twisted.python.failure.Failure` which occurred during - the transaction. - """ - logging.debug("EmailServer._ebReplyToMailFailure() called with %r" % fail) - error = fail.getErrorMessage() or "unknown failure." - logging.exception("replyToMail Failure: %s" % error) - return None - -def replyToMail(lines, ctx): - """Reply to an incoming email. Maybe. - - If no `response` is returned from :func:`getMailResponse`, then the - incoming email will not be responded to at all. This can happen for - several reasons, for example: if the DKIM signature was invalid or - missing, or if the incoming email came from an unacceptable domain, or if - there have been too many emails from this client in the allotted time - period. - - :param list lines: A list of lines from an incoming email message. - :type ctx: :class:`MailContext` - :param ctx: The configured context for the email server. - :rtype: :api:`twisted.internet.defer.Deferred` - :returns: A ``Deferred`` which will callback when the response has been - successfully sent, or errback if an error occurred while sending the - email. - """ - logging.info("Got an email; deciding whether to reply.") - sendToUser, response = getMailResponse(lines, ctx) - - d = defer.Deferred() - - if response is None: - logging.debug("We don't feel like talking to %s." % sendToUser) - return d - - response.seek(0) - logging.info("Sending reply to %s" % sendToUser) - factory = smtp.SMTPSenderFactory(ctx.smtpFromAddr, sendToUser, - response, d, retries=0, timeout=30) - d.addErrback(_ebReplyToMailFailure) - reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory) - return d - -def composeEmail(fromAddr, clientAddr, subject, body, - msgID=None, gpgContext=None): - - if not subject.startswith("Re:"): - subject = "Re: %s" % subject - - msg = smtp.rfc822.Message(io.StringIO()) - msg.setdefault("From", fromAddr) - msg.setdefault("To", clientAddr) - msg.setdefault("Message-ID", smtp.messageid()) - msg.setdefault("Subject", subject) - if msgID: - msg.setdefault("In-Reply-To", msgID) - msg.setdefault("Date", smtp.rfc822date()) - msg.setdefault('Content-Type', 'text/plain; charset="utf-8"') - headers = [': '.join(m) for m in msg.items()] - - if NEW_BUFFER_INTERFACE: - mail = io.BytesIO() - buff = buffer - else: - mail = io.StringIO() - buff = unicode - - mail.writelines(buff("\r\n".join(headers))) - mail.writelines(buff("\r\n")) - mail.writelines(buff("\r\n")) - - if not gpgContext: - mail.write(buff(body)) - else: - signature, siglist = gpgSignMessage(gpgContext, body) - if signature: - mail.writelines(buff(signature)) - mail.seek(0) - - # Only log the email text (including all headers) if SAFE_LOGGING is - # disabled: - if not safelog.safe_logging: - logging.debug("Email contents:\n\n%s" % mail.read()) - mail.seek(0) - else: - logging.debug("Email text for %r created." % clientAddr) - - return clientAddr, mail - - -class MailContext(object): - """Helper object that holds information used by email subsystem.""" - - def __init__(self, cfg, dist, sched): - # Reject any RCPT TO lines that aren't to this user. - self.username = (cfg.EMAIL_USERNAME or "bridges") - # Reject any mail longer than this. - self.maximumSize = 32*1024 - # Use this server for outgoing mail. - self.smtpServer = (cfg.EMAIL_SMTP_HOST or "127.0.0.1") - self.smtpPort = (cfg.EMAIL_SMTP_PORT or 25) - # Use this address in the MAIL FROM line for outgoing mail. - self.smtpFromAddr = (cfg.EMAIL_SMTP_FROM_ADDR or - "bridges(a)torproject.org") - # Use this address in the "From:" header for outgoing mail. - self.fromAddr = (cfg.EMAIL_FROM_ADDR or - "bridges(a)torproject.org") - # An EmailBasedDistributor object - self.distributor = dist - # An IntervalSchedule object - self.schedule = sched - # The number of bridges to send for each email. - self.N = cfg.EMAIL_N_BRIDGES_PER_ANSWER - - # Initialize a gpg context or set to None for backward compatibliity. - self.gpgContext = getGPGContext(cfg) - - self.cfg = cfg - -class MailMessage(object): - """Plugs into the Twisted Mail and receives an incoming message.""" - implements(smtp.IMessage) - - def __init__(self, ctx): - """Create a new MailMessage from a MailContext.""" - self.ctx = ctx - self.lines = [] - self.nBytes = 0 - self.ignoring = False - - def lineReceived(self, line): - """Called when we get another line of an incoming message.""" - self.nBytes += len(line) - if not safelog.safe_logging: - logging.debug("> %s", line.rstrip("\r\n")) - if self.nBytes > self.ctx.maximumSize: - self.ignoring = True - else: - self.lines.append(line) - - def eomReceived(self): - """Called when we receive the end of a message.""" - if not self.ignoring: - replyToMail(self.lines, self.ctx) - return defer.succeed(None) - - def connectionLost(self): - """Called if we die partway through reading a message.""" - pass - -class MailDelivery(object): - """Plugs into Twisted Mail and handles SMTP commands.""" - implements(smtp.IMessageDelivery) - - def setBridgeDBContext(self, ctx): - self.ctx = ctx - - def receivedHeader(self, helo, origin, recipients): - """Create the ``Received:`` header for an incoming email. - - :type helo: tuple - :param helo: The lines received during SMTP client HELO. - :type origin: :api:`twisted.mail.smtp.Address` - :param origin: The email address of the sender. - :type recipients: list - :param recipients: A list of :api:`twisted.mail.smtp.User` instances. - """ - cameFrom = "%s (%s [%s])" % (helo[0] or origin, helo[0], helo[1]) - cameFor = ', '.join(["<{0}>".format(recp.dest) for recp in recipients]) - hdr = str("Received: from %s for %s; %s" % (cameFrom, cameFor, - smtp.rfc822date())) - return hdr - - def validateFrom(self, helo, origin): - return origin - - def validateTo(self, user): - """If the local user that was addressed isn't our configured local user - or doesn't contain a '+' with a prefix matching the local configured - user: Yell. - """ - u = user.dest.local - # Hasplus? If yes, strip '+foo' - idx = u.find('+') - if idx != -1: - u = u[:idx] - if u != self.ctx.username: - raise smtp.SMTPBadRcpt(user) - return lambda: MailMessage(self.ctx) - -class MailFactory(smtp.SMTPFactory): - """Plugs into Twisted Mail; creates a new MailDelivery whenever we get - a connection on the SMTP port.""" - - def __init__(self, *a, **kw): - smtp.SMTPFactory.__init__(self, *a, **kw) - self.delivery = MailDelivery() - - def setBridgeDBContext(self, ctx): - self.ctx = ctx - self.delivery.setBridgeDBContext(ctx) - - def buildProtocol(self, addr): - p = smtp.SMTPFactory.buildProtocol(self, addr) - p.delivery = self.delivery - return p - -def addSMTPServer(cfg, dist, sched): - """Set up a smtp server. - cfg -- a configuration object from Main. We use these options: - EMAIL_BIND_IP - EMAIL_PORT - EMAIL_N_BRIDGES_PER_ANSWER - EMAIL_DOMAIN_RULES - dist -- an EmailBasedDistributor object. - sched -- an IntervalSchedule object. - """ - ctx = MailContext(cfg, dist, sched) - factory = MailFactory() - factory.setBridgeDBContext(ctx) - ip = cfg.EMAIL_BIND_IP or "" - reactor.listenTCP(cfg.EMAIL_PORT, factory, interface=ip) - # Set up a LoopingCall to run every 30 minutes and forget old email times. - lc = LoopingCall(dist.cleanDatabase) - lc.start(1800, now=False) - return factory diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 90a03ef..1b4420e 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -443,7 +443,7 @@ def startup(options): state = persistent.State(config=config) - from bridgedb import EmailServer + from bridgedb.email.server import addServer as addSMTPServer from bridgedb import HTTPServer # Load the master key, or create a new one. @@ -596,7 +596,7 @@ def startup(options): if config.EMAIL_DIST and config.EMAIL_SHARE: #emailSchedule = Time.IntervalSchedule("day", 1) emailSchedule = Time.NoSchedule() - EmailServer.addSMTPServer(config, emailDistributor, emailSchedule) + addSMTPServer(config, emailDistributor, emailSchedule) # Actually run the servers. try: @@ -623,7 +623,7 @@ def runSubcommand(options, config): """ # Make sure that the runner module is only imported after logging is set # up, otherwise we run into the same logging configuration problem as - # mentioned above with the EmailServer and HTTPServer. + # mentioned above with the email.server and HTTPServer. from bridgedb import runner statuscode = 0 diff --git a/lib/bridgedb/email/__init__.py b/lib/bridgedb/email/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/bridgedb/email/request.py b/lib/bridgedb/email/request.py new file mode 100644 index 0000000..e5b1fb9 --- /dev/null +++ b/lib/bridgedb/email/request.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- + +from __future__ import print_function +from __future__ import unicode_literals + +import logging +import re + +from bridgedb import bridgerequest +from bridgedb.Dist import EmailRequestedHelp +from bridgedb.Dist import EmailRequestedKey + + +#: A regular expression for matching the Pluggable Transport method TYPE in +#: emailed requests for Pluggable Transports. +TRANSPORT_REGEXP = ".*transport ([a-z][_a-z0-9]*)" +TRANSPORT_PATTERN = re.compile(TRANSPORT_REGEXP) + +#: A regular expression that matches country codes in requests for unblocked +#: bridges. +UNBLOCKED_REGEXP = ".*unblocked ([a-z]{2,4})" +UNBLOCKED_PATTERN = re.compile(UNBLOCKED_REGEXP) + + +def determineBridgeRequestOptions(lines): + """Figure out which :class:`Bridges.BridgeFilter`s to apply, or offer help. + + .. note:: If any ``'transport TYPE'`` was requested, or bridges not + blocked in a specific CC (``'unblocked CC'``), then the ``TYPE`` + and/or ``CC`` will *always* be stored as a *lowercase* string. + + :param list lines: A list of lines from an email, including the headers. + :raises EmailRequestedHelp: if the client requested help. + :raises EmailRequestedKey: if the client requested our GnuPG key. + :rtype: :class:`EmailBridgeRequest` + :returns: A :class:`~bridgerequst.BridgeRequest` with all of the requested + parameters set. The returned ``BridgeRequest`` will have already had + its filters generated via :meth:`~EmailBridgeRequest.generateFilters`. + """ + request = EmailBridgeRequest() + skippedHeaders = False + + for line in lines: + line = line.strip().lower() + # Ignore all lines before the first empty line: + if not line: skippedHeaders = True + if not skippedHeaders: continue + + if ("help" in line) or ("halp" in line): + raise EmailRequestedHelp("Client requested help.") + + if "get" in line: + request.isValid(True) + logging.debug("Email request was valid.") + if "key" in line: + request.wantsKey(True) + raise EmailRequestedKey("Email requested a copy of our GnuPG key.") + if "ipv6" in line: + request.withIPv6() + if "transport" in line: + request.withPluggableTransportType(line) + if "unblocked" in line: + request.withoutBlockInCountry(line) + + logging.debug("Generating hashring filters for request.") + request.generateFilters() + return request + + +class EmailBridgeRequest(bridgerequest.BridgeRequestBase): + """We received a request for bridges through the email distributor.""" + + def __init__(self): + """Process a new bridge request received through the + :class:`~bridgedb.Dist.EmailBasedDistributor`. + """ + super(EmailBridgeRequest, self).__init__() + self._isValid = False + self._wantsKey = False + + def isValid(self, valid=None): + """Get or set the validity of this bridge request. + + If called without parameters, this method will return the current + state, otherwise (if called with the **valid** parameter), it will set + the current state of validity for this request. + + :param bool valid: If given, set the validity state of this + request. Otherwise, get the current state. + """ + if valid is not None: + self._isValid = bool(valid) + return self._isValid + + def wantsKey(self, wantsKey=None): + """Get or set whether this bridge request wanted our GnuPG key. + + If called without parameters, this method will return the current + state, otherwise (if called with the **wantsKey** parameter set), it + will set the current state for whether or not this request wanted our + key. + + :param bool wantsKey: If given, set the validity state of this + request. Otherwise, get the current state. + """ + if wantsKey is not None: + self._wantsKey = bool(wantsKey) + return self._wantsKey + + def withoutBlockInCountry(self, line): + """This request was for bridges not blocked in **country**. + + Add any country code found in the **line** to the list of + ``notBlockedIn``. Currently, a request for a transport is recognized + if the email line contains the ``'unblocked'`` command. + + :param str country: The line from the email wherein the client + requested some type of Pluggable Transport. + """ + unblocked = None + + logging.debug("Parsing 'unblocked' line: %r" % line) + try: + unblocked = UNBLOCKED_PATTERN.match(line).group(1) + except (TypeError, AttributeError): + pass + + if unblocked: + self.notBlockedIn.append(unblocked) + logging.info("Email requested bridges not blocked in: %r" + % unblocked) + + def withPluggableTransportType(self, line): + """This request included a specific Pluggable Transport identifier. + + Add any Pluggable Transport method TYPE found in the **line** to the + list of ``transports``. Currently, a request for a transport is + recognized if the email line contains the ``'transport'`` command. + + :param str line: The line from the email wherein the client + requested some type of Pluggable Transport. + """ + transport = None + logging.debug("Parsing 'transport' line: %r" % line) + + try: + transport = TRANSPORT_PATTERN.match(line).group(1) + except (TypeError, AttributeError): + pass + + if transport: + self.transports.append(transport) + logging.info("Email requested transport type: %r" % transport) diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py new file mode 100644 index 0000000..97ddcde --- /dev/null +++ b/lib/bridgedb/email/server.py @@ -0,0 +1,746 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_email_server -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson <nickm(a)torproject.org> +# Isis Lovecruft <isis(a)torproject.org> 0xA3ADB67A2CDB8B35 +# Matthew Finkel <sysrqb(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2007-2014, The Tor Project, Inc. +# (c) 2013-2014, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + + +"""Servers which interface with clients and distribute bridges over SMTP.""" + +from __future__ import unicode_literals + +import logging +import io +import time + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet.task import LoopingCall +from twisted.mail import smtp + +from zope.interface import implements + +from bridgedb import safelog +from bridgedb import translations +from bridgedb.crypto import getGPGContext +from bridgedb.crypto import gpgSignMessage +from bridgedb.crypto import NEW_BUFFER_INTERFACE +from bridgedb.Dist import EmailRequestedHelp +from bridgedb.Dist import EmailRequestedKey +from bridgedb.Dist import TooSoonEmail +from bridgedb.Dist import IgnoreEmail +from bridgedb.email import templates +from bridgedb.email import request +from bridgedb.parse import addr +from bridgedb.parse.addr import BadEmail +from bridgedb.parse.addr import UnsupportedDomain +from bridgedb.parse.addr import canonicalizeEmailDomain + + +def checkDKIM(message, rules): + """Check the DKIM verification results header. + + This check is only run if the incoming email, **message**, originated from + a domain for which we're configured (in the ``EMAIL_DOMAIN_RULES`` + dictionary in the config file) to check DKIM verification results for. + + :type message: :api:`twisted.mail.smtp.rfc822.Message` + :param message: The incoming client request email, including headers. + :param dict rules: The list of configured ``EMAIL_DOMAIN_RULES`` for the + canonical domain which the client's email request originated from. + + :rtype: bool + :returns: ``False`` if: + 1. We're supposed to expect and check the DKIM headers for the + client's email provider domain. + 2. Those headers were *not* okay. + Otherwise, returns ``True``. + """ + if 'dkim' in rules: + # getheader() returns the last of a given kind of header; we want + # to get the first, so we use getheaders() instead. + dkimHeaders = message.getheaders("X-DKIM-Authentication-Results") + dkimHeader = "<no header>" + if dkimHeaders: + dkimHeader = dkimHeaders[0] + if not dkimHeader.startswith("pass"): + logging.info("Rejecting bad DKIM header on incoming email: %r " + % dkimHeader) + return False + return True + +def createResponseBody(lines, context, toAddress, lang='en'): + """Parse the **lines** from an incoming email request and determine how to + respond. + + :param list lines: The list of lines from the original request sent by the + client. + :type context: class:`MailContext` + :param context: The context which contains settings for the email server. + :param str toAddress: The rfc:`2821` email address which should be in the + :header:`To:` header of the response email. + :param str lang: The 2-5 character locale code to use for translating the + email. This is obtained from a client sending a email to a valid plus + address which includes the translation desired, i.e. by sending an + email to ``bridges+fa(a)torproject.org``, the client should receive a + response in Farsi. + :rtype: None or str + :returns: None if we shouldn't respond to the client (i.e., if they have + already received a rate-limiting warning email). Otherwise, returns a + string containing the (optionally translated) body for the email + response which we should send out. + """ + t = translations.installTranslations(lang) + + bridges = None + try: + bridgeRequest = request.determineBridgeRequestOptions(lines) + + # The request was invalid, respond with a help email which explains + # valid email commands: + if not bridgeRequest.isValid(): + raise EmailRequestedHelp("Email request from %r was invalid." + % toAddress) + + # Otherwise they must have requested bridges: + interval = context.schedule.getInterval(time.time()) + bridges = context.distributor.getBridgesForEmail( + toAddress, + interval, + context.nBridges, + countryCode=None, + bridgeFilterRules=bridgeRequest.filters) + except EmailRequestedHelp as error: + logging.info(error) + return templates.buildWelcomeText(t) + except EmailRequestedKey as error: + logging.info(error) + return templates.buildKeyfile(t) + except TooSoonEmail as error: + logging.info("Got a mail too frequently: %s." % error) + return templates.buildSpamWarning(t) + except (IgnoreEmail, BadEmail) as error: + logging.info(error) + # Don't generate a response if their email address is unparsable or + # invalid, or if we've already warned them about rate-limiting: + return None + else: + answer = "(no bridges currently available)\r\n" + if bridges: + transport = bridgeRequest.justOnePTType() + answer = "".join(" %s\r\n" % b.getConfigLine( + includeFingerprint=context.includeFingerprints, + addressClass=bridgeRequest.addressClass, + transport=transport, + request=toAddress) for b in bridges) + return templates.buildMessage(t) % answer + +def generateResponse(fromAddress, clientAddress, subject, body, + messageID=None, gpgContext=None): + """Create a :class:`MailResponse`, which acts like an in-memory + ``io.StringIO`` file, by creating and writing all headers and the email + body into the file-like ``MailResponse.mailfile``. + + :param str fromAddress: The rfc:`2821` email address which should be in + the :header:`From:` header. + :param str clientAddress: The rfc:`2821` email address which should be in + the :header:`To:` header. + :param str subject: The string to write to the :header:`subject` header. + :param str body: The body of the email. If a **gpgContext** is also given, + and that ``Context`` has email signing configured, then + :meth:`MailResponse.writeBody` will generate and include any + ascii-armored OpenPGP signatures in the **body**. + :type messageID: None or str + :param messageID: The :rfc:`2822` specifier for the :header:`Message-ID:` + header, if including one is desirable. + :type gpgContext: None or ``gpgme.Context``. + :param gpgContext: A pre-configured GPGME context. See + :meth:`~crypto.getGPGContext`. + :rtype: :class:`MailResponse` + :returns: A ``MailResponse`` which contains the entire email. To obtain + the contents of the email, including all headers, simply use + :meth:`MailResponse.read`. + """ + response = MailResponse(gpgContext) + response.writeHeaders(fromAddress, clientAddress, subject, + inReplyTo=messageID) + response.writeBody(body) + + # Only log the email text (including all headers) if SAFE_LOGGING is + # disabled: + if not safelog.safe_logging: + contents = response.readContents() + logging.debug("Email contents:\n%s" % contents) + else: + logging.debug("Email text for %r created." % clientAddress) + response.rewind() + return response + + +class MailContext(object): + """Helper object that holds information used by email subsystem.""" + + def __init__(self, config, distributor, schedule): + """DOCDOC + + :ivar str username: Reject any RCPT TO lines that aren't to this + user. See the ``EMAIL_USERNAME`` option in the config file. + (default: ``'bridges'``) + :ivar int maximumSize: Reject any incoming emails longer than + this size (in bytes). (default: 3084 bytes). + :ivar int smtpPort: The port to use for outgoing SMTP. + :ivar str smtpServer: The IP address to use for outgoing SMTP. + :ivar str smtpFromAddr: Use this address in the raw SMTP ``MAIL FROM`` + line for outgoing mail. (default: ``bridges(a)torproject.org``) + :ivar str fromAddr: Use this address in the email :header:`From:` + line for outgoing mail. (default: ``bridges(a)torproject.org``) + :ivar int nBridges: The number of bridges to send for each email. + :ivar gpgContext: A ``gpgme.GpgmeContext`` (as created by + :func:`bridgedb.crypto.getGPGContext`), or None if we couldn't + create a proper GPGME context for some reason. + + :type config: :class:`bridgedb.persistent.Conf` + :type distributor: :class:`bridgedb.Dist.EmailBasedDistributor`. + :param distributor: DOCDOC + :type schedule: :class:`bridgedb.Time.IntervalSchedule`. + :param schedule: DOCDOC + """ + self.config = config + self.distributor = distributor + self.schedule = schedule + + self.maximumSize = 32*1024 + self.includeFingerprints = config.EMAIL_INCLUDE_FINGERPRINTS + self.nBridges = config.EMAIL_N_BRIDGES_PER_ANSWER + + self.username = (config.EMAIL_USERNAME or "bridges") + self.fromAddr = (config.EMAIL_FROM_ADDR or "bridges(a)torproject.org") + self.smtpFromAddr = (config.EMAIL_SMTP_FROM_ADDR or self.fromAddr) + self.smtpServerPort = (config.EMAIL_SMTP_PORT or 25) + self.smtpServerIP = (config.EMAIL_SMTP_HOST or "127.0.0.1") + + self.domainRules = config.EMAIL_DOMAIN_RULES or {} + self.domainMap = config.EMAIL_DOMAIN_MAP or {} + self.canon = self.buildCanonicalDomainMap() + + self.gpgContext = getGPGContext(config) + + def buildCanonicalDomainMap(self): + """Build a map for all email provider domains from which we will accept + emails to their canonical domain name. + + .. note:: Be sure that ``MailContext.domainRules`` and + ``MailContext.domainMap`` are set appropriately before calling + this method. + + This method is automatically called during initialisation, and the + resulting domain map is stored as ``MailContext.canon``. + + :rtype: dict + :returns: A dictionary which maps all domains and subdomains which we + accept emails from to their second-level, canonical domain names. + """ + canon = self.domainMap + for domain, rule in self.domainRules.items(): + if domain not in canon.keys(): + canon[domain] = domain + for domain in self.config.EMAIL_DOMAINS: + canon[domain] = domain + return canon + + +class MailResponse(object): + """Holds information for generating a response email for a request. + + .. todo:: At some point, we may want to change this class to optionally + handle creating Multipart MIME encoding messages, so that we can + include attachments. (This would be useful for attaching our GnuPG + keyfile, for example, rather than simply pasting it into the body of + the email.) + + :type _buff: unicode or buffer + :cvar _buff: Used internally to write lines for the response email into + the ``_mailfile``. The reason why both of these attributes have two + possible types is for the same Python-buggy reasons which require + :data:`~bridgedb.crypto.NEW_BUFFER_INTERFACE`. + :type mailfile: :class:`io.StringIO` or :class:`io.BytesIO`. + :cvar mailfile: An in-memory file for storing the formatted headers and + body of the response email. + """ + + implements(smtp.IMessage) + + _buff = buffer if NEW_BUFFER_INTERFACE else unicode + mailfile = io.BytesIO if NEW_BUFFER_INTERFACE else io.StringIO + + def __init__(self, gpgContext=None): + """Create a response to an email we have recieved. + + This class deals with correctly formatting text for the response email + headers and the response body into an instance of :cvar:`mailfile`. + + :type gpgContext: None or ``gpgme.Context`` + :param gpgContext: A pre-configured GPGME context. See + :meth:`bridgedb.crypto.getGPGContext` for obtaining a + pre-configured **gpgContext**. If given, and the ``Context`` has + been configured to sign emails, then a response email body string + given to :meth:`writeBody` will be signed before being written + into the ``mailfile``. + """ + self.gpgContext = gpgContext + self.mailfile = self.mailfile() + self.closed = False + + # These are methods and attributes for controlling I/O operations on our + # underlying ``mailfile``. + + def close(self): + self.mailfile.close() + self.closed = True + close.__doc__ = mailfile.close.__doc__ + + def flush(self, *args, **kwargs): self.mailfile.flush(*args, **kwargs) + flush.__doc__ = mailfile.flush.__doc__ + + def read(self, *args, **kwargs): + self.mailfile.read(*args, **kwargs) + read.__doc__ = mailfile.read.__doc__ + + def readline(self, *args, **kwargs): + self.mailfile.readline(*args, **kwargs) + readline.__doc__ = mailfile.readline.__doc__ + + def readlines(self, *args, **kwargs): + self.mailfile.readlines(*args, **kwargs) + readlines.__doc__ = mailfile.readlines.__doc__ + + def seek(self, *args, **kwargs): + self.mailfile.seek(*args, **kwargs) + seek.__doc__ = mailfile.seek.__doc__ + + def tell(self, *args, **kwargs): + self.mailfile.tell(*args, **kwargs) + tell.__doc__ = mailfile.tell.__doc__ + + def truncate(self, *args, **kwargs): + self.mailfile.truncate(*args, **kwargs) + truncate.__doc__ = mailfile.truncate.__doc__ + + # The following are custom methods to control reading and writing to the + # underlying ``mailfile``. + + def readContents(self): + """Read the all the contents written thus far to the :cvar:`mailfile`, + and then :meth:`seek` to return to the original pointer position we + were at before this method was called. + + :rtype: str + :returns: The entire contents of the :cvar:`mailfile`. + """ + pointer = self.mailfile.tell() + self.mailfile.seek(0) + contents = self.mailfile.read() + self.mailfile.seek(pointer) + return contents + + def rewind(self): + """Rewind to the very beginning of the :cvar:`mailfile`.""" + self.seek(0) + + def write(self, line): + """Any **line** written to me will have ``'\r\n'`` appended to it.""" + self.mailfile.write(self._buff(line + '\r\n')) + self.mailfile.flush() + + def writelines(self, lines): + """Calls :meth:`write` for each line in **lines**.""" + if isinstance(lines, basestring): + for ln in lines.split('\n'): + self.write(ln) + elif isinstance(lines, (list, tuple,)): + for ln in lines: + self.write(ln) + + def writeHeaders(self, fromAddress, toAddress, subject=None, + inReplyTo=None, includeMessageID=True, + contentType='text/plain; charset="utf-8"', **kwargs): + """Write all headers into the response email. + + :param str fromAddress: The email address for the ``From:`` header. + :param str toAddress: The email address for the ``To:`` header. + :type subject: None or str + :param subject: The ``Subject:`` header. + :type inReplyTo: None or str + :param inReplyTo: If set, an ``In-Reply-To:`` header will be + generated. This should be set to the ``Message-ID:`` header from + the client's original request email. + :param bool includeMessageID: If ``True``, generate and include a + ``Message-ID:`` header for the response. + :param str contentType: The ``Content-Type:`` header. + :kwargs: If given, the key will become the name of the header, and the + value will become the Contents of that header. + """ + self.write("From: %s" % fromAddress) + self.write("To: %s" % toAddress) + if includeMessageID: + self.write("Message-ID: %s" % smtp.messageid()) + if inReplyTo: + self.write("In-Reply-To: %s" % inReplyTo) + self.write("Content-Type: %s" % contentType) + self.write("Date: %s" % smtp.rfc822date()) + + if not subject: + subject = '[no subject]' + if not subject.lower().startswith('re'): + subject = "Re: " + subject + self.write("Subject: %s" % subject) + + if kwargs: + for headerName, headerValue in kwargs.items(): + headerName = headerName.capitalize() + headerName = headerName.replace(' ', '-') + headerName = headerName.replace('_', '-') + self.write("%s: %s" % (headerName, headerValue)) + + # The first blank line designates that the headers have ended: + self.write("\r\n") + + def writeBody(self, body): + """Write the response body into the :cvar:`mailfile`. + + If ``MailResponse.gpgContext`` is set, and signing is configured, the + **body** will be automatically signed before writing its contents into + the ``mailfile``. + + :param str body: The body of the response email. + """ + if self.gpgContext: + body, _ = gpgSignMessage(self.gpgContext, body) + self.writelines(body) + + # The following methods implement the IMessage interface. + + def lineReceived(self, line): + """Called when we receive a line from an underlying transport.""" + self.write(line) + + def eomRecieved(self): + """Called when we receive an EOM. + + :rtype: :api:`twisted.internet.defer.Deferred` + :returns: A ``Deferred`` which has already been callbacked with the + entire response email contents retrieved from + :meth:`readContents`. + """ + contents = self.readContents() + if not self.closed: + self.connectionLost() + return defer.succeed(contents) + + def connectionLost(self): + """Called if we die partway through reading a message. + + Truncate the :cvar:`mailfile` to null length, then close it. + """ + self.mailfile.truncate(0) + self.mailfile.close() + + +class MailMessage(object): + """Plugs into the Twisted Mail and receives an incoming message.""" + implements(smtp.IMessage) + + def __init__(self, context, fromCanonical=None): + """Create a new MailMessage from a MailContext. + + :param list lines: A list of lines from an incoming email message. + :type context: :class:`MailContext` + :param context: The configured context for the email server. + :type canonicalFrom: str or None + :param canonicalFrom: The canonical domain which this message was + received from. For example, if ``'gmail.com'`` is the configured + canonical domain for ``'googlemail.com'`` and a message is + received from the latter domain, then this would be set to the + former. + """ + self.context = context + self.fromCanonical = fromCanonical + self.lines = [] + self.nBytes = 0 + self.ignoring = False + + def lineReceived(self, line): + """Called when we get another line of an incoming message.""" + self.nBytes += len(line) + if self.nBytes > self.context.maximumSize: + self.ignoring = True + else: + self.lines.append(line) + if not safelog.safe_logging: + logging.debug("> %s", line.rstrip("\r\n")) + + def eomReceived(self): + """Called when we receive the end of a message.""" + if not self.ignoring: + self.reply() + return defer.succeed(None) + + def connectionLost(self): + """Called if we die partway through reading a message.""" + pass + + def getIncomingMessage(self): + """Create and parse an :rfc:`2822` message object for all ``lines`` + received thus far. + + :rtype: :api:`twisted.mail.smtp.rfc822.Message`. + :returns: A ``Message`` comprised of all lines received thus far. + """ + rawMessage = io.StringIO() + rawMessage.writelines([unicode('{0}\n'.format(ln)) for ln in self.lines]) + rawMessage.seek(0) + return smtp.rfc822.Message(rawMessage) + + def getClientAddress(self, incoming): + addrHeader = None + try: fromAddr = incoming.getaddr("From")[1] + except (IndexError, TypeError, AttributeError): pass + else: addrHeader = fromAddr + + if not addrHeader: + logging.warn("No From header on incoming mail.") + try: senderHeader = incoming.getaddr("Sender")[1] + except (IndexError, TypeError, AttributeError): pass + else: addrHeader = senderHeader + if not addrHeader: + logging.warn("No Sender header on incoming mail.") + else: + try: + client = smtp.Address(addr.normalizeEmail( + addrHeader, + self.context.domainMap, + self.context.domainRules)) + except (UnsupportedDomain, BadEmail, smtp.AddressError) as error: + logging.warn(error) + else: + return client + + def getRecipient(self, incoming): + """Find our **address** in a list of ``('NAME', '<ADDRESS>')`` pairs. + + If our address isn't found (which can't happen), return the default + context :header:`From` address so we can keep on working. + + :param str address: Our email address, as set in the + ``EMAIL_SMTP_FROM`` config option. + :param list addressList: A list of 2-tuples of strings, the first + string is a full name, username, common name, etc., and the second + is the entity's email address. + """ + address = self.context.fromAddr + addressList = incoming.getaddrlist("To") + + try: + ours = smtp.Address(address) + except smtp.AddressError as error: + logging.warn("Our address seems invalid: %r" % address) + logging.warn(error) + else: + for _, addr in addressList: + try: + maybeOurs = smtp.Address(addr) + except smtp.AddressError: + pass + else: + # See if the user looks familiar. We do a 'find' instead of + # compare because we might have a '+' address here. + if maybeOurs.local.find(ours.local) != -1: + return '@'.join([maybeOurs.local, maybeOurs.domain]) + return address + + def getCanonicalDomain(self, domain): + try: + canonical = canonicalizeEmailDomain(domain, self.context.canon) + except (UnsupportedDomain, BadEmail) as error: + logging.warn(error) + else: + return canonical + + def reply(self): + """Reply to an incoming email. Maybe. + + If no `response` is returned from :func:`createMailResponse`, then the + incoming email will not be responded to at all. This can happen for + several reasons, for example: if the DKIM signature was invalid or + missing, or if the incoming email came from an unacceptable domain, or + if there have been too many emails from this client in the allotted + time period. + + :rtype: :api:`twisted.internet.defer.Deferred` + :returns: A ``Deferred`` which will callback when the response has + been successfully sent, or errback if an error occurred while + sending the email. + """ + logging.info("Got an email; deciding whether to reply.") + + def _replyEB(fail): + """Errback for a :api:`twisted.mail.smtp.SMTPSenderFactory`. + + :param fail: A :api:`twisted.python.failure.Failure` which occurred during + the transaction. + """ + logging.debug("_replyToMailEB() called with %r" % fail) + error = fail.getTraceback() or "Unknown" + logging.error(error) + + d = defer.Deferred() + d.addErrback(_replyEB) + + incoming = self.getIncomingMessage() + recipient = self.getRecipient(incoming) + client = self.getClientAddress(incoming) + + if not client: + return d + + if not self.fromCanonical: + self.fromCanonical = self.getCanonicalDomain(client.domain) + rules = self.context.domainRules.get(self.fromCanonical, []) + if not checkDKIM(incoming, rules): + return d + + clientAddr = '@'.join([client.local, client.domain]) + messageID = incoming.getheader("Message-ID", None) + subject = incoming.getheader("Subject", None) or "[no subject]" + + # Look up the locale part in the 'To:' address, if there is one and + # get the appropriate Translation object: + lang = translations.getLocaleFromPlusAddr(recipient) + logging.info("Client requested email translation: %s" % lang) + + body = createResponseBody(self.lines, self.context, clientAddr, lang) + if not body: return d # The client was already warned. + + response = generateResponse(self.context.fromAddr, clientAddr, subject, + body, messageID, self.context.gpgContext) + if not response: return d + + logging.info("Sending reply to %s" % client) + factory = smtp.SMTPSenderFactory(self.context.smtpFromAddr, clientAddr, + response, d, retries=0, timeout=30) + reactor.connectTCP(self.context.smtpServerIP, + self.context.smtpServerPort, + factory) + return d + + +class MailDelivery(object): + """Plugs into Twisted Mail and handles SMTP commands.""" + implements(smtp.IMessageDelivery) + + def setBridgeDBContext(self, context): + self.context = context + self.fromCanonical = None + + def receivedHeader(self, helo, origin, recipients): + """Create the ``Received:`` header for an incoming email. + + :type helo: tuple + :param helo: The lines received during SMTP client HELO. + :type origin: :api:`twisted.mail.smtp.Address` + :param origin: The email address of the sender. + :type recipients: list + :param recipients: A list of :api:`twisted.mail.smtp.User` instances. + """ + cameFrom = "%s (%s [%s])" % (helo[0] or origin, helo[0], helo[1]) + cameFor = ', '.join(["<{0}>".format(recp.dest) for recp in recipients]) + hdr = str("Received: from %s for %s; %s" + % (cameFrom, cameFor, smtp.rfc822date())) + return hdr + + def validateFrom(self, helo, origin): + try: + logging.debug("ORIGIN: %r" % repr(origin.addrstr)) + canonical = canonicalizeEmailDomain(origin.domain, + self.context.canon) + except UnsupportedDomain as error: + logging.info(error) + raise smtp.SMTPBadSender(origin.domain) + except Exception as error: + logging.exception(error) + else: + logging.debug("Got canonical domain: %r" % canonical) + self.fromCanonical = canonical + return origin # This method *cannot* return None, or it'll cause a 503. + + def validateTo(self, user): + """If the local user that was addressed isn't our configured local user + or doesn't contain a '+' with a prefix matching the local configured + user: Yell. + """ + u = user.dest.local + # Hasplus? If yes, strip '+foo' + idx = u.find('+') + if idx != -1: + u = u[:idx] + if u != self.context.username: + raise smtp.SMTPBadRcpt(user) + return lambda: MailMessage(self.context, self.fromCanonical) + + +class MailFactory(smtp.SMTPFactory): + """Plugs into Twisted Mail; creates a new MailDelivery whenever we get + a connection on the SMTP port.""" + + def __init__(self, context=None, **kw): + smtp.SMTPFactory.__init__(self, **kw) + self.delivery = MailDelivery() + if context: + self.setBridgeDBContext(context) + + def setBridgeDBContext(self, context): + self.context = context + self.delivery.setBridgeDBContext(context) + + def buildProtocol(self, addr): + p = smtp.SMTPFactory.buildProtocol(self, addr) + p.delivery = self.delivery + return p + + +def addServer(config, distributor, schedule): + """Set up a SMTP server for responding to requests for bridges. + + :param config: A configuration object from Main. We use these + options:: + EMAIL_BIND_IP + EMAIL_PORT + EMAIL_N_BRIDGES_PER_ANSWER + EMAIL_DOMAIN_RULES + :type distributor: :class:`bridgedb.Dist.EmailBasedDistributor` + :param dist: A distributor which will handle database interactions, and + will decide which bridges to give to who and when. + :type schedule: :class:`bridgedb.Time.IntervalSchedule` + :param schedule: The schedule. XXX: Is this even used? + """ + context = MailContext(config, distributor, schedule) + factory = MailFactory(context) + + addr = config.EMAIL_BIND_IP or "" + port = config.EMAIL_PORT + + reactor.listenTCP(port, factory, interface=addr) + + # Set up a LoopingCall to run every 30 minutes and forget old email times. + lc = LoopingCall(distributor.cleanDatabase) + lc.start(1800, now=False) + + return factory diff --git a/lib/bridgedb/email/templates.py b/lib/bridgedb/email/templates.py new file mode 100644 index 0000000..6c25038 --- /dev/null +++ b/lib/bridgedb/email/templates.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_email_templates -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft <isis(a)torproject.org> 0xA3ADB67A2CDB8B35 +# please also see AUTHORS file +# :copyright: (c) 2007-2014, The Tor Project, Inc. +# (c) 2013-2014, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +"""Templates for formatting emails sent out by the email distributor.""" + +from __future__ import print_function +from __future__ import unicode_literals + +import logging +import os + +from bridgedb import strings +from bridgedb.Dist import MAX_EMAIL_RATE +from bridgedb.HTTPServer import TEMPLATE_DIR + + +def buildCommands(template): + # Tell them about the various email commands: + cmdlist = [] + cmdlist.append(template.gettext(strings.EMAIL_MISC_TEXT.get(3))) + for cmd, desc in strings.EMAIL_COMMANDS.items(): + command = ' ' + command += cmd + while not len(command) >= 25: # Align the command descriptions + command += ' ' + command += template.gettext(desc) + cmdlist.append(command) + + commands = "\n".join(cmdlist) + "\n\n" + # And include the currently supported transports: + commands += template.gettext(strings.EMAIL_MISC_TEXT.get(5)) + commands += "\n" + for pt in strings.CURRENT_TRANSPORTS: + commands += ' ' + pt + "\n" + + return commands + +def buildHowto(template): + howToTBB = template.gettext(strings.HOWTO_TBB[1]) % strings.EMAIL_SPRINTF["HOWTO_TBB1"] + howToTBB += u'\n\n' + howToTBB += template.gettext(strings.HOWTO_TBB[2]) + howToTBB += u'\n\n' + howToTBB += u'\n'.join(["> {0}".format(ln) for ln in + template.gettext(strings.HOWTO_TBB[3]).split('\n')]) + howToTBB += u'\n\n' + howToTBB += template.gettext(strings.HOWTO_TBB[4]) + howToTBB += u'\n\n' + howToTBB += strings.EMAIL_REFERENCE_LINKS.get("HOWTO_TBB1") + howToTBB += u'\n\n' + return howToTBB + +def buildKeyfile(template): + filename = os.path.join(TEMPLATE_DIR, 'bridgedb.asc') + + try: + with open(filename) as fh: + keyFile = fh.read() + except Exception as error: # pragma: no cover + logging.exception(error) + keyFile = u'' + else: + keyFile += u'\n\n' + + return keyFile + +def buildWelcomeText(template): + sections = [] + sections.append(template.gettext(strings.EMAIL_MISC_TEXT[4])) + + commands = buildCommands(template) + sections.append(commands) + + # Include the same messages as the homepage of the HTTPS distributor: + welcome = template.gettext(strings.WELCOME[0]) % strings.EMAIL_SPRINTF["WELCOME0"] + welcome += template.gettext(strings.WELCOME[1]) + welcome += template.gettext(strings.WELCOME[2]) % strings.EMAIL_SPRINTF["WELCOME2"] + sections.append(welcome) + + message = u"\n\n".join(sections) + # Add the markdown links at the end: + message += strings.EMAIL_REFERENCE_LINKS.get("WELCOME0") + message += u"\n" + + return message + +def buildBridgeAnswer(template): + # Give the user their bridges, i.e. the `answer`: + message = template.gettext(strings.EMAIL_MISC_TEXT[0]) + u"\n\n" \ + + template.gettext(strings.EMAIL_MISC_TEXT[1]) + u"\n\n" \ + + u"%s\n\n" + return message + +def buildMessage(template): + message = None + try: + message = buildBridgeAnswer(template) + message += buildHowto(template) + message += u'\n\n' + message += buildCommands(template) + except Exception as error: # pragma: no cover + logging.error("Error while formatting email message template:") + logging.exception(error) + return message + +def buildSpamWarning(template): + message = None + try: + message = template.gettext(strings.EMAIL_MISC_TEXT[0]) + u"\n\n" \ + + template.gettext(strings.EMAIL_MISC_TEXT[2]) + u"\n" + message = message % str(MAX_EMAIL_RATE / 3600) + except Exception as error: # pragma: no cover + logging.error("Error while formatting email spam template:") + logging.exception(error) + return message diff --git a/setup.py b/setup.py index 1b8c259..e21dbef 100644 --- a/setup.py +++ b/setup.py @@ -278,6 +278,7 @@ setuptools.setup( download_url='https://gitweb.torproject.org/bridgedb.git', package_dir={'': 'lib'}, packages=['bridgedb', + 'bridgedb.email', 'bridgedb.parse', 'bridgedb.test'], scripts=['scripts/bridgedb'],

1 0

[bridgedb/master] Simplify handling of incoming emails without a Subject: header.
by isis＠torproject.org 16 May '14

16 May '14

commit 712bacce51622f9fbfb67fd267bad6fd09d1c412 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 5 20:55:43 2014 +0000 Simplify handling of incoming emails without a Subject: header. --- lib/bridgedb/email/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py index 97ddcde..838093e 100644 --- a/lib/bridgedb/email/server.py +++ b/lib/bridgedb/email/server.py @@ -143,7 +143,7 @@ def createResponseBody(lines, context, toAddress, lang='en'): request=toAddress) for b in bridges) return templates.buildMessage(t) % answer -def generateResponse(fromAddress, clientAddress, subject, body, +def generateResponse(fromAddress, clientAddress, body, subject=None, messageID=None, gpgContext=None): """Create a :class:`MailResponse`, which acts like an in-memory ``io.StringIO`` file, by creating and writing all headers and the email @@ -619,7 +619,7 @@ class MailMessage(object): clientAddr = '@'.join([client.local, client.domain]) messageID = incoming.getheader("Message-ID", None) - subject = incoming.getheader("Subject", None) or "[no subject]" + subject = incoming.getheader("Subject", None) # Look up the locale part in the 'To:' address, if there is one and # get the appropriate Translation object: @@ -629,8 +629,8 @@ class MailMessage(object): body = createResponseBody(self.lines, self.context, clientAddr, lang) if not body: return d # The client was already warned. - response = generateResponse(self.context.fromAddr, clientAddr, subject, - body, messageID, self.context.gpgContext) + response = generateResponse(self.context.fromAddr, clientAddr, body, + subject, messageID, self.context.gpgContext) if not response: return d logging.info("Sending reply to %s" % client)

1 0

[bridgedb/master] Move tests for getGPGContext() into the test_crypto unittest file.
by isis＠torproject.org 16 May '14

16 May '14

commit 3c1ec086c1148eed86d393ba7c2702841dd1bee6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 5 20:32:34 2014 +0000 Move tests for getGPGContext() into the test_crypto unittest file. --- lib/bridgedb/test/test_crypto.py | 68 +++++++++++++++++++ lib/bridgedb/test/test_email_server.py | 112 ++++++++------------------------ 2 files changed, 96 insertions(+), 84 deletions(-) diff --git a/lib/bridgedb/test/test_crypto.py b/lib/bridgedb/test/test_crypto.py index 5d69a55..238002d 100644 --- a/lib/bridgedb/test/test_crypto.py +++ b/lib/bridgedb/test/test_crypto.py @@ -16,6 +16,7 @@ from __future__ import unicode_literals import logging import os +import shutil import OpenSSL @@ -26,6 +27,8 @@ from twisted.web.test import test_agent as txtagent from bridgedb import crypto from bridgedb import txrecaptcha +from bridgedb.persistent import Conf +from bridgedb.test.util import fileCheckDecorator logging.disable(50) @@ -160,3 +163,68 @@ class SSLVerifyingContextFactoryTests(unittest.TestCase, contextFactory = crypto.SSLVerifyingContextFactory(self.url) self.assertIsInstance(contextFactory.getContext(), OpenSSL.SSL.Context) + + +class GetGPGContextTest(unittest.TestCase): + """Unittests for :func:`bridgedb.crypto.getGPGContext`.""" + + timeout = 15 + + @fileCheckDecorator + def doCopyFile(self, src, dst, description=None): + shutil.copy(src, dst) + + def removeRundir(self): + if os.path.isdir(self.runDir): + shutil.rmtree(self.runDir) + + def makeBadKey(self): + self.setKey(self.badKeyfile) + + def setKey(self, keyfile=''): + setattr(self.config, 'EMAIL_GPG_SIGNING_KEY', keyfile) + + def setUp(self): + here = os.getcwd() + topDir = here.rstrip('_trial_temp') + self.runDir = os.path.join(here, 'rundir') + self.gpgMoved = os.path.join(self.runDir, 'TESTING.subkeys.sec') + self.gpgFile = os.path.join(topDir, 'gnupghome', + 'TESTING.subkeys.sec') + + if not os.path.isdir(self.runDir): + os.makedirs(self.runDir) + + self.badKeyfile = os.path.join(here, 'badkey.asc') + with open(self.badKeyfile, 'w') as badkey: + badkey.write('NO PASARAN, DEATH CAKES!') + badkey.flush() + + self.doCopyFile(self.gpgFile, self.gpgMoved, "GnuPG test keyfile") + + self.config = Conf() + setattr(self.config, 'EMAIL_GPG_SIGNING_ENABLED', True) + setattr(self.config, 'EMAIL_GPG_SIGNING_KEY', + 'gnupghome/TESTING.subkeys.sec') + + self.addCleanup(self.removeRundir) + + def test_getGPGContext_good_keyfile(self): + """Test EmailServer.getGPGContext() with a good key filename.""" + self.skip = True + raise unittest.SkipTest("see ticket #5264") + + ctx = crypto.getGPGContext(self.config) + self.assertIsInstance(ctx, crypto.gpgme.Context) + + def test_getGPGContext_missing_keyfile(self): + """Test EmailServer.getGPGContext() with a missing key filename.""" + self.setKey('missing-keyfile.asc') + ctx = crypto.getGPGContext(self.config) + self.assertTrue(ctx is None) + + def test_getGPGContext_bad_keyfile(self): + """Test EmailServer.getGPGContext() with a missing key filename.""" + self.makeBadKey() + ctx = crypto.getGPGContext(self.config) + self.assertTrue(ctx is None) diff --git a/lib/bridgedb/test/test_email_server.py b/lib/bridgedb/test/test_email_server.py index e925f07..f980d83 100644 --- a/lib/bridgedb/test/test_email_server.py +++ b/lib/bridgedb/test/test_email_server.py @@ -13,15 +13,14 @@ from __future__ import print_function -import os -import shutil - import io import copy +import os +import shutil +import types from bridgedb.Dist import EmailBasedDistributor from bridgedb.email import server -from bridgedb.email.server import MailContext from bridgedb.Time import NoSchedule from bridgedb.parse.addr import BadEmail from bridgedb.persistent import Conf @@ -57,20 +56,25 @@ EMAIL_BIND_IP = "127.0.0.1" EMAIL_PORT = 5225 """)) -def _createMailContext(distributor=None): +def _createConfig(configFile=TEST_CONFIG_FILE): configuration = {} TEST_CONFIG_FILE.seek(0) - compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') + compiled = compile(configFile.read(), '<string>', 'exec') exec compiled in configuration config = Conf(**configuration) + return config + +def _createMailContext(config=None, distributor=None): + if not config: + config = _createConfig() if not distributor: distributor = DummyEmailDistributor( domainmap=config.EMAIL_DOMAIN_MAP, domainrules=config.EMAIL_DOMAIN_RULES) - ctx = MailContext(config, distributor, NoSchedule()) - return ctx + context = server.MailContext(config, distributor, NoSchedule()) + return context class DummyEmailDistributor(object): @@ -92,85 +96,27 @@ class DummyEmailDistributor(object): countryCode=None, bridgeFilterRules=None): return [DummyBridge() for _ in xrange(N)] + def cleanDatabase(self): + pass -class EmailGnuPGTest(unittest.TestCase): - """Tests for :func:`bridgedb.EmailServer.getGPGContext`.""" - - timeout = 15 - @fileCheckDecorator - def doCopyFile(self, src, dst, description=None): - shutil.copy(src, dst) +class CreateResponseBodyTests(unittest.TestCase): + """Tests for :func:`bridgedb.email.server.createResponseBody`.""" - def removeRundir(self): - if os.path.isdir(self.runDir): - shutil.rmtree(self.runDir) - - def makeBadKey(self): - keyfile = os.path.join(self.runDir, 'badkey.asc') - with open(keyfile, 'wb') as badkey: - badkey.write('NO PASARÁN, DEATH CAKES!') - badkey.flush() - self.setKey(keyfile) - - def setKey(self, keyfile=''): - setattr(self.config, 'EMAIL_GPG_SIGNING_KEY', keyfile) - - def setUp(self): + def _moveGPGTestKeyfile(self): here = os.getcwd() topDir = here.rstrip('_trial_temp') - self.runDir = os.path.join(here, 'rundir') self.gpgFile = os.path.join(topDir, 'gnupghome', 'TESTING.subkeys.sec') self.gpgMoved = os.path.join(here, 'TESTING.subkeys.sec') - - if not os.path.isdir(self.runDir): - os.makedirs(self.runDir) - - configuration = {} - TEST_CONFIG_FILE.seek(0) - compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') - exec compiled in configuration - self.config = Conf(**configuration) - - self.addCleanup(self.removeRundir) - - def test_getGPGContext_good_keyfile(self): - """Test EmailServer.getGPGContext() with a good key filename. - - XXX: See #5463. - """ - raise unittest.SkipTest( - "See #5463 for why this test fails when it should pass") - - self.doCopyFile(self.gpgFile, self.gpgMoved, "GnuPG test keyfile") - ctx = EmailServer.getGPGContext(self.config) - self.assertIsInstance(ctx, EmailServer.gpgme.Context) - - def test_getGPGContext_missing_keyfile(self): - """Test EmailServer.getGPGContext() with a missing key filename.""" - self.setKey('missing-keyfile.asc') - ctx = EmailServer.getGPGContext(self.config) - self.assertTrue(ctx is None) - - def test_getGPGContext_bad_keyfile(self): - """Test EmailServer.getGPGContext() with a missing key filename.""" - self.makeBadKey() - ctx = EmailServer.getGPGContext(self.config) - self.assertTrue(ctx is None) - - -class EmailResponseTests(unittest.TestCase): - """Tests for :func:`bridgedb.EmailServer.getMailResponse`.""" + shutil.copy(self.gpgFile, self.gpgMoved) def setUp(self): """Create fake email, distributor, and associated context data.""" - # TODO: Add headers if we start validating them - self.lines = ["From: %s(a)%s.com", - "To: bridges@localhost", - "Subject: testing", - "", - "get bridges"] - self.ctx = _createMailContext() + self._moveGPGTestKeyfile() + self.toAddress = "user(a)example.com" + self.config = _createConfig() + self.ctx = _createMailContext(self.config) + self.distributor = self.ctx.distributor def _isTwoTupleOfNone(self, reply): """Check that a return value is ``(None, None)``.""" @@ -328,14 +274,12 @@ class EmailReplyTests(unittest.TestCase): class EmailServerServiceTests(unittest.TestCase): def setUp(self): - # TODO: Add headers if we start validating them - self.lines = ["From: %s(a)%s.com", "To: %s(a)example.net", - "Subject: testing", "\n", "get bridges"] - self.distributor = DummyEmailDistributor('key', {}, {}, []) - self.ctx = _createMailContext(self.distributor) + self.config = _createConfig() + self.context = _createMailContext(self.config) + self.distributor = self.context.distributor - def test_receiveMail(self): + def test_addServer(self): self.skip = True raise unittest.SkipTest("Not finished yet") from twisted.internet import reactor - EmailServer.addSMTPServer(self.ctx.cfg, self.distributor, NoSchedule) + server.addServer(self.config, self.distributor, NoSchedule)

1 0

[bridgedb/master] Change unittests for bridgedb.email.server.createResponseBody().
by isis＠torproject.org 16 May '14

16 May '14

commit 051ebe5965b8dc32f604bb69f0a303ac3f32f9a5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 5 20:35:27 2014 +0000 Change unittests for bridgedb.email.server.createResponseBody(). This changes the unittests for EmailServer.getMailResponse() to test bridgedb.email.server.createResponseBody() instead. --- lib/bridgedb/test/test_email_server.py | 146 ++++++++++++-------------------- 1 file changed, 56 insertions(+), 90 deletions(-) diff --git a/lib/bridgedb/test/test_email_server.py b/lib/bridgedb/test/test_email_server.py index f980d83..a308056 100644 --- a/lib/bridgedb/test/test_email_server.py +++ b/lib/bridgedb/test/test_email_server.py @@ -118,127 +118,93 @@ class CreateResponseBodyTests(unittest.TestCase): self.ctx = _createMailContext(self.config) self.distributor = self.ctx.distributor - def _isTwoTupleOfNone(self, reply): - """Check that a return value is ``(None, None)``.""" - self.assertIsInstance(reply, tuple) - self.assertEqual(len(reply), 2) - self.assertEqual(reply[0], None) - self.assertEqual(reply[1], None) - - def _isTwoTupleOfAddrAndClass(self, reply, address="testing@localhost", - klass=io.StringIO): - self.assertIsInstance(reply, tuple) - self.assertEqual(len(reply), 2) - self.assertEqual(reply[0], address) - self.assertIsInstance(reply[1], klass) - - def test_getMailResponse_noFrom(self): + def _getIncomingLines(self, clientAddress="user(a)example.com"): + """Generate the lines of an incoming email from **clientAddress**.""" + self.toAddress = clientAddress + lines = [ + "From: %s" % clientAddress, + "To: bridges@localhost", + "Subject: testing", + "", + "get bridges", + ] + return lines + + def test_createResponseBody_noFrom(self): """A received email without a "From:" or "Sender:" header shouldn't receive a response. """ - lines = self.lines + lines = self._getIncomingLines() lines[0] = "" - ret = EmailServer.getMailResponse(lines, self.ctx) - self._isTwoTupleOfNone(ret) + ret = server.createResponseBody(lines, self.ctx, self.toAddress) + self.assertIsNone(ret) - def test_getMailResponse_badAddress(self): + def test_createResponseBody_badAddress(self): """Don't respond to RFC2822 malformed source addresses.""" - lines = copy.copy(self.lines) - lines[0] = self.lines[0] % ("testing*.?\"", "example") - ret = EmailServer.getMailResponse(lines, self.ctx) - self._isTwoTupleOfNone(ret) + lines = self._getIncomingLines("testing*.?\"@example.com") + ret = server.createResponseBody(lines, self.ctx, self.toAddress) + self.assertIsNone(ret) - def test_getMailResponse_anotherBadAddress(self): + def test_createResponseBody_anotherBadAddress(self): """Don't respond to RFC2822 malformed source addresses.""" - lines = copy.copy(self.lines) - lines[0] = "From: Mallory %s(a)%s.com" % ("<>>", "example") - ret = EmailServer.getMailResponse(lines, self.ctx) - self._isTwoTupleOfNone(ret) + lines = self._getIncomingLines("<>>@example.com") + lines[0] = "From: Mallory %s" % self.toAddress + ret = server.createResponseBody(lines, self.ctx, self.toAddress) + self.assertIsNone(ret) - def test_getMailResponse_invalidDomain(self): + def test_createResponseBody_invalidDomain(self): """Don't respond to RFC2822 malformed source addresses.""" - lines = copy.copy(self.lines) - lines[0] = self.lines[0] % ("testing", "exa#mple") - ret = EmailServer.getMailResponse(lines, self.ctx) - self._isTwoTupleOfNone(ret) + lines = self._getIncomingLines("testing(a)exa#mple.com") + ret = server.createResponseBody(lines, self.ctx, self.toAddress) + self.assertIsNone(ret) - def test_getMailResponse_anotherInvalidDomain(self): + def test_createResponseBody_anotherInvalidDomain(self): """Don't respond to RFC2822 malformed source addresses.""" - lines = copy.copy(self.lines) - lines[0] = self.lines[0] % ("testing", "exam+ple") - ret = EmailServer.getMailResponse(lines, self.ctx) - self._isTwoTupleOfNone(ret) + lines = self._getIncomingLines("testing(a)exam+ple.com") + ret = server.createResponseBody(lines, self.ctx, self.toAddress) + self.assertIsNone(ret) - def test_getMailResponse_DKIM_badDKIMheader(self): + def test_createResponseBody_DKIM_badDKIMheader(self): """An email with an 'X-DKIM-Authentication-Result:' header appended after the body should not receive a response. """ - lines = copy.copy(self.lines) - lines[0] = self.lines[0] % ("testing", "gmail") + lines = self._getIncomingLines("testing(a)gmail.com") lines.append("X-DKIM-Authentication-Result: ") - ret = EmailServer.getMailResponse(lines, self.ctx) - self._isTwoTupleOfNone(ret) + ret = server.createResponseBody(lines, self.ctx, self.toAddress) + self.assertIsNone(ret) - def test_getMailResponse_DKIM(self): + def test_createResponseBody_DKIM(self): """An email with a good DKIM header should be responded to.""" - lines = copy.copy(self.lines) - lines[0] = self.lines[0] % ("testing", "localhost") + lines = self._getIncomingLines("testing@localhost") lines.insert(3, "X-DKIM-Authentication-Result: ") - ret = EmailServer.getMailResponse(lines, self.ctx) - self.skip = True - raise unittest.SkipTest("Broken; not sure why. Manual testing says"\ - " the email distributor should pass these"\ - " tests.") - self._isTwoTupleOfAddrAndClass(ret) - mail = ret[1].getvalue() - self.assertEqual(mail.find("no bridges currently"), -1) - - def test_getMailResponse_bridges_obfs3(self): + ret = server.createResponseBody(lines, self.ctx, self.toAddress) + self.assertEqual(ret.find("no bridges currently"), -1) + + def test_createResponseBody_bridges_obfs3(self): """A request for 'transport obfs3' should receive a response.""" - lines = copy.copy(self.lines) - lines[0] = self.lines[0] % ("testing", "localhost") + lines = self._getIncomingLines("testing@localhost") lines[4] = "transport obfs3" - ret = EmailServer.getMailResponse(lines, self.ctx) - self.skip = True - raise unittest.SkipTest("Broken; not sure why. Manual testing says"\ - " the email distributor should pass these"\ - " tests.") - self._isTwoTupleOfAddrAndClass(ret) - mail = ret[1].getvalue() - self.assertEqual(mail.find("no bridges currently"), -1) - - def test_getMailResponse_bridges_obfsobfswebz(self): + ret = server.createResponseBody(lines, self.ctx, self.toAddress) + self.assertEqual(ret.find("no bridges currently"), -1) + + def test_createResponseBody_bridges_obfsobfswebz(self): """We should only pay attention to the *last* in a crazy request.""" - lines = copy.copy(self.lines) - lines[0] = self.lines[0] % ("testing", "localhost") + lines = self._getIncomingLines("testing@localhost") lines[4] = "unblocked webz" lines.append("transport obfs2") lines.append("transport obfs3") - ret = EmailServer.getMailResponse(lines, self.ctx) - self.skip = True - raise unittest.SkipTest("Broken; not sure why. Manual testing says"\ - " the email distributor should pass these"\ - " tests.") - self._isTwoTupleOfAddrAndClass(ret) - mail = ret[1].getvalue() - self.assertNotEqual(mail.find("no bridges currently"), -1) - - def test_getMailResponse_bridges_obfsobfswebzipv6(self): + ret = server.createResponseBody(lines, self.ctx, self.toAddress) + self.assertNotEqual(ret.find("no bridges currently"), -1) + + def test_createResponseBody_bridges_obfsobfswebzipv6(self): """We should *still* only pay attention to the *last* request.""" - lines = copy.copy(self.lines) - lines[0] = self.lines[0] % ("testing", "localhost") + lines = self._getIncomingLines("testing@localhost") lines[4] = "transport obfs3" lines.append("unblocked webz") lines.append("ipv6") lines.append("transport obfs2") - ret = EmailServer.getMailResponse(lines, self.ctx) - self.skip = True - raise unittest.SkipTest("Broken; not sure why. Manual testing says"\ - " the email distributor should pass these"\ - " tests.") - self._isTwoTupleOfAddrAndClass(ret) - mail = ret[1].getvalue() - self.assertNotEqual(mail.find("no bridges currently"), -1) + ret = server.createResponseBody(lines, self.ctx, self.toAddress) + self.assertNotEqual(ret.find("no bridges currently"), -1) class EmailReplyTests(unittest.TestCase):

1 0