April 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Move locate determination and translation code into separate module.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit ed438d0c2b00f02c29b60e561d37f241f6a6a4b8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 16:22:40 2014 +0000 Move locate determination and translation code into separate module. * ADD new bridgedb.translations module with the following methods: - getFirstSupportedLang(): Essentially the same as the original HTTPServer.getAssumedChosenLang() function, except that the determination of which languages are supported uses _lang.getLangs() instead (this has been in the _lang module for a long time, and was meant for that purpose). - getLocaleFromHTTPRequest(): This is taken mostly from the original setLocaleFromRequestHeader() (in the HTTPServer module) except that there was an extra function in the EmailServer module, called getLocaleFromRequest() ― which was not in use anywhere ― which included the parsing of '?lang=' HTTP GET/POST arguments. The parsing of '?lang=' was taken from that function and added to the original functionality of setLocaleFromRequestHeader(). This fixes part of ticket #9678: "'Select Language' button on bridges.tpo". - getLocaleFromPlusAddr(): Exactly the same as the original in the EmailServer module. - installTranslations(): Produces a gettext translation object, with fallback languages chained to it in order of client preference. - usingRTLLang(): Essentially the same as the original in the HTTPServer module, except that the parsing of supported langs again uses the _lang.getLangs() function, as above. * FIXES part of #9678: "'Select Language' button to bridges.tpo" --- lib/bridgedb/EmailServer.py | 29 ++-------- lib/bridgedb/HTTPServer.py | 94 +++----------------------------- lib/bridgedb/translations.py | 122 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 113 deletions(-) diff --git a/lib/bridgedb/EmailServer.py b/lib/bridgedb/EmailServer.py index f9f43bb..c58ec53 100644 --- a/lib/bridgedb/EmailServer.py +++ b/lib/bridgedb/EmailServer.py @@ -30,6 +30,7 @@ from bridgedb.Dist import BadEmail, TooSoonEmail, IgnoreEmail from bridgedb import Dist from bridgedb import I18n from bridgedb import safelog +from bridgedb import translations from bridgedb.Filters import filterBridgesByIP6 from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByTransport @@ -94,8 +95,8 @@ def getMailResponse(lines, ctx): # Look up the locale part in the 'To:' address, if there is one and get # the appropriate Translation object - lang = getLocaleFromPlusAddr(clientToaddr) - t = I18n.getLang(lang) + lang = translations.getLocaleFromPlusAddr(clientToaddr) + t = translations.installTranslations(lang) try: _, addrdomain = Dist.extractAddrSpec(clientAddr.lower()) @@ -287,30 +288,6 @@ def replyToMail(lines, ctx): reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory) return d -def getLocaleFromPlusAddr(address): - """See whether the user sent his email to a 'plus' address, for - instance to bridgedb+fa@tpo. Plus addresses are the current - mechanism to set the reply language - """ - replyLocale = "en" - r = '.*(<)?(\w+\+(\w+)@\w+(?:\.\w+)+)(?(1)>)' - match = re.match(r, address) - if match: - replyLocale = match.group(3) - - return replyLocale - -def getLocaleFromRequest(request): - # See if we did get a request for a certain locale, otherwise fall back - # to 'en': - # Try evaluating the path /foo first, then check if we got a ?lang=foo - default_lang = lang = "en" - if len(request.path) > 1: - lang = request.path[1:] - if lang == default_lang: - lang = request.args.get("lang", [default_lang]) - lang = lang[0] - return I18n.getLang(lang) class MailContext(object): """Helper object that holds information used by email subsystem.""" diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 5e3ba37..79772c3 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -35,6 +35,7 @@ import bridgedb.I18n as I18n from bridgedb import captcha from bridgedb import crypto +from bridgedb import translations from bridgedb import txrecaptcha from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByIP6 @@ -541,9 +542,10 @@ class WebResourceOptions(resource.Resource): def render_GET(self, request): rtl = False + langs = translations.getLocaleFromHTTPRequest(request) try: - rtl = usingRTLLang(request) + rtl = translations.usingRTLLang(langs) except Exception as err: # pragma: no cover logging.exception(err) @@ -664,7 +666,8 @@ class WebResourceBridges(resource.Resource): if countryCode: logging.debug("Client request from GeoIP CC: %s" % countryCode) - rtl = usingRTLLang(request) + langs = translations.getLocaleFromHTTPRequest(request) + rtl = translations.usingRTLLang(langs) if rtl: logging.debug("Rendering RTL response.") @@ -784,9 +787,10 @@ class WebRoot(resource.Resource): :param request: An incoming request. """ rtl = False + langs = translations.getLocaleFromHTTPRequest(request) try: - rtl = usingRTLLang(request) + rtl = translations.usingRTLLang(langs) except Exception as err: logging.exception(err) logging.error("The gettext files were not properly installed.") @@ -883,87 +887,3 @@ def addWebServer(cfg, dist, sched): raise SystemExit(error) return site - -def usingRTLLang(request): - """Check if we should translate the text into a RTL language - - Retrieve the headers from the request. Obtain the Accept-Language header - and decide if we need to translate the text. Install the requisite - languages via gettext, if so. Then, manually check which languages we - support. Choose the first language from the header that we support and - return True if it is a RTL language, else return False. - - :type request: :api:`twisted.web.server.Request` - :param request: An incoming request. - :rtype: bool - :returns: ``True`` if the preferred language is right-to-left; ``False`` - otherwise. - """ - langs = setLocaleFromRequestHeader(request) - - # Grab only the language (first two characters) so we know if the language - # is read right-to-left - #langs = [ lang[:2] for lang in langs ] - lang = getAssumedChosenLang(langs) - if lang in rtl_langs: - return True - return False - -def getAssumedChosenLang(langs): - """Return the first language in **langs** that we support. - - :param list langs: All requested languages - :rtype: str - :returns: A country code for the client's preferred language. - """ - i18npath = os.path.join(os.path.dirname(__file__), 'i18n') - path = filepath.FilePath(i18npath) - assert path.isdir() - - lang = 'en-US' - supp_langs = path.listdir() + ['en'] - for l in langs: - if l in supp_langs: - lang = l - break - return lang - -def setLocaleFromRequestHeader(request): - """Retrieve the languages from the accept-language header and install them. - - Parse the languages in the header, and attempt to install the first one in - the list. If that fails, we receive a :class:`gettext.NullTranslation` - object, if it worked then we have a :class:`gettext.GNUTranslation` - object. Whichever one we end up with, get the other languages and add them - as fallbacks to the first. Lastly, install this chain of translations. - - :type request: :api:`twisted.web.server.Request` - :param request: An incoming request from a client. - :rtype: list - :returns: All requested languages. - """ - logging.debug("Getting client 'Accept-Language' header...") - header = request.getHeader('accept-language') - - if header is None: - logging.debug("Client sent no 'Accept-Language' header. Using fallback.") - header = 'en,en-US' - - localedir = os.path.join(os.path.dirname(__file__), 'i18n/') - langs = headers.parseAcceptLanguage(header) - ## XXX the 'Accept-Language' header is potentially identifying - logging.debug("Client Accept-Language (top 5): %s" % langs[:5]) - - try: - language = gettext.translation("bridgedb", localedir=localedir, - languages=langs, fallback=True) - for lang in langs: - language.add_fallback(gettext.translation("bridgedb", - localedir=localedir, - languages=langs, - fallback=True)) - except IOError as error: - logging.error(error.message) - - language.install(unicode=True) - return langs diff --git a/lib/bridgedb/translations.py b/lib/bridgedb/translations.py new file mode 100644 index 0000000..3085938 --- /dev/null +++ b/lib/bridgedb/translations.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_translations -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2013-2014, Isis Lovecruft +# (c) 2007-2014, The Tor Project, Inc. +# :license: 3-Clause BSD, see LICENSE for licensing information + +import gettext +import logging +import os +import re + +from bridgedb import _langs +from bridgedb import safelog +from bridgedb.parse import headers + + +TRANSLATIONS_DIR = os.path.join(os.path.dirname(__file__), 'i18n') + + +def getFirstSupportedLang(langs): + """Return the first language in **langs** that we support. + + :param list langs: All requested languages + :rtype: str + :returns: A country code for the client's preferred language. + """ + lang = 'en-US' + supported = _langs.get_langs() + + for l in langs: + if l in supported: + lang = l + break + return lang + +def getLocaleFromHTTPRequest(request): + """Retrieve the languages from an HTTP ``Accept-Language:`` header. + + Parse the languages from the header, use them to install a + ``gettext.translation`` chain via :func:`installTranslations`, and lastly + return the requested languages. + + :type request: :api:`twisted.web.server.Request` + :param request: An incoming request from a client. + :rtype: list + :returns: All requested languages. + """ + header = request.getHeader('accept-language') + if header is None: + logging.debug("Client sent no 'Accept-Language' header. Using fallback.") + header = 'en,en-US' + + langs = headers.parseAcceptLanguage(header) + if not safelog.safe_logging: # pragma: no cover + logging.debug("Client Accept-Language (top 5): %s" % langs[:5]) + + # Check if we got a ?lang=foo argument, and if we did, insert it first + chosenLang = request.args.get("lang", [None,])[0] + if chosenLang: + logging.debug("Client requested language: %r" % chosenLang) + langs.insert(0, chosenLang) + + installTranslations(langs) + return langs + +def getLocaleFromPlusAddr(address): + """See whether the user sent his email to a 'plus' address, for instance to + bridges+fa(a)bridges.torproject.org. Plus addresses are the current + mechanism to set the reply language. + """ + replyLocale = "en" + r = '.*(<)?(\w+\+(\w+)@\w+(?:\.\w+)+)(?(1)>)' + match = re.match(r, address) + if match: + replyLocale = match.group(3) + + return replyLocale + +def installTranslations(langs): + """Create a ``gettext.translation`` chain for all **langs**. + + Attempt to install the first language in the **langs** list. If that + fails, we receive a ``gettext.NullTranslation`` object, and if it worked + then we have a ``gettext.GNUTranslation`` object. Whichever one we end up + with, get the other languages and add them as fallbacks to the + first. Lastly, install this chain of translations. + + :param list langs: A list of language codes. + :returns: A ``gettext.NullTranslation`` or ``gettext.GNUTranslation`` with + fallback languages set. + """ + try: + language = gettext.translation("bridgedb", localedir=TRANSLATIONS_DIR, + languages=langs, fallback=True) + for lang in langs: + language.add_fallback( + gettext.translation("bridgedb", localedir=TRANSLATIONS_DIR, + languages=langs, fallback=True)) + except IOError as error: + logging.error(error.message) + + language.install(unicode=True) + return language + +def usingRTLLang(langs): + """Check if we should translate the text into a RTL language. + + Choose the first language from the **langs** list that we support and + return True if it is a RTL language, else return False. + + :param list langs: An incoming request. + :rtype: bool + :returns: ``True`` if the preferred language is right-to-left; ``False`` + otherwise. + """ + lang = getFirstSupportedLang(langs) + if lang in _langs.RTL_LANGS: + return True + return False

1 0

[bridgedb/master] PEP8; change template_root → TEMPLATE_DIR.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 4a9eb669830ae8a3d12335227cce7c34b5002193 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 19:38:44 2014 +0000 PEP8; change template_root → TEMPLATE_DIR. --- lib/bridgedb/HTTPServer.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 5e3ba37..1d4cdf9 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -44,10 +44,7 @@ from bridgedb.parse import headers from bridgedb.safelog import logSafely - -template_root = os.path.join(os.path.dirname(__file__),'templates') -logging.debug("Set template root to %s" % template_root) - +TEMPLATE_DIR = os.path.join(os.path.dirname(__file__),'templates') rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') # Setting `filesystem_checks` to False is recommended for production servers, @@ -57,13 +54,14 @@ rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') # recompiled). `collection_size` sets the number of compiled templates which # are cached before the least recently used ones are removed. See: # http://docs.makotemplates.org/en/latest/usage.html#using-templatelookup -lookup = TemplateLookup(directories=[template_root], +lookup = TemplateLookup(directories=[TEMPLATE_DIR], output_encoding='utf-8', filesystem_checks=False, collection_size=500) _geoipdb = '/usr/share/GeoIP/GeoIP.dat' +logging.debug("Set template root to %s" % TEMPLATE_DIR) try: # Make sure we have the database before trying to import the module: @@ -822,9 +820,9 @@ def addWebServer(cfg, dist, sched): httpdist = resource.Resource() httpdist.putChild('', WebRoot()) httpdist.putChild('robots.txt', - static.File(os.path.join(template_root, 'robots.txt'))) + static.File(os.path.join(TEMPLATE_DIR, 'robots.txt'))) httpdist.putChild('assets', - static.File(os.path.join(template_root, 'assets/'))) + static.File(os.path.join(TEMPLATE_DIR, 'assets/'))) httpdist.putChild('options', WebResourceOptions()) httpdist.putChild('howto', WebResourceHowto())

1 0

[bridgedb/master] Add unittests for new bridgedb.translations module.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit a6edfaeb6fc8036c60d9c20bf871795cba198337 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 19:19:49 2014 +0000 Add unittests for new bridgedb.translations module. --- lib/bridgedb/test/test_translations.py | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/lib/bridgedb/test/test_translations.py b/lib/bridgedb/test/test_translations.py new file mode 100644 index 0000000..ee15ac4 --- /dev/null +++ b/lib/bridgedb/test/test_translations.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2014, Isis Lovecruft +# (c) 2014, The Tor Project, Inc. +# :license: 3-Clause BSD, see LICENSE for licensing information + + +from twisted.trial import unittest + +from bridgedb import translations +from bridgedb.test.test_HTTPServer import DummyRequest + + +REALISH_HEADERS = { + b'Accept-Encoding': [b'gzip, deflate'], + b'User-Agent': [ + b'Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0'], + b'Accept': [ + b'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'], +} + +# Add this to the above REALISH_HEADERS to use it: +ACCEPT_LANGUAGE_HEADER = { + b'Accept-Language': [b'de-de,en-gb;q=0.8,en;q=0.5,en-us;q=0.3'], +} + + +class TranslationsMiscTests(unittest.TestCase): + """Tests for module-level code in ``bridgedb.translations`` module.""" + + def test_getLocaleFromHTTPRequest_withLangParam(self): + """This request uses a '?lang=ar' param, without an 'Accept-Language' + header. + + The request result should be: ['ar', 'en', 'en-US']. + """ + request = DummyRequest([b"bridges"]) + request.headers.update(REALISH_HEADERS) + request.args.update({ + b'transport': [b'obfs3',], + b'lang': [b'ar',], + }) + + parsed = translations.getLocaleFromHTTPRequest(request) + self.assertEqual(parsed[0], 'ar') + self.assertEqual(parsed[1], 'en') + self.assertEqual(parsed[2], 'en_US') + self.assertEqual(len(parsed), 3) + + def test_getLocaleFromHTTPRequest_withLangParam_AcceptLanguage(self): + """This request uses a '?lang=ar' param, with an 'Accept-Language' + header which includes: ['de-de', 'en-gb', 'en', 'en-us']. + + The request result should be: ['fa', 'de-de', 'en-gb', 'en', 'en-us']. + """ + request = DummyRequest([b"options"]) + request.headers.update(ACCEPT_LANGUAGE_HEADER) + request.args.update({b'lang': [b'fa']}) + + parsed = translations.getLocaleFromHTTPRequest(request) + self.assertEqual(parsed[0], 'fa') + self.assertEqual(parsed[1], 'en') + self.assertEqual(parsed[2], 'en_US') + #self.assertEqual(parsed[3], 'en-gb') + self.assertEqual(len(parsed), 3) + + def test_getLocaleFromPlusAddr(self): + emailAddr = 'bridges(a)torproject.org' + replyLocale = translations.getLocaleFromPlusAddr(emailAddr) + self.assertEqual('en', replyLocale) + + def test_getLocaleFromPlusAddr_ar(self): + emailAddr = 'bridges+ar(a)torproject.org' + replyLocale = translations.getLocaleFromPlusAddr(emailAddr) + self.assertEqual('ar', replyLocale)

1 0

[bridgedb/master] Fix issue with transport URL params in HTTPServer unittests.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 5e37dc53df4704c9b4ab0f3943159a5198f0c515 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 19:26:19 2014 +0000 Fix issue with transport URL params in HTTPServer unittests. Also fixes several PEP8 whitespace issues in test_HTTPServer.py. --- lib/bridgedb/test/test_HTTPServer.py | 83 +++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index 111adc1..52c2fb7 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -178,8 +178,8 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): self.request.method = b'POST' self.request.addArg('captcha_challenge_field', expectedChallenge) - self.request.addArg('captcha_response_field', expectedResponse) - + self.request.addArg('captcha_response_field', expectedResponse) + response = self.captchaResource.extractClientSolution(self.request) (challenge, response) = response self.assertEqual(challenge, expectedChallenge) @@ -192,8 +192,8 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): self.request.method = b'POST' self.request.addArg('captcha_challenge_field', expectedChallenge) - self.request.addArg('captcha_response_field', expectedResponse) - + self.request.addArg('captcha_response_field', expectedResponse) + valid = self.captchaResource.checkSolution(self.request) self.assertFalse(valid) @@ -238,7 +238,7 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): """ self.request.method = b'POST' self.request.addArg('captcha_challenge_field', '') - self.request.addArg('captcha_response_field', '') + self.request.addArg('captcha_response_field', '') page = self.captchaResource.render_POST(self.request) self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'], @@ -253,7 +253,7 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): self.request.method = b'POST' self.request.addArg('captcha_challenge_field', expectedChallenge) - self.request.addArg('captcha_response_field', expectedResponse) + self.request.addArg('captcha_response_field', expectedResponse) page = self.captchaResource.render_POST(self.request) self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'], @@ -357,8 +357,8 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): """ self.request.method = b'POST' self.request.addArg('captcha_challenge_field', '') - self.request.addArg('captcha_response_field', '') - + self.request.addArg('captcha_response_field', '') + self.assertEqual((False, self.request), self.successResultOf( self.captchaResource.checkSolution(self.request))) @@ -398,7 +398,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): """ self.request.method = b'POST' self.request.addArg('captcha_challenge_field', '') - self.request.addArg('captcha_response_field', '') + self.request.addArg('captcha_response_field', '') page = self.captchaResource.render_POST(self.request) self.assertEqual(page, HTTPServer.server.NOT_DONE_YET) @@ -412,7 +412,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): self.request.method = b'POST' self.request.addArg('captcha_challenge_field', expectedChallenge) - self.request.addArg('captcha_response_field', expectedResponse) + self.request.addArg('captcha_response_field', expectedResponse) page = self.captchaResource.render_POST(self.request) self.assertEqual(page, HTTPServer.server.NOT_DONE_YET) @@ -580,34 +580,51 @@ class WebResourceBridgesTests(unittest.TestCase): page = self.bridgesResource.render(request) self.bridgesResource.useForwardedHeader = False # Reset it - self.assertSubstring("To enter bridges into", page) + + # The response should explain how to use the bridge lines: + self.assertTrue("To enter bridges into Tor Browser" in str(page)) def test_render_GET_RTLlang(self): - """Test rendering a request for obfs3 bridges in Arabic.""" - request = DummyRequest(["bridges?transport=obfs3"]) + """Test rendering a request for plain bridges in Arabic.""" + request = DummyRequest([b"bridges?transport=obfs3"]) + request.method = b'GET' + request.getClientIP = lambda: '3.3.3.3' + # For some strange reason, the 'Accept-Language' value *should not* be + # a list, unlike all the other headers and args… + request.headers.update({'accept-language': 'ar,en,en_US,'}) + + page = self.bridgesResource.render(request) + self.assertSubstring("direction: rtl", page) + self.assertSubstring( + "إذا لم يعمل تور بنجاح معك، يجب عليك ارسال بريد إلكتروني إلي", page) + + for bridgeLine in self.parseBridgesFromHTMLPage(page): + # Check that each bridge line had the expected number of fields: + bridgeLine = bridgeLine.split(' ') + self.assertEqual(len(bridgeLine), 2) + + def test_render_GET_RTLlang_obfs3(self): + """Test rendering a request for obfs3 bridges in Farsi.""" + request = DummyRequest([b"bridges?transport=obfs3"]) request.method = b'GET' request.getClientIP = lambda: '3.3.3.3' - request.headers.update({'accept-language': 'ar'}) + request.headers.update({'accept-language': 'fa,en,en_US,'}) # We actually have to set the request args manually when using a # DummyRequest: - request.args.update({'transport': 'obfs3'}) + request.args.update({'transport': ['obfs3']}) page = self.bridgesResource.render(request) self.assertSubstring("direction: rtl", page) self.assertSubstring( - "إذا لم يعمل تور بنجاح معك، يجب عليك ارسال بريد إلكتروني إلي", page) + # "I need an alternative way to get bridges!" + "به یک راه دیگر برای دریافت پل‌ها احتیاج دارم!", + page) for bridgeLine in self.parseBridgesFromHTMLPage(page): # Check that each bridge line had the expected number of fields: bridgeLine = bridgeLine.split(' ') self.assertEqual(len(bridgeLine), 3) - - print(""" - FIXME: The first field should be the transport method: - DummyBridge.getConfigLine() receives transport='o', not 'obfs3'. - What the hell? - """) - #self.assertEqual(bridgeLine[0], 'obfs3') + self.assertEqual(bridgeLine[0], 'obfs3') # Check that the IP and port seem okay: ip, port = bridgeLine[1].rsplit(':') @@ -616,24 +633,28 @@ class WebResourceBridgesTests(unittest.TestCase): self.assertGreater(int(port), 0) self.assertLessEqual(int(port), 65535) + def test_renderAnswer_textplain(self): """If the request format specifies 'plain', we should return content with mimetype 'text/plain'. """ request = DummyRequest([self.pagename]) - request.args.update({'format': 'plain'}) + request.args.update({'format': ['plain']}) request.getClientIP = lambda: '4.4.4.4' request.method = b'GET' page = self.bridgesResource.render(request) + self.assertTrue("html" not in str(page)) - print(""" - FIXME: The result page should have a mimetype of 'text/plain', and yet - it's still HTML. - """) - #self.assertNotSubstring("html", page) + # We just need to strip and split it because it looks like: + # + # 94.235.85.233:9492 0d9d0547c3471cddc473f7288a6abfb54562dc06 + # 255.225.204.145:9511 1fb89d618b3a12afe3529fd072127ea08fb50466 + # + # (Yes, there are two leading spaces at the beginning of each line) + # + bridgeLines = [line.strip() for line in page.strip().split('\n')] - bridgeLines = self.parseBridgesFromHTMLPage(page) for bridgeLine in bridgeLines: bridgeLine = bridgeLine.split(' ') self.assertEqual(len(bridgeLine), 2) @@ -665,7 +686,7 @@ class WebResourceOptionsTests(unittest.TestCase): request.headers.update({'accept-language': 'he'}) # We actually have to set the request args manually when using a # DummyRequest: - request.args.update({'transport': 'obfs2'}) + request.args.update({'transport': ['obfs2']}) page = self.optionsResource.render(request) self.assertSubstring("direction: rtl", page)

1 0

[bridgedb/master] PEP8; change geoipdb → GEOIP_DBFILE.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 1115b5ab2b41780761c9f4fa52bb0dbefd41db45 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 19:40:47 2014 +0000 PEP8; change geoipdb → GEOIP_DBFILE. --- lib/bridgedb/HTTPServer.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 1d4cdf9..9176f6a 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -45,6 +45,7 @@ from bridgedb.safelog import logSafely TEMPLATE_DIR = os.path.join(os.path.dirname(__file__),'templates') +GEOIP_DBFILE = '/usr/share/GeoIP/GeoIP.dat' rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') # Setting `filesystem_checks` to False is recommended for production servers, @@ -58,22 +59,19 @@ lookup = TemplateLookup(directories=[TEMPLATE_DIR], output_encoding='utf-8', filesystem_checks=False, collection_size=500) - - -_geoipdb = '/usr/share/GeoIP/GeoIP.dat' logging.debug("Set template root to %s" % TEMPLATE_DIR) try: # Make sure we have the database before trying to import the module: - if not os.path.isfile(_geoipdb): # pragma: no cover + if not os.path.isfile(GEOIP_DBFILE): # pragma: no cover raise EnvironmentError("Could not find %r. On Debian-based systems, "\ "please install the geoip-database package." - % _geoipdb) + % GEOIP_DBFILE) # This is a "pure" python version which interacts with the Maxmind GeoIP # API (version 1). It requires, in Debian, the libgeoip-dev and # geoip-database packages. import pygeoip - geoip = pygeoip.GeoIP(_geoipdb, flags=pygeoip.MEMORY_CACHE) + geoip = pygeoip.GeoIP(GEOIP_DBFILE, flags=pygeoip.MEMORY_CACHE) logging.info("GeoIP database loaded") except Exception as err: # pragma: no cover logging.warn("Error while loading geoip module: %r" % err)

1 0

[bridgedb/master] Update docstring for I18n._().
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 09a49cd22939986a4b038c7e48f75c7a44853312 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 19:44:47 2014 +0000 Update docstring for I18n._(). --- lib/bridgedb/I18n.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/I18n.py b/lib/bridgedb/I18n.py index 112b8ae..c6fbf3f 100644 --- a/lib/bridgedb/I18n.py +++ b/lib/bridgedb/I18n.py @@ -12,8 +12,10 @@ def getLang(lang, localedir=os.path.expanduser("~") + "/share/locale"): def _(text): """This is necessary because strings are translated when they're imported. - Otherwise this would make it impossible to switch languages more than - once + Otherwise this would make it impossible to switch languages more than + once. + + :returns: The **text**. """ return text

1 0

[bridgedb/master] Split unittests for EmailServer.getMailResponse() into smaller tests.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 8274af32008893b461f81e96c2a037fe5a96b996 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 8 18:34:12 2014 +0000 Split unittests for EmailServer.getMailResponse() into smaller tests. --- lib/bridgedb/test/test_EmailServer.py | 58 +++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/lib/bridgedb/test/test_EmailServer.py b/lib/bridgedb/test/test_EmailServer.py index e8b9360..13f018c 100644 --- a/lib/bridgedb/test/test_EmailServer.py +++ b/lib/bridgedb/test/test_EmailServer.py @@ -134,7 +134,7 @@ class EmailCompositionTests(unittest.TestCase): """Tests for :func:`bridgedb.EmailServer.getMailResponse`.""" def setUp(self): - """Create fake email and associated data""" + """Create fake email, distributor, and associated context data.""" configuration = {} TEST_CONFIG_FILE.seek(0) compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') @@ -142,40 +142,45 @@ class EmailCompositionTests(unittest.TestCase): self.config = Conf(**configuration) # TODO: Add headers if we start validating them - self.lines = ["From: %s(a)%s.com", "To: %s(a)example.net", - "Subject: testing", "\n", "get bridges"] + self.lines = ["From: %s(a)%s.com", + "To: bridges(a)example.net", + "Subject: testing", + "\n", + "get bridges"] self.distributor = FakeDistributor('key', {}, {}, []) self.ctx = MailContext(self.config, self.distributor, NoSchedule()) - def test_getMailResponseNoFrom(self): + def test_getMailResponse_noFrom(self): + """A received email without a "From:" or "Sender:" header shouldn't + receive a response. + """ lines = self.lines lines[0] = "" - lines[1] = self.lines[1] % "bridges" ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple) self.assertEqual(len(ret), 2) self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) - def test_getMailResponseBadAddress(self): + def test_getMailResponse_badAddress(self): lines = copy.copy(self.lines) - lines[0] = self.lines[0] % ("testing?", "example") - lines[1] = self.lines[1] % "bridges" - lines[2] = "" + lines[0] = self.lines[0] % ("testing*.?\"", "example") ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple) self.assertEqual(len(ret), 2) self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) - #lines[0] = self.lines[0] % ("<>>", "example") - lines[0] = "From: %s(a)%s.com" % ("<>>", "example") + + def test_getMailResponse_anotherBadAddress(self): + lines = copy.copy(self.lines) + lines[0] = "From: Mallory %s(a)%s.com" % ("<>>", "example") ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple) self.assertEqual(len(ret), 2) self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) - def test_getMailResponseInvalidDomain(self): + def test_getMailResponse_invalidDomain(self): lines = copy.copy(self.lines) lines[0] = self.lines[0] % ("testing", "exa#mple") ret = EmailServer.getMailResponse(lines, self.ctx) @@ -183,6 +188,9 @@ class EmailCompositionTests(unittest.TestCase): self.assertEqual(len(ret), 2) self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) + + def test_getMailResponse_anotherInvalidDomain(self): + lines = copy.copy(self.lines) lines[0] = self.lines[0] % ("testing", "exam+ple") ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple) @@ -190,7 +198,10 @@ class EmailCompositionTests(unittest.TestCase): self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) - def test_getMailResponseDKIM(self): + def test_getMailResponse_DKIM_badDKIMheader(self): + """An email with an appended 'X-DKIM-Authentication-Result:' header should not + receive a response. + """ lines = copy.copy(self.lines) lines[0] = self.lines[0] % ("testing", "gmail") lines.append("X-DKIM-Authentication-Result: ") @@ -199,7 +210,11 @@ class EmailCompositionTests(unittest.TestCase): self.assertEqual(len(ret), 2) self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) + + def test_getMailResponse_DKIM(self): + lines = copy.copy(self.lines) lines[0] = self.lines[0] % ("testing", "example") + lines.append("X-DKIM-Authentication-Result: ") ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple) self.assertEqual(len(ret), 2) @@ -208,7 +223,8 @@ class EmailCompositionTests(unittest.TestCase): mail = ret[1].getvalue() self.assertNotEqual(mail.find("no bridges currently"), -1) - def test_getMailResponseMailContent(self): + def test_getMailResponse_bridges_obfs(self): + """A request for 'transport obfs' should receive a response.""" lines = copy.copy(self.lines) lines[0] = self.lines[0] % ("testing", "example") lines.append("transport obfs") @@ -219,6 +235,12 @@ class EmailCompositionTests(unittest.TestCase): self.assertIsInstance(ret[1], StringIO) mail = ret[1].getvalue() self.assertNotEqual(mail.find("no bridges currently"), -1) + + def test_getMailResponse_bridges_obfsobfswebz(self): + """We should only pay attention to the first in a crazy request.""" + lines = copy.copy(self.lines) + lines[0] = self.lines[0] % ("testing", "example") + lines.append("transport obfs") lines.append("transport obfs") lines.append("unblocked webz") ret = EmailServer.getMailResponse(lines, self.ctx) @@ -228,6 +250,14 @@ class EmailCompositionTests(unittest.TestCase): self.assertIsInstance(ret[1], StringIO) mail = ret[1].getvalue() self.assertNotEqual(mail.find("no bridges currently"), -1) + + def test_getMailResponse_bridges_obfsobfswebzipv6(self): + """We should *still* only pay attention to the first request.""" + lines = copy.copy(self.lines) + lines[0] = self.lines[0] % ("testing", "example") + lines.append("transport obfs") + lines.append("transport obfs") + lines.append("unblocked webz") lines.append("ipv6") ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple)

1 0

[bridgedb/master] Add unittest for EmailServer.replyToMail().
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit f9e2175750b998366492fbb197594b8eb226fbaa Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 8 21:28:34 2014 +0000 Add unittest for EmailServer.replyToMail(). --- lib/bridgedb/test/test_EmailServer.py | 42 ++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_EmailServer.py b/lib/bridgedb/test/test_EmailServer.py index 13f018c..f828be7 100644 --- a/lib/bridgedb/test/test_EmailServer.py +++ b/lib/bridgedb/test/test_EmailServer.py @@ -26,7 +26,9 @@ from bridgedb.EmailServer import MailContext from bridgedb.Time import NoSchedule from bridgedb.persistent import Conf from bridgedb.test.util import fileCheckDecorator + from twisted.python import log +from twisted.internet import defer from twisted.trial import unittest @@ -130,7 +132,7 @@ class EmailGnuPGTest(unittest.TestCase): ctx = EmailServer.getGPGContext(self.config) self.assertTrue(ctx is None) -class EmailCompositionTests(unittest.TestCase): +class EmailResponseTests(unittest.TestCase): """Tests for :func:`bridgedb.EmailServer.getMailResponse`.""" def setUp(self): @@ -267,6 +269,44 @@ class EmailCompositionTests(unittest.TestCase): mail = ret[1].getvalue() self.assertNotEqual(mail.find("no bridges currently"), -1) + +class EmailReplyTests(unittest.TestCase): + """Tests for ``EmailServer.replyToMail()``.""" + + def setUp(self): + """Create fake email, distributor, and associated context data.""" + configuration = {} + TEST_CONFIG_FILE.seek(0) + compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') + exec compiled in configuration + self.config = Conf(**configuration) + + # TODO: Add headers if we start validating them + self.lines = ["From: %s(a)%s.com", + "To: bridges(a)example.net", + "Subject: testing", + "\n", + "get bridges"] + self.distributor = FakeDistributor('key', {}, {}, []) + self.ctx = MailContext(self.config, self.distributor, NoSchedule()) + + def test_replyToMail(self): + self.skip = True + raise unittest.SkipTest("We'll have to fake the EmailServer for this one,"\ + " it requires a TCP connection to localhost.") + + def callback(reply): + self.assertSubstring("Here are your bridges", reply) + + lines = copy.copy(self.lines) + lines[0] = self.lines[0] % ("testing", "example") + reply = EmailServer.replyToMail(lines, self.ctx) + + self.assertIsInstance(reply, defer.Deferred) + + reply.addCallback(callback) + return reply + class EmailServerServiceTests(unittest.TestCase): def setUp(self): configuration = {}

1 0

[bridgedb/master] Fix unicode encoding errors in outgoing SMTP transport.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit fcaf0a5a4401d9f85bd16b550edfefaa2ef50559 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 8 23:30:27 2014 +0000 Fix unicode encoding errors in outgoing SMTP transport. * REFACTOR bridgedb.EmailServer.composeEmail() to use io.BytesIO. --- lib/bridgedb/EmailServer.py | 83 +++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/lib/bridgedb/EmailServer.py b/lib/bridgedb/EmailServer.py index e1aa57e..db2cf80 100644 --- a/lib/bridgedb/EmailServer.py +++ b/lib/bridgedb/EmailServer.py @@ -8,9 +8,9 @@ from __future__ import unicode_literals from email import message -from io import StringIO import gettext import gpgme +import io import logging import re import rfc822 @@ -315,6 +315,47 @@ def getLocaleFromRequest(request): lang = lang[0] return I18n.getLang(lang) +def composeEmail(fromAddr, clientAddr, subject, body, + msgID=None, gpgContext=None): + + if not subject.startswith("Re:"): + subject = "Re: %s" % subject + + msg = smtp.rfc822.Message(io.StringIO()) + msg.setdefault("From", fromAddr) + msg.setdefault("To", clientAddr) + msg.setdefault("Message-ID", smtp.messageid()) + msg.setdefault("Subject", subject) + if msgID: + msg.setdefault("In-Reply-To", msgID) + msg.setdefault("Date", smtp.rfc822date()) + msg.setdefault('Content-Type', 'text/plain; charset="utf-8"') + headers = [': '.join(m) for m in msg.items()] + + mail = io.BytesIO() + mail.writelines(buffer("\r\n".join(headers))) + mail.writelines(buffer("\r\n")) + mail.writelines(buffer("\r\n")) + + if not gpgContext: + mail.write(buffer(body)) + else: + signature, siglist = gpgSignMessage(gpgContext, body) + if signature: + mail.writelines(buffer(signature)) + mail.seek(0) + + # Only log the email text (including all headers) if SAFE_LOGGING is + # disabled: + if not safelog.safe_logging: + logging.debug("Email contents:\n\n%s" % mail.read()) + mail.seek(0) + else: + logging.debug("Email text for %r created." % clientAddr) + + return clientAddr, mail + + class MailContext(object): """Helper object that holds information used by email subsystem.""" @@ -439,45 +480,9 @@ def addSMTPServer(cfg, dist, sched): lc.start(1800, now=False) return factory -def composeEmail(fromAddr, clientAddr, subject, body, msgID=False, - gpgContext=None): - msg = message.Message() - msg.add_header("From", fromAddr) - msg.add_header("To", clientAddr) - msg.add_header("Message-ID", smtp.messageid()) - if not subject.startswith("Re:"): subject = "Re: %s"%subject - msg.add_header("Subject", subject) - if msgID: - msg.add_header("In-Reply-To", msgID) - msg.add_header("Date", smtp.rfc822date()) - msg.set_default_type("text/plain") - headers = [': '.join(m) for m in msg.items()] - mail = StringIO("\r\n".join(headers)) - mail.writelines(unicode(msg.as_string())) - - # gpg-clearsign messages - if gpgContext: - signature = StringIO() - plaintext = StringIO(body) - sigs = gpgContext.sign(plaintext, signature, gpgme.SIG_MODE_CLEAR) - if (len(sigs) != 1): - logging.warn('Failed to sign message!') - signature.seek(0) - [mail.write(l) for l in signature] - else: - mail.write(body) - # Only log the email text (including all headers) if SAFE_LOGGING is - # disabled: - if not safelog.safe_logging: - mail.seek(0) - logging.debug("Email contents:\n%s" % mail.read()) - else: - logging.debug("Email text for %r created." % clientAddr) - mail.seek(0) - return clientAddr, mail def getGPGContext(cfg): """Import a key from a file and initialise a context for GnuPG operations. @@ -520,8 +525,8 @@ def getGPGContext(cfg): ctx.signers = [ctx.get_key(fingerprint)] logging.info("Testing signature created with GnuPG key...") - message = StringIO('Test') - new_sigs = ctx.sign(message, StringIO(), gpgme.SIG_MODE_CLEAR) + message = io.StringIO('Test') + new_sigs = ctx.sign(message, io.StringIO(), gpgme.SIG_MODE_CLEAR) if not len(new_sigs) == 1: raise gpgme.GpgmeError( "Testing was unable to produce a signature with GnuPG key.")

1 0

[bridgedb/master] Move email address parsers from bridgedb.Dist → bridgedb.parse.addr.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit dfe81deffb272e4585af141ad58ece7d68e352f8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 8 21:37:11 2014 +0000 Move email address parsers from bridgedb.Dist → bridgedb.parse.addr. --- lib/bridgedb/Dist.py | 117 +++------------------- lib/bridgedb/EmailServer.py | 41 ++++---- lib/bridgedb/Tests.py | 9 +- lib/bridgedb/parse/addr.py | 171 +++++++++++++++++++++++++++++++-- lib/bridgedb/test/test_EmailServer.py | 2 +- 5 files changed, 208 insertions(+), 132 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 5b5a602..4e0a1aa 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -28,9 +28,20 @@ from bridgedb.Filters import filterAssignBridgesToRing from bridgedb.Filters import filterBridgesByRules from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.parse import addr +from bridgedb.parse.addr import UnsupportedDomain from bridgedb.safelog import logSafely +MAX_EMAIL_RATE = 3*3600 + +class IgnoreEmail(addr.BadEmail): + """Raised when we get requests from this address after rate warning.""" + +class TooSoonEmail(addr.BadEmail): + """Raised when we got a request from this address too recently.""" + + def uniformMap(ip): """Map an IP to an arbitrary 'area' string, such that any two /24 addresses get the same string. @@ -322,103 +333,6 @@ class IPBasedDistributor(Distributor): def dumpAssignments(self, f, description=""): self.splitter.dumpAssignments(f, description) - -# These characters are the ones that RFC2822 allows. -#ASPECIAL = '!#$%&*+-/=?^_`{|}~' -#ASPECIAL += "\\\'" -# These are the ones we're pretty sure we can handle right. -ASPECIAL = '-_+/=_~' - -ACHAR = r'[\w%s]' % "".join("\\%s"%c for c in ASPECIAL) -DOTATOM = r'%s+(?:\.%s+)*' % (ACHAR,ACHAR) -DOMAIN = r'\w+(?:\.\w+)*' -ADDRSPEC = r'(%s)\@(%s)' % (DOTATOM, DOMAIN) - -SPACE_PAT = re.compile(r'\s+') -ADDRSPEC_PAT = re.compile(ADDRSPEC) - -MAX_EMAIL_RATE = 3*3600 - -class BadEmail(Exception): - """Exception raised when we get a bad email address.""" - def __init__(self, msg, email): - Exception.__init__(self, msg) - self.email = email - -class UnsupportedDomain(BadEmail): - """Exception raised when we get an email address from a domain we - don't know.""" - -class TooSoonEmail(BadEmail): - """Raised when we got a request from this address too recently.""" - -class IgnoreEmail(BadEmail): - """Raised when we get requests from this address after rate warning.""" - -def extractAddrSpec(addr): - """Given an email From line, try to extract and parse the addrspec - portion. Returns localpart,domain on success; raises BadEmail - on failure. - """ - orig_addr = addr - addr = SPACE_PAT.sub(' ', addr) - addr = addr.strip() - # Only works on usual-form addresses; raises BadEmail on weird - # address form. That's okay, since we'll only get those when - # people are trying to fool us. - if '<' in addr: - # Take the _last_ index of <, so that we don't need to bother - # with quoting tricks. - idx = addr.rindex('<') - addr = addr[idx:] - m = re.search(r'<([^>]*)>', addr) - if m is None: - raise BadEmail("Couldn't extract address spec", orig_addr) - addr = m.group(1) - - # At this point, addr holds a putative addr-spec. We only allow the - # following form: - # addr-spec = local-part "@" domain - # local-part = dot-atom - # domain = dot-atom - # - # In particular, we are disallowing: obs-local-part, obs-domain, - # comment, obs-FWS, - # - # Other forms exist, but none of the incoming services we recognize - # support them. - addr = addr.replace(" ", "") - m = ADDRSPEC_PAT.match(addr) - if not m: - raise BadEmail("Bad address spec format", orig_addr) - localpart, domain = m.groups() - return localpart, domain - -def normalizeEmail(addr, domainmap, domainrules): - """Given the contents of a from line, and a map of supported email - domains (in lowercase), raise BadEmail or return a normalized - email address. - """ - addr = addr.lower() - localpart, domain = extractAddrSpec(addr) - if domainmap is not None: - domain = domainmap.get(domain, None) - if domain is None: - raise UnsupportedDomain("Domain not supported", addr) - - #XXXX Do these rules also hold for Yahoo? - - # addr+foo@ is an alias for addr@ - idx = localpart.find('+') - if idx >= 0: - localpart = localpart[:idx] - rules = domainrules.get(domain, []) - if 'ignore_dots' in rules: - # j.doe@ is the same as jdoe@. - localpart = localpart.replace(".", "") - - return "%s@%s"%(localpart, domain) - class EmailBasedDistributor(Distributor): """Object that hands out bridges based on the email address of an incoming request and the current time period. @@ -475,12 +389,13 @@ class EmailBasedDistributor(Distributor): bridgeFilterRules=[] now = time.time() try: - emailaddress = normalizeEmail(emailaddress, self.domainmap, - self.domainrules) - except BadEmail as err: + emailaddress = addr.normalizeEmail(emailaddress, self.domainmap, + self.domainrules) + except addr.BadEmail as err: logging.warn(err) return [] - if emailaddress is None: + + if not emailaddress: return [] #XXXX raise an exception. with bridgedb.Storage.getDB() as db: diff --git a/lib/bridgedb/EmailServer.py b/lib/bridgedb/EmailServer.py index f9f43bb..e1aa57e 100644 --- a/lib/bridgedb/EmailServer.py +++ b/lib/bridgedb/EmailServer.py @@ -26,7 +26,6 @@ from twisted.mail import smtp from zope.interface import implements -from bridgedb.Dist import BadEmail, TooSoonEmail, IgnoreEmail from bridgedb import Dist from bridgedb import I18n from bridgedb import safelog @@ -34,6 +33,10 @@ from bridgedb.Filters import filterBridgesByIP6 from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByTransport from bridgedb.Filters import filterBridgesByNotBlockedIn +from bridgedb.parse import addr +from bridgedb.parse.addr import BadEmail +from bridgedb.parse.addr import UnsupportedDomain +from bridgedb.parse.addr import canonicalizeEmailDomain class MailFile: @@ -97,23 +100,24 @@ def getMailResponse(lines, ctx): lang = getLocaleFromPlusAddr(clientToaddr) t = I18n.getLang(lang) + canon = ctx.cfg.EMAIL_DOMAIN_MAP + for domain, rule in ctx.cfg.EMAIL_DOMAIN_RULES.items(): + if domain not in canon.keys(): + canon[domain] = domain + for domain in ctx.cfg.EMAIL_DOMAINS: + canon[domain] = domain + try: - _, addrdomain = Dist.extractAddrSpec(clientAddr.lower()) - except BadEmail: - logging.info("Ignoring bad address on incoming email.") + _, clientDomain = addr.extractEmailAddress(clientAddr.lower()) + canonical = canonicalizeEmailDomain(clientDomain, canon) + except UnsupportedDomain as error: + logging.warn(error) return None, None - - if not addrdomain: - logging.info("Couldn't parse domain from %r" % clientAddr) - - if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP: - addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain) - - if addrdomain not in ctx.cfg.EMAIL_DOMAINS: - logging.warn("Unrecognized email domain %r", addrdomain) + except BadEmail as error: + logging.warn(error) return None, None - rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, []) + rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(canonical, []) if 'dkim' in rules: # getheader() returns the last of a given kind of header; we want @@ -123,8 +127,8 @@ def getMailResponse(lines, ctx): if dkimHeaders: dkimHeader = dkimHeaders[0] if not dkimHeader.startswith("pass"): - logging.info("Got a bad dkim header (%r) on an incoming mail; " - "rejecting it.", dkimHeader) + logging.info("Rejecting bad DKIM header on incoming email: %r " + % dkimHeader) return None, None # Was the magic string included @@ -186,17 +190,16 @@ def getMailResponse(lines, ctx): bridgeFilterRules=bridgeFilterRules) # Handle rate limited email - except TooSoonEmail as err: + except Dist.TooSoonEmail as err: logging.info("Got a mail too frequently; warning '%s': %s." % (clientAddr, err)) - # Compose a warning email # MAX_EMAIL_RATE is in seconds, convert to hours body = buildSpamWarningTemplate(t) % (Dist.MAX_EMAIL_RATE / 3600) return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID, gpgContext=ctx.gpgContext) - except IgnoreEmail as err: + except Dist.IgnoreEmail as err: logging.info("Got a mail too frequently; ignoring '%s': %s." % (clientAddr, err)) return None, None diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 72dfe5e..4147549 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -232,10 +232,11 @@ class EmailBridgeDistTests(unittest.TestCase): def testUnsupportedDomain(self): db = self.db - self.assertRaises(bridgedb.Dist.UnsupportedDomain, - bridgedb.Dist.normalizeEmail, 'bad(a)email.com', - {'example.com':'example.com'}, - {'example.com':[]}) + self.assertRaises(bridgedb.parse.addr.UnsupportedDomain, + bridgedb.parse.addr.normalizeEmail, + 'bad(a)email.com', + {'example.com':'example.com'}, + {'example.com':[]}) class IPBridgeDistTests(unittest.TestCase): def dumbAreaMapper(self, ip): diff --git a/lib/bridgedb/parse/addr.py b/lib/bridgedb/parse/addr.py index 455b953..f34f416 100644 --- a/lib/bridgedb/parse/addr.py +++ b/lib/bridgedb/parse/addr.py @@ -13,19 +13,22 @@ ** Module Overview: ** -.. +:: parse ||_ parse.addr - | |_ isIPAddress - Check if an arbitrary string is an IP address. - | |_ isIPv4 - Check if an arbitrary string is an IPv4 address. - | |_ isIPv6 - Check if an arbitrary string is an IPv6 address. - | \_ isValidIP - Check that an IP address is valid. + | | |_ extractEmailAddress - Validate a :rfc:2822 email address. + | | |_ isIPAddress - Check if an arbitrary string is an IP address. + | | |_ isIPv4 - Check if an arbitrary string is an IPv4 address. + | | |_ isIPv6 - Check if an arbitrary string is an IPv6 address. + | | \_ isValidIP - Check that an IP address is valid. + | | + | |_ :class:`PortList` - A container class for validated port ranges. | - |__ :mod:`bridgedbparse.headers` + |__ :mod:`bridgedb.parse.headers` |__ :mod:`bridgedb.parse.options` \__ :mod:`bridgedb.parse.versions` -.. +:: Private IP Address Ranges: '''''''''''''''''''''''''' @@ -147,12 +150,119 @@ from __future__ import print_function from __future__ import unicode_literals import logging +import re + import ipaddr +#: These are the special characters which RFC2822 allows within email addresses: +#ASPECIAL = '!#$%&*+-/=?^_`{|}~' + "\\\'" +#: These are the ones we're pretty sure we can handle right: +ASPECIAL = '-_+/=_~' +ACHAR = r'[\w%s]' % "".join("\\%s" % c for c in ASPECIAL) +DOTATOM = r'%s+(?:\.%s+)*' % (ACHAR, ACHAR) +DOMAIN = r'\w+(?:\.\w+)*' +ADDRSPEC = r'(%s)\@(%s)' % (DOTATOM, DOMAIN) +SPACE_PAT = re.compile(r'\s+') +#: A compiled regex with matches RFC2822 email address strings: +ADDRSPEC_PAT = re.compile(ADDRSPEC) + + +class BadEmail(Exception): + """Exception raised when we get a bad email address.""" + def __init__(self, msg, email): + Exception.__init__(self, msg) + self.email = email + class InvalidPort(ValueError): """Raised when a given port number is invalid.""" +class UnsupportedDomain(ValueError): + """Raised when we get an email address from an unsupported domain.""" + + +def canonicalizeEmailDomain(domain, domainmap): + """Decide if an email was sent from a permitted domain. + + :param str domain: The domain portion of an email address to validate. It + will be checked that it is one of the domains allowed to email + requests for bridges to the + :class:`~bridgedb.Dist.EmailBasedDistributor`. + :param dict domainmap: A map of permitted alternate domains (in lowercase) + to their canonical domain names (in lowercase). This can be configured + with the ``EMAIL_DOMAIN_MAP`` option in ``bridgedb.conf``, for + example:: + EMAIL_DOMAIN_MAP = {'mail.google.com': 'gmail.com', + 'googlemail.com': 'gmail.com'} + :raises UnsupportedDomain: if the domain portion of the email address is + not within the map of alternate to canonical allowed domain names. + :rtype: str + :returns: The canonical domain name for the email address. + """ + permitted = None + + try: + permitted = domainmap.get(domain) + except AttributeError: + logging.debug("Got non-dict for 'domainmap' parameter: %r" % domainmap) + + if not permitted: + raise UnsupportedDomain("Domain not permitted: %s" % domain) + + return permitted + +def extractEmailAddress(emailaddr): + """Given an email address, obtained, for example, via a ``From:`` or + ``Sender:`` email header, try to extract and parse (according to + :rfc:2822) the username and domain portions. Returns ``(username, + domain)`` on success; raises BadEmail on failure. + + We only allow the following form:: + ADDRSPEC := LOCAL_PART "@" DOMAIN + LOCAL_PART := DOTATOM + DOMAIN := DOTATOM + + In particular, we are disallowing: obs-local-part, obs-domain, comment, + and obs-FWS. Other forms exist, but none of the incoming services we + recognize support them. + + :param emailaddr: An email address to validate. + :raises BadEmail: if the **emailaddr** couldn't be validated or parsed. + :rtype: tuple + :returns: A tuple of the validated email address, containing the mail + username and the domain:: + (LOCALPART, DOMAIN) + """ + orig = emailaddr + + try: + addr = SPACE_PAT.sub(' ', emailaddr).strip() + except TypeError as error: + logging.debug(error) + raise BadEmail("Can't extract address from object type %r!" + % type(orig), orig) + + # Only works on usual-form addresses; raises BadEmail on weird + # address form. That's okay, since we'll only get those when + # people are trying to fool us. + if '<' in addr: + # Take the _last_ index of <, so that we don't need to bother + # with quoting tricks. + idx = addr.rindex('<') + addr = addr[idx:] + m = re.search(r'<([^>]*)>', addr) + if m is None: + raise BadEmail("Couldn't extract address spec", orig) + addr = m.group(1) + + # At this point, addr holds a putative addr-spec. + addr = addr.replace(" ", "") + m = ADDRSPEC_PAT.match(addr) + if not m: + raise BadEmail("Bad address spec format", orig) + + localpart, domain = m.groups() + return localpart, domain def isIPAddress(ip, compressed=True): """Check if an arbitrary string is an IP address, and that it's valid. @@ -275,6 +385,53 @@ def isValidIP(ip): return False return True +def normalizeEmail(emailaddr, domainmap, domainrules, ignorePlus=True): + """Normalise an email address according to the processing rules for its + canonical originating domain. + + The email address, **emailaddr**, will be parsed and validated, and then + checked that it originated from one of the domains allowed to email + requests for bridges to the :class:`~bridgedb.Dist.EmailBasedDistributor` + via the :func:`canonicaliseEmailDomain` function. + + :param str emailaddr: An email address to normalise. + :param dict domainmap: A map of permitted alternate domains (in lowercase) + to their canonical domain names (in lowercase). This can be configured + with the ``EMAIL_DOMAIN_MAP`` option in ``bridgedb.conf``, for + example:: + EMAIL_DOMAIN_MAP = {'mail.google.com': 'gmail.com', + 'googlemail.com': 'gmail.com'} + :param dict domainrules: A mapping of canonical permitted domain names to + a list of rules which should be applied to processing them, for + example:: + EMAIL_DOMAIN_RULES = {'gmail.com': ["ignore_dots", "dkim"] + Currently, ``"ignore_dots"`` means that all ``"."`` characters will be + removed from the local part of the validated email address. + :param bool ignorePlus: If ``True``, assume that + ``blackhole+kerr(a)torproject.org`` is an alias for + ``blackhole(a)torproject.org``, and remove everything after the first + ``'+'`` character. + :raises BadEmail: if the email address could not be parsed or validated. + :rtype: str + :returns: The validated, normalised email address, if it was from a + permitted domain. Otherwise, returns an empty string. + """ + emailaddr = emailaddr.lower() + localpart, domain = extractEmailAddress(emailaddr) + canonical = canonicalizeEmailDomain(domain, domainmap) + + if ignorePlus: + idx = localpart.find('+') + if idx >= 0: + localpart = localpart[:idx] + + rules = domainrules.get(canonical, []) + if 'ignore_dots' in rules: + localpart = localpart.replace(".", "") + + normalized = "%s@%s" % (localpart, domain) + return normalized + class PortList(object): """A container class for validated port ranges. diff --git a/lib/bridgedb/test/test_EmailServer.py b/lib/bridgedb/test/test_EmailServer.py index f828be7..8521762 100644 --- a/lib/bridgedb/test/test_EmailServer.py +++ b/lib/bridgedb/test/test_EmailServer.py @@ -20,10 +20,10 @@ from io import StringIO import copy from bridgedb import EmailServer -from bridgedb.Dist import BadEmail from bridgedb.Dist import EmailBasedDistributor from bridgedb.EmailServer import MailContext from bridgedb.Time import NoSchedule +from bridgedb.parse.addr import BadEmail from bridgedb.persistent import Conf from bridgedb.test.util import fileCheckDecorator

1 0