April 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Merge branch 'fix/10803-remove-vidalia-update-options' into develop
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 42e83f5843bf02fe971a7ca74f85eb49f5d50214 Merge: ebf301d 6c5c358 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 14 19:36:40 2014 +0000 Merge branch 'fix/10803-remove-vidalia-update-options' into develop lib/bridgedb/HTTPServer.py | 26 ++ lib/bridgedb/templates/assets/bootstrap.min.css | 9 - .../templates/assets/css/bootstrap.min.css | 7 + lib/bridgedb/templates/assets/css/custom.css | 104 +++++ .../templates/assets/css/font-awesome-ie7.min.css | 384 +++++++++++++++++++ .../templates/assets/css/font-awesome.min.css | 403 ++++++++++++++++++++ lib/bridgedb/templates/assets/css/main.css | 24 ++ lib/bridgedb/templates/assets/custom.css | 104 ----- .../templates/assets/font/fontawesome-webfont.eot | Bin 0 -> 37405 bytes .../templates/assets/font/fontawesome-webfont.svg | 399 +++++++++++++++++++ .../templates/assets/font/fontawesome-webfont.ttf | Bin 0 -> 79076 bytes .../templates/assets/font/fontawesome-webfont.woff | Bin 0 -> 43572 bytes lib/bridgedb/templates/assets/font/lato-bold.woff | Bin 0 -> 46160 bytes .../templates/assets/font/lato-italic.woff | Bin 0 -> 47168 bytes .../templates/assets/font/lato-regular.woff | Bin 0 -> 46108 bytes lib/bridgedb/templates/assets/tor-roots-blue.svg | 95 +++++ lib/bridgedb/templates/base.html | 69 ++-- lib/bridgedb/templates/bridges.html | 92 ++++- lib/bridgedb/templates/captcha.html | 79 +++- lib/bridgedb/templates/howto.html | 47 +++ lib/bridgedb/templates/index.html | 33 +- lib/bridgedb/templates/options.html | 198 ++++++++-- lib/bridgedb/test/test_HTTPServer.py | 23 +- setup.py | 8 +- 24 files changed, 1855 insertions(+), 249 deletions(-)

1 0

[bridgedb/master] Move locate determination and translation code into separate module.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit ed438d0c2b00f02c29b60e561d37f241f6a6a4b8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 16:22:40 2014 +0000 Move locate determination and translation code into separate module. * ADD new bridgedb.translations module with the following methods: - getFirstSupportedLang(): Essentially the same as the original HTTPServer.getAssumedChosenLang() function, except that the determination of which languages are supported uses _lang.getLangs() instead (this has been in the _lang module for a long time, and was meant for that purpose). - getLocaleFromHTTPRequest(): This is taken mostly from the original setLocaleFromRequestHeader() (in the HTTPServer module) except that there was an extra function in the EmailServer module, called getLocaleFromRequest() ― which was not in use anywhere ― which included the parsing of '?lang=' HTTP GET/POST arguments. The parsing of '?lang=' was taken from that function and added to the original functionality of setLocaleFromRequestHeader(). This fixes part of ticket #9678: "'Select Language' button on bridges.tpo". - getLocaleFromPlusAddr(): Exactly the same as the original in the EmailServer module. - installTranslations(): Produces a gettext translation object, with fallback languages chained to it in order of client preference. - usingRTLLang(): Essentially the same as the original in the HTTPServer module, except that the parsing of supported langs again uses the _lang.getLangs() function, as above. * FIXES part of #9678: "'Select Language' button to bridges.tpo" --- lib/bridgedb/EmailServer.py | 29 ++-------- lib/bridgedb/HTTPServer.py | 94 +++----------------------------- lib/bridgedb/translations.py | 122 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 113 deletions(-) diff --git a/lib/bridgedb/EmailServer.py b/lib/bridgedb/EmailServer.py index f9f43bb..c58ec53 100644 --- a/lib/bridgedb/EmailServer.py +++ b/lib/bridgedb/EmailServer.py @@ -30,6 +30,7 @@ from bridgedb.Dist import BadEmail, TooSoonEmail, IgnoreEmail from bridgedb import Dist from bridgedb import I18n from bridgedb import safelog +from bridgedb import translations from bridgedb.Filters import filterBridgesByIP6 from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByTransport @@ -94,8 +95,8 @@ def getMailResponse(lines, ctx): # Look up the locale part in the 'To:' address, if there is one and get # the appropriate Translation object - lang = getLocaleFromPlusAddr(clientToaddr) - t = I18n.getLang(lang) + lang = translations.getLocaleFromPlusAddr(clientToaddr) + t = translations.installTranslations(lang) try: _, addrdomain = Dist.extractAddrSpec(clientAddr.lower()) @@ -287,30 +288,6 @@ def replyToMail(lines, ctx): reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory) return d -def getLocaleFromPlusAddr(address): - """See whether the user sent his email to a 'plus' address, for - instance to bridgedb+fa@tpo. Plus addresses are the current - mechanism to set the reply language - """ - replyLocale = "en" - r = '.*(<)?(\w+\+(\w+)@\w+(?:\.\w+)+)(?(1)>)' - match = re.match(r, address) - if match: - replyLocale = match.group(3) - - return replyLocale - -def getLocaleFromRequest(request): - # See if we did get a request for a certain locale, otherwise fall back - # to 'en': - # Try evaluating the path /foo first, then check if we got a ?lang=foo - default_lang = lang = "en" - if len(request.path) > 1: - lang = request.path[1:] - if lang == default_lang: - lang = request.args.get("lang", [default_lang]) - lang = lang[0] - return I18n.getLang(lang) class MailContext(object): """Helper object that holds information used by email subsystem.""" diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 5e3ba37..79772c3 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -35,6 +35,7 @@ import bridgedb.I18n as I18n from bridgedb import captcha from bridgedb import crypto +from bridgedb import translations from bridgedb import txrecaptcha from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByIP6 @@ -541,9 +542,10 @@ class WebResourceOptions(resource.Resource): def render_GET(self, request): rtl = False + langs = translations.getLocaleFromHTTPRequest(request) try: - rtl = usingRTLLang(request) + rtl = translations.usingRTLLang(langs) except Exception as err: # pragma: no cover logging.exception(err) @@ -664,7 +666,8 @@ class WebResourceBridges(resource.Resource): if countryCode: logging.debug("Client request from GeoIP CC: %s" % countryCode) - rtl = usingRTLLang(request) + langs = translations.getLocaleFromHTTPRequest(request) + rtl = translations.usingRTLLang(langs) if rtl: logging.debug("Rendering RTL response.") @@ -784,9 +787,10 @@ class WebRoot(resource.Resource): :param request: An incoming request. """ rtl = False + langs = translations.getLocaleFromHTTPRequest(request) try: - rtl = usingRTLLang(request) + rtl = translations.usingRTLLang(langs) except Exception as err: logging.exception(err) logging.error("The gettext files were not properly installed.") @@ -883,87 +887,3 @@ def addWebServer(cfg, dist, sched): raise SystemExit(error) return site - -def usingRTLLang(request): - """Check if we should translate the text into a RTL language - - Retrieve the headers from the request. Obtain the Accept-Language header - and decide if we need to translate the text. Install the requisite - languages via gettext, if so. Then, manually check which languages we - support. Choose the first language from the header that we support and - return True if it is a RTL language, else return False. - - :type request: :api:`twisted.web.server.Request` - :param request: An incoming request. - :rtype: bool - :returns: ``True`` if the preferred language is right-to-left; ``False`` - otherwise. - """ - langs = setLocaleFromRequestHeader(request) - - # Grab only the language (first two characters) so we know if the language - # is read right-to-left - #langs = [ lang[:2] for lang in langs ] - lang = getAssumedChosenLang(langs) - if lang in rtl_langs: - return True - return False - -def getAssumedChosenLang(langs): - """Return the first language in **langs** that we support. - - :param list langs: All requested languages - :rtype: str - :returns: A country code for the client's preferred language. - """ - i18npath = os.path.join(os.path.dirname(__file__), 'i18n') - path = filepath.FilePath(i18npath) - assert path.isdir() - - lang = 'en-US' - supp_langs = path.listdir() + ['en'] - for l in langs: - if l in supp_langs: - lang = l - break - return lang - -def setLocaleFromRequestHeader(request): - """Retrieve the languages from the accept-language header and install them. - - Parse the languages in the header, and attempt to install the first one in - the list. If that fails, we receive a :class:`gettext.NullTranslation` - object, if it worked then we have a :class:`gettext.GNUTranslation` - object. Whichever one we end up with, get the other languages and add them - as fallbacks to the first. Lastly, install this chain of translations. - - :type request: :api:`twisted.web.server.Request` - :param request: An incoming request from a client. - :rtype: list - :returns: All requested languages. - """ - logging.debug("Getting client 'Accept-Language' header...") - header = request.getHeader('accept-language') - - if header is None: - logging.debug("Client sent no 'Accept-Language' header. Using fallback.") - header = 'en,en-US' - - localedir = os.path.join(os.path.dirname(__file__), 'i18n/') - langs = headers.parseAcceptLanguage(header) - ## XXX the 'Accept-Language' header is potentially identifying - logging.debug("Client Accept-Language (top 5): %s" % langs[:5]) - - try: - language = gettext.translation("bridgedb", localedir=localedir, - languages=langs, fallback=True) - for lang in langs: - language.add_fallback(gettext.translation("bridgedb", - localedir=localedir, - languages=langs, - fallback=True)) - except IOError as error: - logging.error(error.message) - - language.install(unicode=True) - return langs diff --git a/lib/bridgedb/translations.py b/lib/bridgedb/translations.py new file mode 100644 index 0000000..3085938 --- /dev/null +++ b/lib/bridgedb/translations.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_translations -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2013-2014, Isis Lovecruft +# (c) 2007-2014, The Tor Project, Inc. +# :license: 3-Clause BSD, see LICENSE for licensing information + +import gettext +import logging +import os +import re + +from bridgedb import _langs +from bridgedb import safelog +from bridgedb.parse import headers + + +TRANSLATIONS_DIR = os.path.join(os.path.dirname(__file__), 'i18n') + + +def getFirstSupportedLang(langs): + """Return the first language in **langs** that we support. + + :param list langs: All requested languages + :rtype: str + :returns: A country code for the client's preferred language. + """ + lang = 'en-US' + supported = _langs.get_langs() + + for l in langs: + if l in supported: + lang = l + break + return lang + +def getLocaleFromHTTPRequest(request): + """Retrieve the languages from an HTTP ``Accept-Language:`` header. + + Parse the languages from the header, use them to install a + ``gettext.translation`` chain via :func:`installTranslations`, and lastly + return the requested languages. + + :type request: :api:`twisted.web.server.Request` + :param request: An incoming request from a client. + :rtype: list + :returns: All requested languages. + """ + header = request.getHeader('accept-language') + if header is None: + logging.debug("Client sent no 'Accept-Language' header. Using fallback.") + header = 'en,en-US' + + langs = headers.parseAcceptLanguage(header) + if not safelog.safe_logging: # pragma: no cover + logging.debug("Client Accept-Language (top 5): %s" % langs[:5]) + + # Check if we got a ?lang=foo argument, and if we did, insert it first + chosenLang = request.args.get("lang", [None,])[0] + if chosenLang: + logging.debug("Client requested language: %r" % chosenLang) + langs.insert(0, chosenLang) + + installTranslations(langs) + return langs + +def getLocaleFromPlusAddr(address): + """See whether the user sent his email to a 'plus' address, for instance to + bridges+fa(a)bridges.torproject.org. Plus addresses are the current + mechanism to set the reply language. + """ + replyLocale = "en" + r = '.*(<)?(\w+\+(\w+)@\w+(?:\.\w+)+)(?(1)>)' + match = re.match(r, address) + if match: + replyLocale = match.group(3) + + return replyLocale + +def installTranslations(langs): + """Create a ``gettext.translation`` chain for all **langs**. + + Attempt to install the first language in the **langs** list. If that + fails, we receive a ``gettext.NullTranslation`` object, and if it worked + then we have a ``gettext.GNUTranslation`` object. Whichever one we end up + with, get the other languages and add them as fallbacks to the + first. Lastly, install this chain of translations. + + :param list langs: A list of language codes. + :returns: A ``gettext.NullTranslation`` or ``gettext.GNUTranslation`` with + fallback languages set. + """ + try: + language = gettext.translation("bridgedb", localedir=TRANSLATIONS_DIR, + languages=langs, fallback=True) + for lang in langs: + language.add_fallback( + gettext.translation("bridgedb", localedir=TRANSLATIONS_DIR, + languages=langs, fallback=True)) + except IOError as error: + logging.error(error.message) + + language.install(unicode=True) + return language + +def usingRTLLang(langs): + """Check if we should translate the text into a RTL language. + + Choose the first language from the **langs** list that we support and + return True if it is a RTL language, else return False. + + :param list langs: An incoming request. + :rtype: bool + :returns: ``True`` if the preferred language is right-to-left; ``False`` + otherwise. + """ + lang = getFirstSupportedLang(langs) + if lang in _langs.RTL_LANGS: + return True + return False

1 0

[bridgedb/master] PEP8; change template_root → TEMPLATE_DIR.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 4a9eb669830ae8a3d12335227cce7c34b5002193 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 19:38:44 2014 +0000 PEP8; change template_root → TEMPLATE_DIR. --- lib/bridgedb/HTTPServer.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 5e3ba37..1d4cdf9 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -44,10 +44,7 @@ from bridgedb.parse import headers from bridgedb.safelog import logSafely - -template_root = os.path.join(os.path.dirname(__file__),'templates') -logging.debug("Set template root to %s" % template_root) - +TEMPLATE_DIR = os.path.join(os.path.dirname(__file__),'templates') rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') # Setting `filesystem_checks` to False is recommended for production servers, @@ -57,13 +54,14 @@ rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') # recompiled). `collection_size` sets the number of compiled templates which # are cached before the least recently used ones are removed. See: # http://docs.makotemplates.org/en/latest/usage.html#using-templatelookup -lookup = TemplateLookup(directories=[template_root], +lookup = TemplateLookup(directories=[TEMPLATE_DIR], output_encoding='utf-8', filesystem_checks=False, collection_size=500) _geoipdb = '/usr/share/GeoIP/GeoIP.dat' +logging.debug("Set template root to %s" % TEMPLATE_DIR) try: # Make sure we have the database before trying to import the module: @@ -822,9 +820,9 @@ def addWebServer(cfg, dist, sched): httpdist = resource.Resource() httpdist.putChild('', WebRoot()) httpdist.putChild('robots.txt', - static.File(os.path.join(template_root, 'robots.txt'))) + static.File(os.path.join(TEMPLATE_DIR, 'robots.txt'))) httpdist.putChild('assets', - static.File(os.path.join(template_root, 'assets/'))) + static.File(os.path.join(TEMPLATE_DIR, 'assets/'))) httpdist.putChild('options', WebResourceOptions()) httpdist.putChild('howto', WebResourceHowto())

1 0

[bridgedb/master] Add unittests for new bridgedb.translations module.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit a6edfaeb6fc8036c60d9c20bf871795cba198337 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 19:19:49 2014 +0000 Add unittests for new bridgedb.translations module. --- lib/bridgedb/test/test_translations.py | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/lib/bridgedb/test/test_translations.py b/lib/bridgedb/test/test_translations.py new file mode 100644 index 0000000..ee15ac4 --- /dev/null +++ b/lib/bridgedb/test/test_translations.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2014, Isis Lovecruft +# (c) 2014, The Tor Project, Inc. +# :license: 3-Clause BSD, see LICENSE for licensing information + + +from twisted.trial import unittest + +from bridgedb import translations +from bridgedb.test.test_HTTPServer import DummyRequest + + +REALISH_HEADERS = { + b'Accept-Encoding': [b'gzip, deflate'], + b'User-Agent': [ + b'Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0'], + b'Accept': [ + b'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'], +} + +# Add this to the above REALISH_HEADERS to use it: +ACCEPT_LANGUAGE_HEADER = { + b'Accept-Language': [b'de-de,en-gb;q=0.8,en;q=0.5,en-us;q=0.3'], +} + + +class TranslationsMiscTests(unittest.TestCase): + """Tests for module-level code in ``bridgedb.translations`` module.""" + + def test_getLocaleFromHTTPRequest_withLangParam(self): + """This request uses a '?lang=ar' param, without an 'Accept-Language' + header. + + The request result should be: ['ar', 'en', 'en-US']. + """ + request = DummyRequest([b"bridges"]) + request.headers.update(REALISH_HEADERS) + request.args.update({ + b'transport': [b'obfs3',], + b'lang': [b'ar',], + }) + + parsed = translations.getLocaleFromHTTPRequest(request) + self.assertEqual(parsed[0], 'ar') + self.assertEqual(parsed[1], 'en') + self.assertEqual(parsed[2], 'en_US') + self.assertEqual(len(parsed), 3) + + def test_getLocaleFromHTTPRequest_withLangParam_AcceptLanguage(self): + """This request uses a '?lang=ar' param, with an 'Accept-Language' + header which includes: ['de-de', 'en-gb', 'en', 'en-us']. + + The request result should be: ['fa', 'de-de', 'en-gb', 'en', 'en-us']. + """ + request = DummyRequest([b"options"]) + request.headers.update(ACCEPT_LANGUAGE_HEADER) + request.args.update({b'lang': [b'fa']}) + + parsed = translations.getLocaleFromHTTPRequest(request) + self.assertEqual(parsed[0], 'fa') + self.assertEqual(parsed[1], 'en') + self.assertEqual(parsed[2], 'en_US') + #self.assertEqual(parsed[3], 'en-gb') + self.assertEqual(len(parsed), 3) + + def test_getLocaleFromPlusAddr(self): + emailAddr = 'bridges(a)torproject.org' + replyLocale = translations.getLocaleFromPlusAddr(emailAddr) + self.assertEqual('en', replyLocale) + + def test_getLocaleFromPlusAddr_ar(self): + emailAddr = 'bridges+ar(a)torproject.org' + replyLocale = translations.getLocaleFromPlusAddr(emailAddr) + self.assertEqual('ar', replyLocale)

1 0

[bridgedb/master] Fix issue with transport URL params in HTTPServer unittests.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 5e37dc53df4704c9b4ab0f3943159a5198f0c515 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 19:26:19 2014 +0000 Fix issue with transport URL params in HTTPServer unittests. Also fixes several PEP8 whitespace issues in test_HTTPServer.py. --- lib/bridgedb/test/test_HTTPServer.py | 83 +++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index 111adc1..52c2fb7 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -178,8 +178,8 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): self.request.method = b'POST' self.request.addArg('captcha_challenge_field', expectedChallenge) - self.request.addArg('captcha_response_field', expectedResponse) - + self.request.addArg('captcha_response_field', expectedResponse) + response = self.captchaResource.extractClientSolution(self.request) (challenge, response) = response self.assertEqual(challenge, expectedChallenge) @@ -192,8 +192,8 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): self.request.method = b'POST' self.request.addArg('captcha_challenge_field', expectedChallenge) - self.request.addArg('captcha_response_field', expectedResponse) - + self.request.addArg('captcha_response_field', expectedResponse) + valid = self.captchaResource.checkSolution(self.request) self.assertFalse(valid) @@ -238,7 +238,7 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): """ self.request.method = b'POST' self.request.addArg('captcha_challenge_field', '') - self.request.addArg('captcha_response_field', '') + self.request.addArg('captcha_response_field', '') page = self.captchaResource.render_POST(self.request) self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'], @@ -253,7 +253,7 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): self.request.method = b'POST' self.request.addArg('captcha_challenge_field', expectedChallenge) - self.request.addArg('captcha_response_field', expectedResponse) + self.request.addArg('captcha_response_field', expectedResponse) page = self.captchaResource.render_POST(self.request) self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'], @@ -357,8 +357,8 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): """ self.request.method = b'POST' self.request.addArg('captcha_challenge_field', '') - self.request.addArg('captcha_response_field', '') - + self.request.addArg('captcha_response_field', '') + self.assertEqual((False, self.request), self.successResultOf( self.captchaResource.checkSolution(self.request))) @@ -398,7 +398,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): """ self.request.method = b'POST' self.request.addArg('captcha_challenge_field', '') - self.request.addArg('captcha_response_field', '') + self.request.addArg('captcha_response_field', '') page = self.captchaResource.render_POST(self.request) self.assertEqual(page, HTTPServer.server.NOT_DONE_YET) @@ -412,7 +412,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): self.request.method = b'POST' self.request.addArg('captcha_challenge_field', expectedChallenge) - self.request.addArg('captcha_response_field', expectedResponse) + self.request.addArg('captcha_response_field', expectedResponse) page = self.captchaResource.render_POST(self.request) self.assertEqual(page, HTTPServer.server.NOT_DONE_YET) @@ -580,34 +580,51 @@ class WebResourceBridgesTests(unittest.TestCase): page = self.bridgesResource.render(request) self.bridgesResource.useForwardedHeader = False # Reset it - self.assertSubstring("To enter bridges into", page) + + # The response should explain how to use the bridge lines: + self.assertTrue("To enter bridges into Tor Browser" in str(page)) def test_render_GET_RTLlang(self): - """Test rendering a request for obfs3 bridges in Arabic.""" - request = DummyRequest(["bridges?transport=obfs3"]) + """Test rendering a request for plain bridges in Arabic.""" + request = DummyRequest([b"bridges?transport=obfs3"]) + request.method = b'GET' + request.getClientIP = lambda: '3.3.3.3' + # For some strange reason, the 'Accept-Language' value *should not* be + # a list, unlike all the other headers and args… + request.headers.update({'accept-language': 'ar,en,en_US,'}) + + page = self.bridgesResource.render(request) + self.assertSubstring("direction: rtl", page) + self.assertSubstring( + "إذا لم يعمل تور بنجاح معك، يجب عليك ارسال بريد إلكتروني إلي", page) + + for bridgeLine in self.parseBridgesFromHTMLPage(page): + # Check that each bridge line had the expected number of fields: + bridgeLine = bridgeLine.split(' ') + self.assertEqual(len(bridgeLine), 2) + + def test_render_GET_RTLlang_obfs3(self): + """Test rendering a request for obfs3 bridges in Farsi.""" + request = DummyRequest([b"bridges?transport=obfs3"]) request.method = b'GET' request.getClientIP = lambda: '3.3.3.3' - request.headers.update({'accept-language': 'ar'}) + request.headers.update({'accept-language': 'fa,en,en_US,'}) # We actually have to set the request args manually when using a # DummyRequest: - request.args.update({'transport': 'obfs3'}) + request.args.update({'transport': ['obfs3']}) page = self.bridgesResource.render(request) self.assertSubstring("direction: rtl", page) self.assertSubstring( - "إذا لم يعمل تور بنجاح معك، يجب عليك ارسال بريد إلكتروني إلي", page) + # "I need an alternative way to get bridges!" + "به یک راه دیگر برای دریافت پل‌ها احتیاج دارم!", + page) for bridgeLine in self.parseBridgesFromHTMLPage(page): # Check that each bridge line had the expected number of fields: bridgeLine = bridgeLine.split(' ') self.assertEqual(len(bridgeLine), 3) - - print(""" - FIXME: The first field should be the transport method: - DummyBridge.getConfigLine() receives transport='o', not 'obfs3'. - What the hell? - """) - #self.assertEqual(bridgeLine[0], 'obfs3') + self.assertEqual(bridgeLine[0], 'obfs3') # Check that the IP and port seem okay: ip, port = bridgeLine[1].rsplit(':') @@ -616,24 +633,28 @@ class WebResourceBridgesTests(unittest.TestCase): self.assertGreater(int(port), 0) self.assertLessEqual(int(port), 65535) + def test_renderAnswer_textplain(self): """If the request format specifies 'plain', we should return content with mimetype 'text/plain'. """ request = DummyRequest([self.pagename]) - request.args.update({'format': 'plain'}) + request.args.update({'format': ['plain']}) request.getClientIP = lambda: '4.4.4.4' request.method = b'GET' page = self.bridgesResource.render(request) + self.assertTrue("html" not in str(page)) - print(""" - FIXME: The result page should have a mimetype of 'text/plain', and yet - it's still HTML. - """) - #self.assertNotSubstring("html", page) + # We just need to strip and split it because it looks like: + # + # 94.235.85.233:9492 0d9d0547c3471cddc473f7288a6abfb54562dc06 + # 255.225.204.145:9511 1fb89d618b3a12afe3529fd072127ea08fb50466 + # + # (Yes, there are two leading spaces at the beginning of each line) + # + bridgeLines = [line.strip() for line in page.strip().split('\n')] - bridgeLines = self.parseBridgesFromHTMLPage(page) for bridgeLine in bridgeLines: bridgeLine = bridgeLine.split(' ') self.assertEqual(len(bridgeLine), 2) @@ -665,7 +686,7 @@ class WebResourceOptionsTests(unittest.TestCase): request.headers.update({'accept-language': 'he'}) # We actually have to set the request args manually when using a # DummyRequest: - request.args.update({'transport': 'obfs2'}) + request.args.update({'transport': ['obfs2']}) page = self.optionsResource.render(request) self.assertSubstring("direction: rtl", page)

1 0

[bridgedb/master] PEP8; change geoipdb → GEOIP_DBFILE.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 1115b5ab2b41780761c9f4fa52bb0dbefd41db45 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 19:40:47 2014 +0000 PEP8; change geoipdb → GEOIP_DBFILE. --- lib/bridgedb/HTTPServer.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 1d4cdf9..9176f6a 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -45,6 +45,7 @@ from bridgedb.safelog import logSafely TEMPLATE_DIR = os.path.join(os.path.dirname(__file__),'templates') +GEOIP_DBFILE = '/usr/share/GeoIP/GeoIP.dat' rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') # Setting `filesystem_checks` to False is recommended for production servers, @@ -58,22 +59,19 @@ lookup = TemplateLookup(directories=[TEMPLATE_DIR], output_encoding='utf-8', filesystem_checks=False, collection_size=500) - - -_geoipdb = '/usr/share/GeoIP/GeoIP.dat' logging.debug("Set template root to %s" % TEMPLATE_DIR) try: # Make sure we have the database before trying to import the module: - if not os.path.isfile(_geoipdb): # pragma: no cover + if not os.path.isfile(GEOIP_DBFILE): # pragma: no cover raise EnvironmentError("Could not find %r. On Debian-based systems, "\ "please install the geoip-database package." - % _geoipdb) + % GEOIP_DBFILE) # This is a "pure" python version which interacts with the Maxmind GeoIP # API (version 1). It requires, in Debian, the libgeoip-dev and # geoip-database packages. import pygeoip - geoip = pygeoip.GeoIP(_geoipdb, flags=pygeoip.MEMORY_CACHE) + geoip = pygeoip.GeoIP(GEOIP_DBFILE, flags=pygeoip.MEMORY_CACHE) logging.info("GeoIP database loaded") except Exception as err: # pragma: no cover logging.warn("Error while loading geoip module: %r" % err)

1 0

[bridgedb/master] Update docstring for I18n._().
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 09a49cd22939986a4b038c7e48f75c7a44853312 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 16 19:44:47 2014 +0000 Update docstring for I18n._(). --- lib/bridgedb/I18n.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/I18n.py b/lib/bridgedb/I18n.py index 112b8ae..c6fbf3f 100644 --- a/lib/bridgedb/I18n.py +++ b/lib/bridgedb/I18n.py @@ -12,8 +12,10 @@ def getLang(lang, localedir=os.path.expanduser("~") + "/share/locale"): def _(text): """This is necessary because strings are translated when they're imported. - Otherwise this would make it impossible to switch languages more than - once + Otherwise this would make it impossible to switch languages more than + once. + + :returns: The **text**. """ return text

1 0

[bridgedb/master] Split unittests for EmailServer.getMailResponse() into smaller tests.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit 8274af32008893b461f81e96c2a037fe5a96b996 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 8 18:34:12 2014 +0000 Split unittests for EmailServer.getMailResponse() into smaller tests. --- lib/bridgedb/test/test_EmailServer.py | 58 +++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/lib/bridgedb/test/test_EmailServer.py b/lib/bridgedb/test/test_EmailServer.py index e8b9360..13f018c 100644 --- a/lib/bridgedb/test/test_EmailServer.py +++ b/lib/bridgedb/test/test_EmailServer.py @@ -134,7 +134,7 @@ class EmailCompositionTests(unittest.TestCase): """Tests for :func:`bridgedb.EmailServer.getMailResponse`.""" def setUp(self): - """Create fake email and associated data""" + """Create fake email, distributor, and associated context data.""" configuration = {} TEST_CONFIG_FILE.seek(0) compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') @@ -142,40 +142,45 @@ class EmailCompositionTests(unittest.TestCase): self.config = Conf(**configuration) # TODO: Add headers if we start validating them - self.lines = ["From: %s(a)%s.com", "To: %s(a)example.net", - "Subject: testing", "\n", "get bridges"] + self.lines = ["From: %s(a)%s.com", + "To: bridges(a)example.net", + "Subject: testing", + "\n", + "get bridges"] self.distributor = FakeDistributor('key', {}, {}, []) self.ctx = MailContext(self.config, self.distributor, NoSchedule()) - def test_getMailResponseNoFrom(self): + def test_getMailResponse_noFrom(self): + """A received email without a "From:" or "Sender:" header shouldn't + receive a response. + """ lines = self.lines lines[0] = "" - lines[1] = self.lines[1] % "bridges" ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple) self.assertEqual(len(ret), 2) self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) - def test_getMailResponseBadAddress(self): + def test_getMailResponse_badAddress(self): lines = copy.copy(self.lines) - lines[0] = self.lines[0] % ("testing?", "example") - lines[1] = self.lines[1] % "bridges" - lines[2] = "" + lines[0] = self.lines[0] % ("testing*.?\"", "example") ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple) self.assertEqual(len(ret), 2) self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) - #lines[0] = self.lines[0] % ("<>>", "example") - lines[0] = "From: %s(a)%s.com" % ("<>>", "example") + + def test_getMailResponse_anotherBadAddress(self): + lines = copy.copy(self.lines) + lines[0] = "From: Mallory %s(a)%s.com" % ("<>>", "example") ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple) self.assertEqual(len(ret), 2) self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) - def test_getMailResponseInvalidDomain(self): + def test_getMailResponse_invalidDomain(self): lines = copy.copy(self.lines) lines[0] = self.lines[0] % ("testing", "exa#mple") ret = EmailServer.getMailResponse(lines, self.ctx) @@ -183,6 +188,9 @@ class EmailCompositionTests(unittest.TestCase): self.assertEqual(len(ret), 2) self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) + + def test_getMailResponse_anotherInvalidDomain(self): + lines = copy.copy(self.lines) lines[0] = self.lines[0] % ("testing", "exam+ple") ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple) @@ -190,7 +198,10 @@ class EmailCompositionTests(unittest.TestCase): self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) - def test_getMailResponseDKIM(self): + def test_getMailResponse_DKIM_badDKIMheader(self): + """An email with an appended 'X-DKIM-Authentication-Result:' header should not + receive a response. + """ lines = copy.copy(self.lines) lines[0] = self.lines[0] % ("testing", "gmail") lines.append("X-DKIM-Authentication-Result: ") @@ -199,7 +210,11 @@ class EmailCompositionTests(unittest.TestCase): self.assertEqual(len(ret), 2) self.assertEqual(ret[0], None) self.assertEqual(ret[1], None) + + def test_getMailResponse_DKIM(self): + lines = copy.copy(self.lines) lines[0] = self.lines[0] % ("testing", "example") + lines.append("X-DKIM-Authentication-Result: ") ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple) self.assertEqual(len(ret), 2) @@ -208,7 +223,8 @@ class EmailCompositionTests(unittest.TestCase): mail = ret[1].getvalue() self.assertNotEqual(mail.find("no bridges currently"), -1) - def test_getMailResponseMailContent(self): + def test_getMailResponse_bridges_obfs(self): + """A request for 'transport obfs' should receive a response.""" lines = copy.copy(self.lines) lines[0] = self.lines[0] % ("testing", "example") lines.append("transport obfs") @@ -219,6 +235,12 @@ class EmailCompositionTests(unittest.TestCase): self.assertIsInstance(ret[1], StringIO) mail = ret[1].getvalue() self.assertNotEqual(mail.find("no bridges currently"), -1) + + def test_getMailResponse_bridges_obfsobfswebz(self): + """We should only pay attention to the first in a crazy request.""" + lines = copy.copy(self.lines) + lines[0] = self.lines[0] % ("testing", "example") + lines.append("transport obfs") lines.append("transport obfs") lines.append("unblocked webz") ret = EmailServer.getMailResponse(lines, self.ctx) @@ -228,6 +250,14 @@ class EmailCompositionTests(unittest.TestCase): self.assertIsInstance(ret[1], StringIO) mail = ret[1].getvalue() self.assertNotEqual(mail.find("no bridges currently"), -1) + + def test_getMailResponse_bridges_obfsobfswebzipv6(self): + """We should *still* only pay attention to the first request.""" + lines = copy.copy(self.lines) + lines[0] = self.lines[0] % ("testing", "example") + lines.append("transport obfs") + lines.append("transport obfs") + lines.append("unblocked webz") lines.append("ipv6") ret = EmailServer.getMailResponse(lines, self.ctx) self.assertIsInstance(ret, tuple)

1 0

[bridgedb/master] Add unittest for EmailServer.replyToMail().
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit f9e2175750b998366492fbb197594b8eb226fbaa Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 8 21:28:34 2014 +0000 Add unittest for EmailServer.replyToMail(). --- lib/bridgedb/test/test_EmailServer.py | 42 ++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_EmailServer.py b/lib/bridgedb/test/test_EmailServer.py index 13f018c..f828be7 100644 --- a/lib/bridgedb/test/test_EmailServer.py +++ b/lib/bridgedb/test/test_EmailServer.py @@ -26,7 +26,9 @@ from bridgedb.EmailServer import MailContext from bridgedb.Time import NoSchedule from bridgedb.persistent import Conf from bridgedb.test.util import fileCheckDecorator + from twisted.python import log +from twisted.internet import defer from twisted.trial import unittest @@ -130,7 +132,7 @@ class EmailGnuPGTest(unittest.TestCase): ctx = EmailServer.getGPGContext(self.config) self.assertTrue(ctx is None) -class EmailCompositionTests(unittest.TestCase): +class EmailResponseTests(unittest.TestCase): """Tests for :func:`bridgedb.EmailServer.getMailResponse`.""" def setUp(self): @@ -267,6 +269,44 @@ class EmailCompositionTests(unittest.TestCase): mail = ret[1].getvalue() self.assertNotEqual(mail.find("no bridges currently"), -1) + +class EmailReplyTests(unittest.TestCase): + """Tests for ``EmailServer.replyToMail()``.""" + + def setUp(self): + """Create fake email, distributor, and associated context data.""" + configuration = {} + TEST_CONFIG_FILE.seek(0) + compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') + exec compiled in configuration + self.config = Conf(**configuration) + + # TODO: Add headers if we start validating them + self.lines = ["From: %s(a)%s.com", + "To: bridges(a)example.net", + "Subject: testing", + "\n", + "get bridges"] + self.distributor = FakeDistributor('key', {}, {}, []) + self.ctx = MailContext(self.config, self.distributor, NoSchedule()) + + def test_replyToMail(self): + self.skip = True + raise unittest.SkipTest("We'll have to fake the EmailServer for this one,"\ + " it requires a TCP connection to localhost.") + + def callback(reply): + self.assertSubstring("Here are your bridges", reply) + + lines = copy.copy(self.lines) + lines[0] = self.lines[0] % ("testing", "example") + reply = EmailServer.replyToMail(lines, self.ctx) + + self.assertIsInstance(reply, defer.Deferred) + + reply.addCallback(callback) + return reply + class EmailServerServiceTests(unittest.TestCase): def setUp(self): configuration = {}

1 0

[bridgedb/master] Fix unicode encoding errors in outgoing SMTP transport.
by isis＠torproject.org 19 Apr '14

19 Apr '14

commit fcaf0a5a4401d9f85bd16b550edfefaa2ef50559 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 8 23:30:27 2014 +0000 Fix unicode encoding errors in outgoing SMTP transport. * REFACTOR bridgedb.EmailServer.composeEmail() to use io.BytesIO. --- lib/bridgedb/EmailServer.py | 83 +++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/lib/bridgedb/EmailServer.py b/lib/bridgedb/EmailServer.py index e1aa57e..db2cf80 100644 --- a/lib/bridgedb/EmailServer.py +++ b/lib/bridgedb/EmailServer.py @@ -8,9 +8,9 @@ from __future__ import unicode_literals from email import message -from io import StringIO import gettext import gpgme +import io import logging import re import rfc822 @@ -315,6 +315,47 @@ def getLocaleFromRequest(request): lang = lang[0] return I18n.getLang(lang) +def composeEmail(fromAddr, clientAddr, subject, body, + msgID=None, gpgContext=None): + + if not subject.startswith("Re:"): + subject = "Re: %s" % subject + + msg = smtp.rfc822.Message(io.StringIO()) + msg.setdefault("From", fromAddr) + msg.setdefault("To", clientAddr) + msg.setdefault("Message-ID", smtp.messageid()) + msg.setdefault("Subject", subject) + if msgID: + msg.setdefault("In-Reply-To", msgID) + msg.setdefault("Date", smtp.rfc822date()) + msg.setdefault('Content-Type', 'text/plain; charset="utf-8"') + headers = [': '.join(m) for m in msg.items()] + + mail = io.BytesIO() + mail.writelines(buffer("\r\n".join(headers))) + mail.writelines(buffer("\r\n")) + mail.writelines(buffer("\r\n")) + + if not gpgContext: + mail.write(buffer(body)) + else: + signature, siglist = gpgSignMessage(gpgContext, body) + if signature: + mail.writelines(buffer(signature)) + mail.seek(0) + + # Only log the email text (including all headers) if SAFE_LOGGING is + # disabled: + if not safelog.safe_logging: + logging.debug("Email contents:\n\n%s" % mail.read()) + mail.seek(0) + else: + logging.debug("Email text for %r created." % clientAddr) + + return clientAddr, mail + + class MailContext(object): """Helper object that holds information used by email subsystem.""" @@ -439,45 +480,9 @@ def addSMTPServer(cfg, dist, sched): lc.start(1800, now=False) return factory -def composeEmail(fromAddr, clientAddr, subject, body, msgID=False, - gpgContext=None): - msg = message.Message() - msg.add_header("From", fromAddr) - msg.add_header("To", clientAddr) - msg.add_header("Message-ID", smtp.messageid()) - if not subject.startswith("Re:"): subject = "Re: %s"%subject - msg.add_header("Subject", subject) - if msgID: - msg.add_header("In-Reply-To", msgID) - msg.add_header("Date", smtp.rfc822date()) - msg.set_default_type("text/plain") - headers = [': '.join(m) for m in msg.items()] - mail = StringIO("\r\n".join(headers)) - mail.writelines(unicode(msg.as_string())) - - # gpg-clearsign messages - if gpgContext: - signature = StringIO() - plaintext = StringIO(body) - sigs = gpgContext.sign(plaintext, signature, gpgme.SIG_MODE_CLEAR) - if (len(sigs) != 1): - logging.warn('Failed to sign message!') - signature.seek(0) - [mail.write(l) for l in signature] - else: - mail.write(body) - # Only log the email text (including all headers) if SAFE_LOGGING is - # disabled: - if not safelog.safe_logging: - mail.seek(0) - logging.debug("Email contents:\n%s" % mail.read()) - else: - logging.debug("Email text for %r created." % clientAddr) - mail.seek(0) - return clientAddr, mail def getGPGContext(cfg): """Import a key from a file and initialise a context for GnuPG operations. @@ -520,8 +525,8 @@ def getGPGContext(cfg): ctx.signers = [ctx.get_key(fingerprint)] logging.info("Testing signature created with GnuPG key...") - message = StringIO('Test') - new_sigs = ctx.sign(message, StringIO(), gpgme.SIG_MODE_CLEAR) + message = io.StringIO('Test') + new_sigs = ctx.sign(message, io.StringIO(), gpgme.SIG_MODE_CLEAR) if not len(new_sigs) == 1: raise gpgme.GpgmeError( "Testing was unable to produce a signature with GnuPG key.")

1 0