commit ed438d0c2b00f02c29b60e561d37f241f6a6a4b8 Author: Isis Lovecruft isis@torproject.org Date: Wed Apr 16 16:22:40 2014 +0000

Move locate determination and translation code into separate module.

* ADD new bridgedb.translations module with the following methods:

- getFirstSupportedLang(): Essentially the same as the original HTTPServer.getAssumedChosenLang() function, except that the determination of which languages are supported uses _lang.getLangs() instead (this has been in the _lang module for a long time, and was meant for that purpose).

- getLocaleFromHTTPRequest(): This is taken mostly from the original setLocaleFromRequestHeader() (in the HTTPServer module) except that there was an extra function in the EmailServer module, called getLocaleFromRequest() ― which was not in use anywhere ― which included the parsing of '?lang=' HTTP GET/POST arguments. The parsing of '?lang=' was taken from that function and added to the original functionality of setLocaleFromRequestHeader(). This fixes part of ticket #9678: "'Select Language' button on bridges.tpo".

- getLocaleFromPlusAddr(): Exactly the same as the original in the EmailServer module.

- installTranslations(): Produces a gettext translation object, with fallback languages chained to it in order of client preference.

- usingRTLLang(): Essentially the same as the original in the HTTPServer module, except that the parsing of supported langs again uses the _lang.getLangs() function, as above.

* FIXES part of #9678: "'Select Language' button to bridges.tpo" --- lib/bridgedb/EmailServer.py | 29 ++-------- lib/bridgedb/HTTPServer.py | 94 +++----------------------------- lib/bridgedb/translations.py | 122 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 113 deletions(-)

diff --git a/lib/bridgedb/EmailServer.py b/lib/bridgedb/EmailServer.py index f9f43bb..c58ec53 100644 --- a/lib/bridgedb/EmailServer.py +++ b/lib/bridgedb/EmailServer.py @@ -30,6 +30,7 @@ from bridgedb.Dist import BadEmail, TooSoonEmail, IgnoreEmail from bridgedb import Dist from bridgedb import I18n from bridgedb import safelog +from bridgedb import translations from bridgedb.Filters import filterBridgesByIP6 from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByTransport @@ -94,8 +95,8 @@ def getMailResponse(lines, ctx):

# Look up the locale part in the 'To:' address, if there is one and get # the appropriate Translation object - lang = getLocaleFromPlusAddr(clientToaddr) - t = I18n.getLang(lang) + lang = translations.getLocaleFromPlusAddr(clientToaddr) + t = translations.installTranslations(lang)

try: _, addrdomain = Dist.extractAddrSpec(clientAddr.lower()) @@ -287,30 +288,6 @@ def replyToMail(lines, ctx): reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory) return d

-def getLocaleFromPlusAddr(address): - """See whether the user sent his email to a 'plus' address, for - instance to bridgedb+fa@tpo. Plus addresses are the current - mechanism to set the reply language - """ - replyLocale = "en" - r = '.*(<)?(\w++(\w+)@\w+(?:.\w+)+)(?(1)>)' - match = re.match(r, address) - if match: - replyLocale = match.group(3) - - return replyLocale - -def getLocaleFromRequest(request): - # See if we did get a request for a certain locale, otherwise fall back - # to 'en': - # Try evaluating the path /foo first, then check if we got a ?lang=foo - default_lang = lang = "en" - if len(request.path) > 1: - lang = request.path[1:] - if lang == default_lang: - lang = request.args.get("lang", [default_lang]) - lang = lang[0] - return I18n.getLang(lang)

class MailContext(object): """Helper object that holds information used by email subsystem.""" diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 5e3ba37..79772c3 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -35,6 +35,7 @@ import bridgedb.I18n as I18n

from bridgedb import captcha from bridgedb import crypto +from bridgedb import translations from bridgedb import txrecaptcha from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByIP6 @@ -541,9 +542,10 @@ class WebResourceOptions(resource.Resource):

def render_GET(self, request): rtl = False + langs = translations.getLocaleFromHTTPRequest(request)

try: - rtl = usingRTLLang(request) + rtl = translations.usingRTLLang(langs) except Exception as err: # pragma: no cover logging.exception(err)

@@ -664,7 +666,8 @@ class WebResourceBridges(resource.Resource): if countryCode: logging.debug("Client request from GeoIP CC: %s" % countryCode)

- rtl = usingRTLLang(request) + langs = translations.getLocaleFromHTTPRequest(request) + rtl = translations.usingRTLLang(langs) if rtl: logging.debug("Rendering RTL response.")

@@ -784,9 +787,10 @@ class WebRoot(resource.Resource): :param request: An incoming request. """ rtl = False + langs = translations.getLocaleFromHTTPRequest(request)

try: - rtl = usingRTLLang(request) + rtl = translations.usingRTLLang(langs) except Exception as err: logging.exception(err) logging.error("The gettext files were not properly installed.") @@ -883,87 +887,3 @@ def addWebServer(cfg, dist, sched): raise SystemExit(error)

return site - -def usingRTLLang(request): - """Check if we should translate the text into a RTL language - - Retrieve the headers from the request. Obtain the Accept-Language header - and decide if we need to translate the text. Install the requisite - languages via gettext, if so. Then, manually check which languages we - support. Choose the first language from the header that we support and - return True if it is a RTL language, else return False. - - :type request: :api:`twisted.web.server.Request` - :param request: An incoming request. - :rtype: bool - :returns: ``True`` if the preferred language is right-to-left; ``False`` - otherwise. - """ - langs = setLocaleFromRequestHeader(request) - - # Grab only the language (first two characters) so we know if the language - # is read right-to-left - #langs = [ lang[:2] for lang in langs ] - lang = getAssumedChosenLang(langs) - if lang in rtl_langs: - return True - return False - -def getAssumedChosenLang(langs): - """Return the first language in **langs** that we support. - - :param list langs: All requested languages - :rtype: str - :returns: A country code for the client's preferred language. - """ - i18npath = os.path.join(os.path.dirname(__file__), 'i18n') - path = filepath.FilePath(i18npath) - assert path.isdir() - - lang = 'en-US' - supp_langs = path.listdir() + ['en'] - for l in langs: - if l in supp_langs: - lang = l - break - return lang - -def setLocaleFromRequestHeader(request): - """Retrieve the languages from the accept-language header and install them. - - Parse the languages in the header, and attempt to install the first one in - the list. If that fails, we receive a :class:`gettext.NullTranslation` - object, if it worked then we have a :class:`gettext.GNUTranslation` - object. Whichever one we end up with, get the other languages and add them - as fallbacks to the first. Lastly, install this chain of translations. - - :type request: :api:`twisted.web.server.Request` - :param request: An incoming request from a client. - :rtype: list - :returns: All requested languages. - """ - logging.debug("Getting client 'Accept-Language' header...") - header = request.getHeader('accept-language') - - if header is None: - logging.debug("Client sent no 'Accept-Language' header. Using fallback.") - header = 'en,en-US' - - localedir = os.path.join(os.path.dirname(__file__), 'i18n/') - langs = headers.parseAcceptLanguage(header) - ## XXX the 'Accept-Language' header is potentially identifying - logging.debug("Client Accept-Language (top 5): %s" % langs[:5]) - - try: - language = gettext.translation("bridgedb", localedir=localedir, - languages=langs, fallback=True) - for lang in langs: - language.add_fallback(gettext.translation("bridgedb", - localedir=localedir, - languages=langs, - fallback=True)) - except IOError as error: - logging.error(error.message) - - language.install(unicode=True) - return langs diff --git a/lib/bridgedb/translations.py b/lib/bridgedb/translations.py new file mode 100644 index 0000000..3085938 --- /dev/null +++ b/lib/bridgedb/translations.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_translations -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# :copyright: (c) 2013-2014, Isis Lovecruft +# (c) 2007-2014, The Tor Project, Inc. +# :license: 3-Clause BSD, see LICENSE for licensing information + +import gettext +import logging +import os +import re + +from bridgedb import _langs +from bridgedb import safelog +from bridgedb.parse import headers + + +TRANSLATIONS_DIR = os.path.join(os.path.dirname(__file__), 'i18n') + + +def getFirstSupportedLang(langs): + """Return the first language in **langs** that we support. + + :param list langs: All requested languages + :rtype: str + :returns: A country code for the client's preferred language. + """ + lang = 'en-US' + supported = _langs.get_langs() + + for l in langs: + if l in supported: + lang = l + break + return lang + +def getLocaleFromHTTPRequest(request): + """Retrieve the languages from an HTTP ``Accept-Language:`` header. + + Parse the languages from the header, use them to install a + ``gettext.translation`` chain via :func:`installTranslations`, and lastly + return the requested languages. + + :type request: :api:`twisted.web.server.Request` + :param request: An incoming request from a client. + :rtype: list + :returns: All requested languages. + """ + header = request.getHeader('accept-language') + if header is None: + logging.debug("Client sent no 'Accept-Language' header. Using fallback.") + header = 'en,en-US' + + langs = headers.parseAcceptLanguage(header) + if not safelog.safe_logging: # pragma: no cover + logging.debug("Client Accept-Language (top 5): %s" % langs[:5]) + + # Check if we got a ?lang=foo argument, and if we did, insert it first + chosenLang = request.args.get("lang", [None,])[0] + if chosenLang: + logging.debug("Client requested language: %r" % chosenLang) + langs.insert(0, chosenLang) + + installTranslations(langs) + return langs + +def getLocaleFromPlusAddr(address): + """See whether the user sent his email to a 'plus' address, for instance to + bridges+fa@bridges.torproject.org. Plus addresses are the current + mechanism to set the reply language. + """ + replyLocale = "en" + r = '.*(<)?(\w++(\w+)@\w+(?:.\w+)+)(?(1)>)' + match = re.match(r, address) + if match: + replyLocale = match.group(3) + + return replyLocale + +def installTranslations(langs): + """Create a ``gettext.translation`` chain for all **langs**. + + Attempt to install the first language in the **langs** list. If that + fails, we receive a ``gettext.NullTranslation`` object, and if it worked + then we have a ``gettext.GNUTranslation`` object. Whichever one we end up + with, get the other languages and add them as fallbacks to the + first. Lastly, install this chain of translations. + + :param list langs: A list of language codes. + :returns: A ``gettext.NullTranslation`` or ``gettext.GNUTranslation`` with + fallback languages set. + """ + try: + language = gettext.translation("bridgedb", localedir=TRANSLATIONS_DIR, + languages=langs, fallback=True) + for lang in langs: + language.add_fallback( + gettext.translation("bridgedb", localedir=TRANSLATIONS_DIR, + languages=langs, fallback=True)) + except IOError as error: + logging.error(error.message) + + language.install(unicode=True) + return language + +def usingRTLLang(langs): + """Check if we should translate the text into a RTL language. + + Choose the first language from the **langs** list that we support and + return True if it is a RTL language, else return False. + + :param list langs: An incoming request. + :rtype: bool + :returns: ``True`` if the preferred language is right-to-left; ``False`` + otherwise. + """ + lang = getFirstSupportedLang(langs) + if lang in _langs.RTL_LANGS: + return True + return False