January 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Add bridgedb.test package init file.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 96d887e5399d2b7a8ed4d97c71e518c6bd6e9caf Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Oct 26 12:51:40 2013 +0000 Add bridgedb.test package init file. --- 0 files changed diff --git a/lib/bridgedb/test/__init__.py b/lib/bridgedb/test/__init__.py new file mode 100644 index 0000000..e69de29

1 0

[bridgedb/master] Fix with combined with os.open; doesn't work on Python2.6.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 11d976b50b8983174d9fdf8e6a276b1639cd0b95 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Oct 26 13:00:14 2013 +0000 Fix with combined with os.open; doesn't work on Python2.6. --- lib/bridgedb/crypto.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/crypto.py b/lib/bridgedb/crypto.py index ea00733..43b2b79 100644 --- a/lib/bridgedb/crypto.py +++ b/lib/bridgedb/crypto.py @@ -44,9 +44,10 @@ def getKey(filename): except IOError: key = OpenSSL.rand.bytes(32) flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT | getattr(os, "O_BIN", 0) - with os.open(filename, flags, 0400) as fd: - os.write(fd, key) - os.fsync(fd) + fd = os.open(filename, flags, 0400) + os.write(fd, key) + os.fsync(fd) + fd.close() else: key = fh.read() fh.close()

1 0

[bridgedb/master] Add logging to crypto module.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 6764b9cfeea5b221557a81e6111044bb0fe280ae Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Oct 26 23:33:18 2013 +0000 Add logging to crypto module. --- lib/bridgedb/crypto.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/bridgedb/crypto.py b/lib/bridgedb/crypto.py index 43b2b79..124d337 100644 --- a/lib/bridgedb/crypto.py +++ b/lib/bridgedb/crypto.py @@ -12,6 +12,9 @@ from __future__ import absolute_import from __future__ import unicode_literals +import logging +log = logging.getLogger(__name__) + import os import OpenSSL.rand @@ -42,6 +45,7 @@ def getKey(filename): try: fh = open(filename, 'rb') except IOError: + log.debug("getKey(): Creating new secret key.") key = OpenSSL.rand.bytes(32) flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT | getattr(os, "O_BIN", 0) fd = os.open(filename, flags, 0400) @@ -49,6 +53,7 @@ def getKey(filename): os.fsync(fd) fd.close() else: + log.debug("getKey(): Secret key file found. Loading...") key = fh.read() fh.close() return key

1 0

[bridgedb/master] Handle GPGME errors better in the EmailServer.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit fe43555b261f37660bb02d31ca4f5dfaf9668197 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Oct 29 05:45:28 2013 +0000 Handle GPGME errors better in the EmailServer. * CHANGE getGPGContext() function to output at least some useful information when it encounters an error, rather than just spewing 'Unusable GPG Key' without any context. --- lib/bridgedb/EmailServer.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/EmailServer.py b/lib/bridgedb/EmailServer.py index 23fc8c1..1ee27c0 100644 --- a/lib/bridgedb/EmailServer.py +++ b/lib/bridgedb/EmailServer.py @@ -465,8 +465,13 @@ def getGPGContext(cfg): # make sure we can sign message = StringIO('Test') signature = StringIO() - new_sigs = ctx.sign(message, signature, gpgme.SIG_MODE_CLEAR) - assert len(new_sigs) == 1 + try: + new_sigs = ctx.sign(message, signature, gpgme.SIG_MODE_CLEAR) + except gpgme.GpgmeError as error: + logging.error(error.message) + return None + + assert len(new_sigs) == 1, "Testing signature creation failed" # return the ctx return ctx @@ -474,6 +479,7 @@ def getGPGContext(cfg): except IOError, e: # exit noisily if keyfile not found exit(e) - except AssertionError: + except AssertionError as error: + logging.error(error.message) # exit noisily if key does not pass tests exit('Invalid GPG Signing Key')

1 0

[bridgedb/master] Merge branch 'feature/cleanup-config' into develop
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 8318b3e276c429a8d5a3b04d9fc20e4be3a4d757 Merge: 6764b9c 5f26b66 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Oct 29 05:45:28 2013 +0000 Merge branch 'feature/cleanup-config' into develop bridgedb.conf | 268 +++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 185 insertions(+), 83 deletions(-)

1 0

[bridgedb/master] Fix the parsing of Accept-Language to actually support fallbacks.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit e1ae7a0ac0b5a8363fce07732d1b0e685547e582 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Oct 29 05:45:28 2013 +0000 Fix the parsing of Accept-Language to actually support fallbacks. All right. Going through the most significant bugs in this function: * The line from the original function: ``langs = request.getHeader('accept-language').split(',')`` getHeader() returns ``None`` if the header isn't present, so this results in a TypeError on the split(). * The line from the original function: ``langs = filter(lambda x: re.match('^[a-z\-]{1,5}', x), langs)`` This chucks locales with capital letters, and doesn't much at all to insure that we're actually getting a well-formed header, all at the expense of a (rather expensive; they're slow in Python) regex call. * These lines from the original: # add fallback languages langs_only = filter(lambda x: '-' in x, langs) langs.extend(map(lambda x: x.split('-')[0], langs_only)) If my 'Accept-Language' header starts with 'en-GB,en-US;q=0.92[…]', then this would add ['en','en'] to the end of my header, without even checking if I already have 'en'. Instead, we should check if 'en' is already there, and iff not, then add it *after the other English headers*. Not after Mandarin, Japanese, Arabic, Russian, and the other slew of languages that I half-assedly learned at some point. * These lines from the original: # gettext wants _, not - map(lambda x: x.replace('-', '_'), langs) Great. A pretty mapping. Good thing the returned values weren't saved as anything; otherwise they might have been useful! * Lastly, the way that languages, once parsed were added to gettext, would raise UnhandledErrors, *and* it didn't even add the fallbacks correctly, meaning we could only get one language at a time. --- lib/bridgedb/HTTPServer.py | 62 +++++++++++++++++++------------ lib/bridgedb/parse/headers.py | 82 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 24 deletions(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index ad76568..ca850d5 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -30,6 +30,7 @@ from bridgedb.Raptcha import Raptcha from bridgedb.Filters import filterBridgesByIP6, filterBridgesByIP4 from bridgedb.Filters import filterBridgesByTransport from bridgedb.Filters import filterBridgesByNotBlockedIn +from bridgedb.parse import headers from ipaddr import IPv4Address, IPv6Address from random import randint from mako.template import Template @@ -328,29 +329,42 @@ def getAssumedChosenLang(langs): return lang def setLocaleFromRequestHeader(request): + """Retrieve the languages from the accept-language header and install them. + + Parse the languages in the header, and attempt to install the first one in + the list. If that fails, we receive a :class:`gettext.NullTranslation` + object, if it worked then we have a :class:`gettext.GNUTranslation` + object. Whichever one we end up with, add the other get the other + languages and add them as fallbacks to the first. Lastly, install this + chain of translations. + + :type request: :class:`twisted.web.server.Request` + :param request: An incoming request from a client. + :rtype: list + :returns: All requested languages. """ - Retrieve the languages from the accept-language header and insall - - Parse the languages in the header, if any of them contain locales then - add their languages to the list, also. Then install all of them using - gettext, it will choose the best one. - - :param request twisted.web.server.Request: Incoming request - :returns list: All requested languages - """ - langs = request.getHeader('accept-language').split(',') - logging.debug("Accept-Language: %s" % langs) - localedir=os.path.join(os.path.dirname(__file__), 'i18n/') - - if langs: - langs = filter(lambda x: re.match('^[a-z\-]{1,5}', x), langs) - logging.debug("Languages: %s" % langs) - # add fallback languages - langs_only = filter(lambda x: '-' in x, langs) - langs.extend(map(lambda x: x.split('-')[0], langs_only)) - # gettext wants _, not - - map(lambda x: x.replace('-', '_'), langs) - lang = gettext.translation("bridgedb", localedir=localedir, - languages=langs, fallback=True) - lang.install(True) + logging.debug("Getting client 'Accept-Language' header...") + header = request.getHeader('accept-language') + + if header is None: + logging.debug("Client sent no 'Accept-Language' header. Using fallback.") + header = 'en,en-US' + + localedir = os.path.join(os.path.dirname(__file__), 'i18n/') + langs = headers.parseAcceptLanguage(header) + ## XXX the 'Accept-Language' header is potentially identifying + logging.debug("Client Accept-Language (top 5): %s" % langs[:4]) + + try: + language = gettext.translation("bridgedb", localedir=localedir, + languages=langs, fallback=True) + for lang in langs: + language.add_fallback(gettext.translation("bridgedb", + localedir=localedir, + languages=langs, + fallback=True)) + except IOError as error: + logging.error(error.message) + + language.install(unicode=True) return langs diff --git a/lib/bridgedb/parse/headers.py b/lib/bridgedb/parse/headers.py new file mode 100644 index 0000000..d9f1c4b --- /dev/null +++ b/lib/bridgedb/parse/headers.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013 Isis Lovecruft +# (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: 3-clause BSD, see included LICENSE for information + +"""bridgedb.parse.headers -- Parsers for HTTP and Email headers. + +** Module Overview: ** + +:: + + parseAcceptLanguage - Parse the contents of a client 'Accept-Language' header + +""" + +import logging +log = logging.getLogger() + +import re +import os + +def parseAcceptLanguage(header): + """Parse the contents of a client 'Accept-Language' header. + + Parse the header in the following manner: + + 0. If ``header`` is None or an empty string, return an empty list. + 1. Split the ``header`` string on any commas. + 2. Chop of the RFC2616 quality/level suffix. We ignore these, and just + use the order of the list as the preference order, without any + parsing of quality/level assignments. + 3. Add a fallback language of the same type if it is missing. For + example, if we only got ['es-ES', 'de-DE'], add 'es' after 'es-ES' + and add 'de' after 'de-DE'. + 4. Change all hyphens to underscores. + + :param string header: The contents of an 'Accept-Language' header, i.e. as + if taken from :func:`twisted.web.server.Request.getHeader`. + :rtype: list + :returns: A list of language codes (with and without locales), in order of + preference. + """ + langs = [] + + if not header: + return langs + + langHeader = header.split(',') + + for lang in langHeader: + if lang.find(';') != -1: + # Chop off the RFC2616 Accept `q=` and `level=` feilds + code, _ = lang.split(';') + langs.append(code) + else: + langs.append(lang) + + # Add a fallback language of the same type if it is missing. + langsWithLocales = filter(lambda x: '-' in x, langs) + langsOnly = map(lambda x: x.split('-')[0], langsWithLocales) + for only in langsOnly: + if only not in langs: + # Add the fallback after the other languages like it: + insertAfter = filter(lambda x: x.startswith(only), + [x for x in langs]) + if insertAfter: + placement = langs.index(insertAfter[0]) + 1 + langs.insert(placement, only) + continue + # Otherwise just put it at the end + langs.append(only) + + # Gettext wants underderscores, because that is how it creates the + # directories under i18n/, not hyphens: + langs = map(lambda x: x.replace('-', '_'), [x for x in langs]) + return langs

1 0

[bridgedb/master] Add better error handling if we cannot reach the Recaptcha server.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 9b3cb4e1be26ae4a6967b0acd2dddf2093bf7e92 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Oct 29 05:45:28 2013 +0000 Add better error handling if we cannot reach the Recaptcha server. --- lib/bridgedb/HTTPServer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 1143675..ad76568 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -78,7 +78,10 @@ class CaptchaProtectedResource(twisted.web.resource.Resource): def render_GET(self, request): # get a captcha c = Raptcha(self.recaptchaPubKey, self.recaptchaPrivKey) - c.get() + try: + c.get() + except Exception as error: + log.error("Connection to Recaptcha server failed.") # TODO: this does not work for versions of IE < 8.0 imgstr = 'data:image/jpeg;base64,%s' % base64.b64encode(c.image)

1 0

[bridgedb/master] Get rid of bridgedb.Main.CONFIG_DEFAULTS dict.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 1ef4e2a4bda2fae26adc74b540facaa221611412 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 05:22:46 2013 +0000 Get rid of bridgedb.Main.CONFIG_DEFAULTS dict. * REMOVE dictionary bridgedb.Main.CONFIG_DEFAULTS of config settings. These are already set, so I don't know why they should be duplicated here. * REMOVE method bridgedb.Main.Conf.setMissing(), which did nothing except for set the config items to the settings from the above mentioned dict if they were missing. --- lib/bridgedb/Main.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 5701681..b196bc5 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -28,24 +28,11 @@ import bridgedb.Util as Util class Conf: - """A configuration object. Holds unvalidated attributes. - """ + """A configuration object. Holds unvalidated attributes.""" def __init__(self, **attrs): - self.__dict__.update(attrs) - self.setMissing() - - def setMissing(self): - for k,v in CONFIG_DEFAULTS.items(): - if not hasattr(self, k): - setattr(self,k,v) - -CONFIG_DEFAULTS = { - 'HTTPS_INCLUDE_FINGERPRINTS' : False, - 'EMAIL_INCLUDE_FINGERPRINTS' : False, - 'RECAPTCHA_ENABLED' : False, - 'RECAPTCHA_PUB_KEY' : "", - 'RECAPTCHA_PRIV_KEY' : "" -} + for key, value in attrs.items(): + if key.upper() == key: + self.__dict__[key] = value # An example configuration. Used for testing. See sample # bridgedb.conf for documentation.

1 0

[bridgedb/master] Update .gitignore to exclude log files even in the top-level dir.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 32ee30f487a7f29f660efaeb1ded137f54e87c1c Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 30 01:25:01 2013 +0000 Update .gitignore to exclude log files even in the top-level dir. --- .gitignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index d560f20..f642ca2 100644 --- a/.gitignore +++ b/.gitignore @@ -68,9 +68,9 @@ bridgedb.pid run/* # logs -*/bridgedb*.log -*/assignments.log -*/bridgedist.log +*bridgedb.log* +*assignments.log +*bridgedist.log *bridge-descriptors* *cached-descriptors*

1 0

[bridgedb/master] Update gen_bridge_descriptors docstrings, add TODO note on making NTOR keys.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit d9f08c775387e2eb71b0c66305c55d516260301c Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 30 01:44:44 2013 +0000 Update gen_bridge_descriptors docstrings, add TODO note on making NTOR keys. --- scripts/gen_bridge_descriptors | 49 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/scripts/gen_bridge_descriptors b/scripts/gen_bridge_descriptors index ddfd86e..7050647 100644 --- a/scripts/gen_bridge_descriptors +++ b/scripts/gen_bridge_descriptors @@ -13,6 +13,29 @@ # (c) 2013 The Tor Project, Inc. #______________________________________________________________________________ +"""gen_bridge_descriptors -- Generate fake Tor relay signed descriptors. + +** TODO: ** + +Finish enough CFFI_ bindings for the newer PyNaCl_ (or enough of the +SWIG_ bindings for the older pynacl_) to be able to emulate the following +curvecp_ command (the ``curvecp*`` commands require libchloride_): + + $ curvecpmakekey ntor-key + $ curvecpprintkey ntor-key > ntor-key.hex + $ python -c 'import binascii, sys; \ + key_hex=open('./ntor-key.hex','rb').read();\ + key_b64=binascii.b2a_base64(binascii.unhexlify(key_hex));\ + sys.stdout.write(key_b64);' + + .. _CFFI: https://cffi.readthedocs.org + .. _PyNaCl: https://github.com/seanlynch/pynacl + .. _SWIG: https://github.com/swig/swig + .. _pynacl: https://github.com/seanlynch/pynacl + .. _curvecp: http://curvecp.org/ + .. _libchloride: https://github.com/jedisct1/libchloride +""" + from __future__ import print_function from __future__ import absolute_import from __future__ import unicode_literals @@ -122,6 +145,12 @@ def randomPort(): return random.randint(1025, 65535) def getHexString(size): + """Get a capitalised hexidecimal string ``size`` bytes long. + + :param integer size: The number of bytes in the returned string. + :rtype: string + :returns: A hex string. + """ s = "" for i in xrange(size): s += random.choice("ABCDEF0123456789") @@ -234,9 +263,9 @@ def makeFingerprintLine(fingerprint, version=None): | opt fingerprint D4BB C339 2560 1B7F 226E 133B A85F 72AF E734 0B29 | + :param string fingerprint: A public key fingerprint in groups of four, + separated by spaces. :param string version: One of ``SERVER_VERSIONS``. - :param string timestamp: The timestamp, in seconds since Epoch, to record - in the 'published' line. :rtype: string :returns: An '@type [bridge-]server-descriptor' 'published' line. """ @@ -568,6 +597,16 @@ def generateNetstatus(idkey_digest, server_desc_digest, timestamp, ipv4, orport, ipv6=None, dirport=None, flags='Fast Guard Running Stable Valid', bandwidth_line=None): + """Generate an ``@type networkwork-status`` document (unsigned). + + DOCDOC + + :param XXX idkey_digest: The SHA-1 digest of the router's public identity + key. + :param XXX server_desc_digest: The SHA-1 digest of the router's + ``@type [bridge-]server-descriptor``, before the descriptor is signed. + :param XXX timestamp: + """ idkey_b64 = binascii.b2a_base64(idkey_digest) idb64 = str(idkey_b64).strip().rstrip('=========') @@ -625,6 +664,12 @@ def signDescriptorDigest(key, descriptorDigest, digest='sha1'): see http://www.emc.com/collateral/white-papers/h11300-pkcs-1v2-2-rsa-cryptograp… for why this function is totally wrong. + + :type key: :class:`OpenSSL.crypto.PKey` + :param key: An RSA private key. + :param string descriptorDigest: The raw SHA-1 digest of any descriptor + document. + :param string digest: The digest to use. (default: 'sha1') """ sig = binascii.b2a_base64(OpenSSL.crypto.sign(key, descriptorDigest, digest))

1 0