January 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Only return the re-padded base64 string if no errors were raised.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit c5dc0a3e32169a65e72709e30af2b4ab94ac833d Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 11 13:21:48 2013 +0000 Only return the re-padded base64 string if no errors were raised. * CHANGE parse.padBase64() to only return a value if no exceptions were raised during parsing. --- lib/bridgedb/parse/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/parse/__init__.py b/lib/bridgedb/parse/__init__.py index 21cfe30..f4b75d1 100644 --- a/lib/bridgedb/parse/__init__.py +++ b/lib/bridgedb/parse/__init__.py @@ -48,5 +48,5 @@ def padBase64(b64string): if not addchars: raise ValueError("Invalid base64-encoded string: %r" % b64string) b64string += '=' * addchars - finally: - return b64string + + return b64string

1 0

[bridgedb/master] Merge branch 'fix/9462-refactor-netstatus-parsers_r9462C_r2' into develop
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 2fd3aca79ca1f0fd6fcfc18952e6c79d38c5acfa Merge: d0ea228 454bd12 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Dec 12 06:16:33 2013 +0000 Merge branch 'fix/9462-refactor-netstatus-parsers_r9462C_r2' into develop lib/bridgedb/Bridges.py | 43 +- lib/bridgedb/Dist.py | 4 + lib/bridgedb/parse/__init__.py | 55 ++- lib/bridgedb/parse/addr.py | 275 ++++++++--- lib/bridgedb/parse/networkstatus.py | 182 ++++---- lib/bridgedb/test/test_parse_addr.py | 615 +++++++++++++++++++++++++ lib/bridgedb/test/test_parse_networkstatus.py | 389 ++++++++++++++-- 7 files changed, 1363 insertions(+), 200 deletions(-)

1 0

[bridgedb/master] Change parse.networkstatus unittest values to be clearer when decoded.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit bc993d3e0cae32ddd11139aa8a81ec77715c9034 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 11 13:26:56 2013 +0000 Change parse.networkstatus unittest values to be clearer when decoded. Store the pre-base64-encoded value so that the checks are clearer if they fail. Also, remove a str() around a couple `nick` values, they do not make a difference now that the proper __future__ imports are in place. --- lib/bridgedb/test/test_parse_networkstatus.py | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/bridgedb/test/test_parse_networkstatus.py b/lib/bridgedb/test/test_parse_networkstatus.py index 9b17462..b310939 100644 --- a/lib/bridgedb/test/test_parse_networkstatus.py +++ b/lib/bridgedb/test/test_parse_networkstatus.py @@ -42,6 +42,9 @@ class ParseNetworkStatusRLineTests(unittest.TestCase): The documentation for all class variables, e.g. 'pre' or 'ident', refers to what said value should be in a *valid* descriptor. """ + rawIdent = 'identdigestidentdig' + rawDesc = 'descdigestdescdiges' + #: The prefix for the 'r'-line. Should be an 'r', unless testing that #: lines with unknown prefixes are dropped. pre = 'r ' @@ -51,12 +54,12 @@ class ParseNetworkStatusRLineTests(unittest.TestCase): #: A base64-encoded, SHA-1 digest of the DER-formatted, ASN.1-encoded, #: public portion of an OR identity key, with any trailing base64 padding #: (any '=' characters) removed. - ident = 'bXw2N1K9AAKR5undPaTgNUySN' + ident = binascii.b2a_base64(rawIdent).strip().rstrip('==') #: A base64-encoded, SHA-1 digest of the OR #: `@type-[bridge-]server-descriptor` document (the whole thing, up until #: the 'router signature' line, but not including the signature thereafter), #: with any trailing base64 padding (any '=' characters) removed. - desc = 'Z6cisoPT9s6hEd4JkHFAlIWAw' + desc = binascii.b2a_base64(rawDesc).strip().rstrip('==') #: An ISO-8661 formatted timestamp, with a space separator (rather than a #: 'T' character). ts = '2013-10-31 15:15:15' @@ -168,7 +171,7 @@ class ParseNetworkStatusRLineTests(unittest.TestCase): the(nick).should.be.ok the(nick).should.be.a(basestring) - the(nick).should.be.equal(str(self.nick)) + the(nick).should.be.equal(self.nick) the(ident).should.be.equal(None) the(desc).should.be.equal(None) @@ -180,27 +183,27 @@ class ParseNetworkStatusRLineTests(unittest.TestCase): the(nick).should.be.ok the(nick).should.be.a(basestring) - the(nick).should.be.equal(str(self.nick)) + the(nick).should.be.equal(self.nick) the(ident).should.be.a(basestring) - the(ident).should.be.equal(self.ident) + the(ident).should.be.equal(self.rawIdent) the(desc).should.be.equal(None) the(ts).should.be.equal(None) the(ip).should.be.equal(None) - def test_invalidDescriptorDigest_missingBase64padding(self): + def test_invalidDescriptorDigest_withBase64padding(self): """Test a line with invalid base64 (no padding) descriptor digest.""" - self.makeRLine(desc=self.desc.rstrip('==')) + self.makeRLine(desc=self.desc + '==') fields = networkstatus.parseRLine(self.line) nick, ident, desc, ts, ip = fields[:5] the(nick).should.be.ok the(nick).should.be.a(basestring) - the(nick).should.be.equal(str(self.nick)) + the(nick).should.be.equal(self.nick) the(ident).should.be.a(basestring) - the(ident).should.be.equal(self.ident) + the(ident).should.be.equal(self.rawIdent) the(desc).should.be.equal(None) the(ts).should.be.equal(None) @@ -242,14 +245,11 @@ class ParseNetworkStatusRLineTests(unittest.TestCase): nick, ident, desc, ts, ip, port = fields[:6] the(nick).should.be.equal(self.nick) + the(ident).should.be.equal(self.rawIdent) - the(ident).should.be.a(basestring) - the(ident).should.be.equal(self.ident) - - the(desc).should.be.a(basestring) - the(b64desc).should.be.equal(self.desc) - - the(ts).should.be.a(float) + # descDigest is set to `None` if there is an error while parsing: + the(desc).should.be(None) + the(ts).should.be(None) the(ip).should.be(None)

1 0

[bridgedb/master] Merge branch 'fix/10333-index-by-0L_r2' into fix/9462-refactor-netstatus-parsers_r9462C_r2
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 8581578400200fc0efba1095436ed0a3ce98e145 Merge: 3817ac0 63b5a6e Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 11 04:38:21 2013 +0000 Merge branch 'fix/10333-index-by-0L_r2' into fix/9462-refactor-netstatus-parsers_r9462C_r2 lib/bridgedb/test/test_parse_addr.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-)

1 0

[bridgedb/master] Add three new unittests for parseRLine() base64 encoding issues.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 454bd127cc40da6cc4fbdbf0a1b827b28fad9dd8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 11 14:47:40 2013 +0000 Add three new unittests for parseRLine() base64 encoding issues. --- lib/bridgedb/test/test_parse_networkstatus.py | 56 +++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/lib/bridgedb/test/test_parse_networkstatus.py b/lib/bridgedb/test/test_parse_networkstatus.py index b310939..ddeacdf 100644 --- a/lib/bridgedb/test/test_parse_networkstatus.py +++ b/lib/bridgedb/test/test_parse_networkstatus.py @@ -175,6 +175,18 @@ class ParseNetworkStatusRLineTests(unittest.TestCase): the(ident).should.be.equal(None) the(desc).should.be.equal(None) + def test_invalidIdent_withBase64padding(self): + """Test a line with invalid base64 (no padding) identity digest.""" + self.makeRLine(ident=self.ident + '==') + fields = networkstatus.parseRLine(self.line) + nick, ident, desc = fields[:3] + + the(nick).should.be.ok + the(nick).should.be.a(basestring) + the(nick).should.be.equal(self.nick) + the(ident).should.be.equal(None) + the(desc).should.be.equal(None) + def test_invalidDescriptorDigest(self): """Test an 'r'-line with invalid base64 descriptor digest.""" self.makeRLine(desc='敃噸襶') @@ -192,6 +204,18 @@ class ParseNetworkStatusRLineTests(unittest.TestCase): the(ts).should.be.equal(None) the(ip).should.be.equal(None) + def test_invalidDescriptorDigest_invalidBase64(self): + """Test line with '%$>@,<' for an identity digest.""" + self.makeRLine(desc='%$>#@,<') + fields = networkstatus.parseRLine(self.line) + nick, ident, desc, ts, ip = fields[:5] + + the(nick).should.be.ok + the(nick).should.be.a(basestring) + the(nick).should.equal(self.nick) + the(ident).should.be(None) + the(desc).should.be(None) + def test_invalidDescriptorDigest_withBase64padding(self): """Test a line with invalid base64 (no padding) descriptor digest.""" self.makeRLine(desc=self.desc + '==') @@ -209,6 +233,18 @@ class ParseNetworkStatusRLineTests(unittest.TestCase): the(ts).should.be.equal(None) the(ip).should.be.equal(None) + def test_invalidDescriptorDigest_singleQuoteChar(self): + """Test with a single quote character for the descriptor digest.""" + self.makeRLine(desc=chr(0x27)) + fields = networkstatus.parseRLine(self.line) + nick, ident, desc = fields[:3] + + the(nick).should.be.ok + the(nick).should.be.a(basestring) + the(nick).should.be.equal(self.nick) + the(ident).should.be.equal(self.rawIdent) + the(desc).should.be.equal(None) + def test_missingAfterDesc(self): """Test a line that has a valid descriptor digest, and is missing everything after the descriptor digest, i.e. the timestamp, IP address, @@ -231,6 +267,15 @@ class ParseNetworkStatusRLineTests(unittest.TestCase): """Test line with two large integers for the timestamp.""" self.makeRLine(ts='123456789 987654321') fields = networkstatus.parseRLine(self.line) + nick, ident, desc, ts, ip, port = fields[:6] + + the(nick).should.be.equal(self.nick) + the(ident).should.be.equal(self.rawIdent) + + # descDigest is set to `None` if there is an error while parsing: + the(desc).should.be(None) + the(ts).should.be(None) + the(ip).should.be(None) def test_invalidTimestampMissingDate(self): """Test a line where the timestamp is missing the date portion.""" @@ -252,6 +297,17 @@ class ParseNetworkStatusRLineTests(unittest.TestCase): the(ts).should.be(None) the(ip).should.be(None) + def test_valid(self): + """Test a valid 'r'-line.""" + self.makeRLine() + fields = networkstatus.parseRLine(self.line) + nick, ident, desc, ts, ip, port = fields[:6] + the(nick).should.be.equal(self.nick) + the(ident).should.be.equal(self.rawIdent) + the(desc).should.be.equal(self.rawDesc) + the(ts).should.be.ok + the(ip).should.be.ok + class ParseNetworkStatusALineTests(unittest.TestCase): """Tests for :func:`bridgedb.parse.networkstatus.parseALine`.

1 0

[bridgedb/master] Move base64 re-padding logic into separate function.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 2a3c688715d4ac4dd5ad1dae46be14ed65623d5b Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 11 13:23:15 2013 +0000 Move base64 re-padding logic into separate function. This logic is not specific to parsing networkstatus descriptors, so for now I've placed it in parse.parseUnpaddedBase64() ― though it could live somewhere else later. --- lib/bridgedb/parse/__init__.py | 38 +++++++++++++++++++++++++++++++++++ lib/bridgedb/parse/networkstatus.py | 28 ++++++++++---------------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/lib/bridgedb/parse/__init__.py b/lib/bridgedb/parse/__init__.py index f4b75d1..0618dae 100644 --- a/lib/bridgedb/parse/__init__.py +++ b/lib/bridgedb/parse/__init__.py @@ -26,6 +26,16 @@ \__ :mod:`bridgedb.parse.versions` """ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import binascii + + +class InvalidBase64(ValueError): + """Cannot decode base64 value.""" + def padBase64(b64string): """Re-add any stripped equals sign character padding to a b64 string. @@ -50,3 +60,31 @@ def padBase64(b64string): b64string += '=' * addchars return b64string + +def parseUnpaddedBase64(field): + """Parse an unpadded, base64-encoded field. + + The **field** will be re-padded, if need be, and then base64 decoded. + + :param str field: Should be some base64-encoded thing, with any trailing + '=' characters removed. + :raises: :exc:`InvalidBase64`, if there is an error in either unpadding or + decoding **field**. + :rtype: str + :returns: The base64-decoded **field**. + """ + if field.endswith('='): + raise InvalidBase64("Unpadded, base64-encoded networkstatus field "\ + "must not end with '=': %r" % field) + + try: + paddedField = padBase64(field) # Add the trailing equals sign back in + except ValueError as error: + raise InvalidBase64(error) + + debasedField = binascii.a2b_base64(paddedField) + if not debasedField: + raise InvalidBase64("Base64-encoded networkstatus field %r is invalid!" + % field) + + return debasedField diff --git a/lib/bridgedb/parse/networkstatus.py b/lib/bridgedb/parse/networkstatus.py index 7fd9e18..09c9a65 100644 --- a/lib/bridgedb/parse/networkstatus.py +++ b/lib/bridgedb/parse/networkstatus.py @@ -32,7 +32,8 @@ import warnings from twisted.python.log import showwarning from bridgedb.parse import addr -from bridgedb.parse import padBase64 +from bridgedb.parse import parseUnpaddedBase64 +from bridgedb.parse import InvalidBase64 class NetworkstatusParsingError(Exception): @@ -103,19 +104,10 @@ def parseRLine(line): nickname, ID = fields[:2] isValidRouterNickname(nickname) - if ID.endswith('='): - raise InvalidNetworkstatusRouterIdentity( - "Skipping networkstatus parsing for router with nickname "\ - "'%s':\n Unpadded, base64-encoded networkstatus router identity "\ - "string ends with '=': %r" % (nickname, ID)) - paddedID = padBase64(ID) # Add the trailing equals sign back in - debasedID = binascii.a2b_base64(paddedID) - if not debasedID: - raise InvalidNetworkstatusRouterIdentity( - "Skipping networkstatus parsing for router with nickname "\ - "'%s':\n Base64-encoding for networkstatus router identity "\ - "string is invalid!\n Line: %r" % (nickname, line)) - ID = debasedID + try: + ID = parseUnpaddedBase64(ID) + except InvalidBase64 as error: + raise InvalidNetworkstatusRouterIdentity(error) except NetworkstatusParsingError as error: logging.error(error) @@ -128,16 +120,18 @@ def parseRLine(line): ID = None else: try: - paddedDigest = padBase64(fields[2]) - descDigest = binascii.b2a_base64(paddedDigest) + descDigest = parseUnpaddedBase64(fields[2]) timestamp = time.mktime(time.strptime(" ".join(fields[3:5]), "%Y-%m-%d %H:%M:%S")) ORaddr = fields[5] ORport = fields[6] dirport = fields[7] - except (AttributeError, ValueError, IndexError) as error: + except InvalidBase64 as error: logging.error(error) descDigest = None + except (AttributeError, ValueError, IndexError) as error: + logging.error(error) + timestamp = None finally: return (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport)

1 0

[bridgedb/master] Raise ValueError rather than logging in parse.padBase64().
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit e2ff03d8680c5298ee86f845f5f3ff3eb74bf87c Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 11 10:37:48 2013 +0000 Raise ValueError rather than logging in parse.padBase64(). --- lib/bridgedb/parse/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/bridgedb/parse/__init__.py b/lib/bridgedb/parse/__init__.py index 1fdbd8a..21cfe30 100644 --- a/lib/bridgedb/parse/__init__.py +++ b/lib/bridgedb/parse/__init__.py @@ -32,17 +32,20 @@ def padBase64(b64string): :param string b64string: A base64-encoded string which might have had its trailing equals sign padding removed. + :raises: :exc:`ValueError` if there was any error while manipulating the + string. + :returns: A properly-padded (according to base64) string. """ + addchars = 0 try: b64string = b64string.strip() - except AttributeError: - logging.error("Cannot pad base64 string %r: not a string." % b64string) - else: - addchars = 0 remainder = len(b64string) % 4 if 2 <= remainder <= 3: addchars = 4 - remainder - else: + except AttributeError as error: + raise ValueError(error) + else: + if not addchars: raise ValueError("Invalid base64-encoded string: %r" % b64string) b64string += '=' * addchars finally:

1 0

[bridgedb/master] Tor expects the PT argument separator to be a comma
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 5db4f966b2afcdaf61cd902e26a94d2570fa4da7 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sat Oct 5 16:28:27 2013 +0000 Tor expects the PT argument separator to be a comma --- lib/bridgedb/Bridges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 57f7a03..e6cf210 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -405,7 +405,7 @@ class PluggableTransport: host = "%s %s:%d" % (self.methodname, address, self.port) fp = '' if includeFingerprint: fp = "keyid=%s" % self.bridge.fingerprint - args = " ".join(["%s=%s"%(k,v) for k,v in self.argdict.items()]).strip() + args = ",".join(["%s=%s"%(k,v) for k,v in self.argdict.items()]).strip() return "%s %s %s" % (host, fp, args) def parseExtraInfoFile(f):

1 0

[bridgedb/master] Merge branch 'fix/9013_r1' into develop
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 8fd5e5c2cd028c29fe827dcb5f7e08c9ee43da3a Merge: 2fd3aca 5db4f96 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 13 04:10:26 2013 +0000 Merge branch 'fix/9013_r1' into develop lib/bridgedb/Bridges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[bridgedb/master] Minor grammatical and clarifying corrections
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 8e793ee3db63537d814438ed193c2ba8ce0d9cc0 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Thu Aug 15 03:30:12 2013 +0000 Minor grammatical and clarifying corrections --- bridge-db-spec.txt | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bridge-db-spec.txt b/bridge-db-spec.txt index a90b934..83a6d7e 100644 --- a/bridge-db-spec.txt +++ b/bridge-db-spec.txt @@ -12,7 +12,7 @@ requests. Some of the decisions here may be suboptimal: this document is meant to - specify current behavior as of April 2011, not to specify ideal + specify current behavior as of August 2013, not to specify ideal behavior. 1. Importing bridge network statuses and bridge descriptors @@ -30,11 +30,12 @@ 1.1. Parsing bridge network statuses - Bridge network status documents contain the information which bridges + Bridge network status documents contain the information of which bridges are known to the bridge authority and which flags the bridge authority assigns to them. We expect bridge network statuses to contain at least the following two - lines for every bridge in the given order: + lines for every bridge in the given order (format fully specified in Tor's + directory protocol): "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort SP DirPort NL @@ -59,7 +60,7 @@ bridge descriptor in the future. Bridge descriptor files may contain one or more bridge descriptors. - We expect bridge descriptor to contain at least the following lines in + We expect a bridge descriptor to contain at least the following lines in the stated order: "@purpose" SP purpose NL @@ -128,8 +129,8 @@ BridgeDB fixes the set of bridges to be returned for a defined time period. - BridgeDB considers two IP addresses coming from the same /24 as the - same IP address and return the same set of bridges. + BridgeDB considers all IP addresses coming from the same /24 as the + same IP address and returns the same set of bridges. BridgeDB divides the IP address space equally into a small number of areas (typically 4) and return different results to requests coming from these areas. @@ -201,6 +202,7 @@ value "pass" in their X-DKIM-Authentication-Result header or does not have this header. The X-DKIM-Authentication-Result header is set by the incoming mail stack that needs to check DKIM authentication. + BridgeDB does not return a new set of bridges to the same email address until a given time period (typically a few hours) has passed. # Why don't we fix the bridges we give out for a global 3-hour time period

1 0