February 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Only add bridges for which their documents were verified by the BA
by isis＠torproject.org 21 Feb '14

21 Feb '14

commit 967f1a6a4290154977ab7c0cc2cbe0560d7246ce Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sun Feb 2 03:04:03 2014 +0000 Only add bridges for which their documents were verified by the BA --- lib/bridgedb/Bridges.py | 14 +++++----- lib/bridgedb/Main.py | 69 +++++++++++++++++++++-------------------------- 2 files changed, 37 insertions(+), 46 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 1a2bdd6..a6ca14f 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -473,9 +473,7 @@ def parseDescFile(f, bridge_purpose='bridge'): elif line.startswith("router-signature"): purposeMatches = (purpose == bridge_purpose or bridge_purpose is None) if purposeMatches and nickname and ip and orport and fingerprint: - b = Bridge(nickname, ipaddr.IPAddress(ip), orport, fingerprint) - b.assertOK() - yield b + yield (nickname, ipaddr.IPAddress(ip), orport, fingerprint) nickname = ip = orport = fingerprint = purpose = None @@ -637,9 +635,10 @@ def parseStatusFile(networkstatusFile): if line.startswith("r "): (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport) = networkstatus.parseRLine(line) + hexID = toHex(ID) logging.debug("Parsed networkstatus line:") logging.debug(" Nickname: %s" % nickname) - logging.debug(" Identity: %s" % toHex(ID)) + logging.debug(" Identity: %s" % hexID) if descDigest: logging.debug(" Descriptor: {0}".format(toHex(descDigest))) logging.debug(" Timestamp: {0}".format(timestamp)) @@ -667,14 +666,15 @@ def parseStatusFile(networkstatusFile): "yielding %s nickname=%s descDigest=%s " "running=%s stable=%s oraddr=%s orport=%s " "oraddrs=%s ts=%s" - % (toHex(ID), nickname, descDigest, running, + % (hexID, nickname, descDigest, running, stable, ORaddr, ORport, or_addresses, timestamp)) - yield (ID, nickname, descDigest, running, stable, ORaddr, ORport, + yield (ID, nickname, descDigest, running, stable, + ipaddr.IPAddress(ORaddr), ORport, or_addresses, timestamp) (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport, - addr, portlist) = (None for x in xrange(9)) + addr, portlist, hexID) = (None for x in xrange(10)) running = stable = False or_addresses = {} diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index c3a90ea..36e4e88 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -95,66 +95,57 @@ def load(state, splitter, clear=False): status = {} addresses = {} timestamps = {} + bridges = {} + desc_digests = {} + ei_digests = {} logging.info("Opening network status file: %s" % state.STATUS_FILE) f = open(state.STATUS_FILE, 'r') - for (ID, nickname, descDigest, running, stable, + for (ID, nickname, desc_digest, running, stable, ORaddr, ORport, or_addresses, timestamp) in Bridges.parseStatusFile(f): - - status[ID] = running, stable - addresses[ID] = or_addresses + bridge = Bridges.Bridge(nickname, ORaddr, ORport, id_digest=ID, + or_addresses=or_addresses) + bridge.assertOK() + bridge.setStatus(running, stable) + bridge.setDescriptorDigest(desc_digest) + bridges[ID] = bridge if ID in timestamps.keys(): timestamps[ID].append(timestamp) else: timestamps[ID] = [timestamp] - #transports[ID] = transports logging.debug("Closing network status file") f.close() - bridges = {} db = bridgedb.Storage.getDB() for fname in state.BRIDGE_FILES: logging.info("Opening bridge-server-descriptor file: '%s'" % fname) f = open(fname, 'r') - for bridge in Bridges.parseDescFile(f, state.BRIDGE_PURPOSE): - if bridge.getID() in bridges: - logging.warn("Parsed a duplicate bridge. Skipping.") - continue - else: - bridges[bridge.getID()] = bridge - s = status.get(bridge.getID()) - if s is not None: - running, stable = s - bridge.setStatus(running=running, stable=stable) - # XXX: what do we do with all these or_addresses? - # - # The bridge stability metrics are only concerned with a - # single ip:port So for now, we will only consider the bridges - # primary IP:port - bridge.or_addresses = addresses.get(bridge.getID()) - # We attempt to insert all bridges. If the bridge is not - # running, then it is skipped during the insertion process. Also, - # if we have a descriptor for the bridge but it was not in the - # ns, then we skip it there, too. - splitter.insert(bridge) - - if state.COLLECT_TIMESTAMPS: - if bridge.getID() in timestamps.keys(): - ts = timestamps[bridge.getID()][:] - ts.sort() - for timestamp in ts: - logging.debug( - "Updating BridgeHistory timestamps for %s: %s" - % (bridge.fingerprint, timestamp)) - bridgedb.Stability.addOrUpdateBridgeHistory( - bridge, timestamp) - + desc_digests.update(Bridges.getDescriptorDigests(f)) + if bridge.getID() in timestamps.keys(): + ts = timestamps[bridge.getID()][:] + ts.sort() + for timestamp in ts: + logging.debug( + "Adding/updating timestamps in BridgeHistory for "\ + "'%s' in database: %s" + % (bridge.fingerprint, timestamp)) + bridgedb.Stability.addOrUpdateBridgeHistory( + bridge, timestamp) logging.debug("Closing bridge-server-descriptor file: '%s'" % fname) f.close() + for ID in bridges.keys(): + bridge = bridges[ID] + if bridge.desc_digest in desc_digests: + bridge.setVerified() + bridge.setExtraInfoDigest(desc_digests[bridge.desc_digest]) + # We attempt to insert all bridges. If the bridge is not + # running, then it is skipped during the insertion process. + splitter.insert(bridge) + # read pluggable transports from extra-info document # XXX: should read from networkstatus after bridge-authority # does a reachability test

1 0

[bridgedb/master] Fix another UnicodeDecodeError in bridgedb.persistent log call.
by isis＠torproject.org 21 Feb '14

21 Feb '14

commit 0e875a1f57581ebfb17dbdfd55cfab6f237de353 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Feb 9 02:31:35 2014 +0000 Fix another UnicodeDecodeError in bridgedb.persistent log call. * CHANGE patch for the same error in previous commit to use twisted.python.reflect.safe_repr for better clarity. * See https://travis-ci.org/isislovecruft/bridgedb/builds/18503773#L1483 --- lib/bridgedb/persistent.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index 84260f6..3b9803d 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -20,6 +20,7 @@ try: except (ImportError, NameError): import pickle +from twisted.python.reflect import safe_repr from twisted.spread import jelly from bridgedb import Filters, Bridges, Dist @@ -253,15 +254,15 @@ class State(jelly.Jellyable): setattr(self, key, value) updated.append(key) logging.debug("Updated %s setting: %r → %r" % - (key.decode('ascii', 'replace'), + (safe_repr(key), self.config.__dict__[key], - value)) + safe_repr(value))) except (KeyError, AttributeError): setattr(self, key, value) new.append(key) logging.debug("New setting: %s = %r" % - (key.decode('ascii', 'replace'), - value)) + (safe_repr(key), + safe_repr(value))) logging.info("Updated setting(s): %s" % ' '.join([x for x in updated])) logging.info("New setting(s): %s" % ' '.join([x for x in new]))

1 0

[bridgedb/master] Merge branch 'hotfix/ude-persistent' into testing/tpo-sysrqb/bug9264_rebased_1
by isis＠torproject.org 21 Feb '14

21 Feb '14

commit 59ec642fcc1df7b9a68b0856b1d45c672844952a Merge: b5c8cfb 0e875a1 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Feb 9 05:31:11 2014 +0000 Merge branch 'hotfix/ude-persistent' into testing/tpo-sysrqb/bug9264_rebased_1 lib/bridgedb/persistent.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)

1 0

[bridgedb/master] We should be consistant and use lowercase
by isis＠torproject.org 21 Feb '14

21 Feb '14

commit b502658ff210630328e56c142368f4b44357a205 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sun Feb 2 05:26:16 2014 +0000 We should be consistant and use lowercase --- lib/bridgedb/Bridges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index a6ca14f..e8201b5 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -469,7 +469,7 @@ def parseDescFile(f, bridge_purpose='bridge'): ip = items[2].strip('[]') orport = int(items[3]) elif line.startswith("fingerprint "): - fingerprint = line[12:].replace(" ", "") + fingerprint = line[12:].replace(" ", "").lower() elif line.startswith("router-signature"): purposeMatches = (purpose == bridge_purpose or bridge_purpose is None) if purposeMatches and nickname and ip and orport and fingerprint: @@ -635,7 +635,7 @@ def parseStatusFile(networkstatusFile): if line.startswith("r "): (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport) = networkstatus.parseRLine(line) - hexID = toHex(ID) + hexID = toHex(ID).lower() logging.debug("Parsed networkstatus line:") logging.debug(" Nickname: %s" % nickname) logging.debug(" Identity: %s" % hexID)

1 0

[bridgedb/master] Merge branch 'hotfix/ude-persistent' into develop
by isis＠torproject.org 21 Feb '14

21 Feb '14

commit bab5d960e0dc0f1565e17d3069947755213c4d10 Merge: 1a8fb99 0e875a1 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Feb 21 20:20:49 2014 +0000 Merge branch 'hotfix/ude-persistent' into develop lib/bridgedb/persistent.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)

1 0

[bridgedb/master] Revert "We should be consistant and use lowercase"
by isis＠torproject.org 21 Feb '14

21 Feb '14

commit d2a2d0c8b73399dbc7ecc0cde2bdd8f93006138c Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Feb 9 22:15:46 2014 +0000 Revert "We should be consistant and use lowercase" This reverts commit b502658ff210630328e56c142368f4b44357a205. Signed-off-by: Isis Lovecruft <isis(a)torproject.org> --- lib/bridgedb/Bridges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index f44be55..4aaf86f 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -470,7 +470,7 @@ def parseDescFile(f, bridge_purpose='bridge'): ip = items[2].strip('[]') orport = int(items[3]) elif line.startswith("fingerprint "): - fingerprint = line[12:].replace(" ", "").lower() + fingerprint = line[12:].replace(" ", "") elif line.startswith("router-signature"): purposeMatches = (purpose == bridge_purpose or bridge_purpose is None) if purposeMatches and nickname and ip and orport and fingerprint: @@ -636,7 +636,7 @@ def parseStatusFile(networkstatusFile): if line.startswith("r "): (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport) = networkstatus.parseRLine(line) - hexID = toHex(ID).lower() + hexID = toHex(ID) logging.debug("Parsed networkstatus line:") logging.debug(" Nickname: %s" % nickname) logging.debug(" Identity: %s" % hexID)

1 0

[bridgedb/master] Add some unit tests for Bridges module
by isis＠torproject.org 21 Feb '14

21 Feb '14

commit b528a3b1469fe93c427bfbc9363f2bc7f2433312 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sun Feb 2 06:59:16 2014 +0000 Add some unit tests for Bridges module --- lib/bridgedb/test/test_Bridges.py | 111 +++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/lib/bridgedb/test/test_Bridges.py b/lib/bridgedb/test/test_Bridges.py new file mode 100644 index 0000000..4e0b27e --- /dev/null +++ b/lib/bridgedb/test/test_Bridges.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: please also see AUTHORS file +# :copyright: (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: 3-Clause BSD, see LICENSE for licensing information + +"""Unittests for the :mod:`bridgedb.EmailServer` module.""" + +from twisted.trial import unittest +from binascii import a2b_hex +from bridgedb import Bridges +import hashlib +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +class BridgeClassTest(unittest.TestCase): + """Tests for :class:`bridgedb.Bridges.Bridge`.""" + + def setUp(self): + self.nickname = 'unnamed' + self.ip = '127.0.0.1' + self.orport = '9001' + self.fingerprint = 'a1cc8dfef1fa11af9c40af1054df9daf45250556' + self.id_digest = a2b_hex(self.fingerprint) + self.or_addresses = {} + + def test_init(self): + try: + Bridges.Bridge(self.nickname, self.ip, self.orport, + fingerprint=self.fingerprint) + Bridges.Bridge(self.nickname, self.ip, self.orport, + id_digest=self.id_digest) + except Exception as e: + self.fail("Unexpected %s exception." % type(e)) + + self.failUnlessRaises(TypeError, Bridges.Bridge, + self.nickname, self.ip, self.orport, + id_digest=self.id_digest[:-1]) + + self.failUnlessRaises(TypeError, Bridges.Bridge, + self.nickname, self.ip, self.orport, + fingerprint=self.fingerprint[:-1]) + + self.failUnlessRaises(TypeError, Bridges.Bridge, + self.nickname, self.ip, self.orport) + + invalid_fingerprint = self.fingerprint[:-1] + 'q' + self.failUnlessRaises(TypeError, Bridges.Bridge, self.nickname, + self.ip, self.orport, + fingerprint=invalid_fingerprint) + + def test_Bridgehelperfunctions(self): + bridge = Bridges.Bridge(self.nickname, self.ip, self.orport, + self.fingerprint) + self.assertEqual(self.id_digest, bridge.getID()) + + testtext = 'thisisatest' + bridge.setDescriptorDigest(testtext) + self.assertEqual(bridge.desc_digest, testtext) + bridge.setExtraInfoDigest(testtext) + self.assertEqual(bridge.ei_digest, testtext) + bridge.setVerified() + self.assertTrue(bridge.isVerified()) + self.assertTrue(bridge.verified) + self.assertEqual(self.id_digest, bridge.getID()) + + self.assertFalse(bridge.running) + self.assertFalse(bridge.stable) + bridge.setStatus(True, True) + self.assertTrue(bridge.running) + self.assertTrue(bridge.stable) + +# def test_isBlocked(self): + + + def test_getDescriptorDigests(self): + sha1hash = hashlib.sha1() + ei_digest = 'abcdefghijklmno' + + test = "this is a test line\nFollowed by another\n" + test += "extra-info-digest %s\n" % ei_digest + sha1hash.update(test) + digest = sha1hash.hexdigest() + test += "-----BEGIN SIGNATURE-----\n" + test += "This is a test line that should be skipped\n" + test += "-----END SIGNATURE-----\n" + digests = Bridges.getDescriptorDigests(StringIO(test)) + self.failUnlessIn(digest, digests) + self.failUnlessEqual(ei_digest, digests[digest]) + + def test_getExtraInfoDigests(self): + sha1hash = hashlib.sha1() + + test = "Many words and line all together\n" + test += "extra info is nothing like weather\n" + test += "it's certain to come, like the key in a hum\n" + test += "but sometimes without a transport and rum\n" + content = test + sha1hash.update(test) + digest = sha1hash.hexdigest() + test += "-----BEGIN SIGNATURE-----\n" + test += "But the rum can't save the world like you\n" + test += "-----END SIGNATURE-----\n" + digests = Bridges.getExtraInfoDigests(StringIO(test)) + self.failUnlessIn(digest, digests) + self.failUnlessEqual(content, digests[digest].read())

1 0

[bridgedb/master] Merge branch 'testing/tpo-sysrqb/bug9264_rebased_1-test2' into develop
by isis＠torproject.org 21 Feb '14

21 Feb '14

commit 558610e3fa321b76af03142d4aa1ab92125a43d5 Merge: bab5d96 d2a2d0c Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Feb 21 20:26:37 2014 +0000 Merge branch 'testing/tpo-sysrqb/bug9264_rebased_1-test2' into develop lib/bridgedb/Bridges.py | 149 ++++++++++++++++++++++++++++++----- lib/bridgedb/Main.py | 72 ++++++++--------- lib/bridgedb/parse/networkstatus.py | 2 +- lib/bridgedb/test/test_Bridges.py | 111 ++++++++++++++++++++++++++ 4 files changed, 275 insertions(+), 59 deletions(-)

1 0

[bridgedb/master] Merge branch 'hotfix/9264-descdigest-none' into develop
by isis＠torproject.org 21 Feb '14

21 Feb '14

commit f6ad2cf9f5571d95dc6e5c4edf3955d3504ecc25 Merge: 1b5b984 a2192c4 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Feb 21 20:59:49 2014 +0000 Merge branch 'hotfix/9264-descdigest-none' into develop lib/bridgedb/Bridges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

1 0

[bridgedb/master] Fix a typo in doc/DESCRIPTORS.md.
by isis＠torproject.org 21 Feb '14

21 Feb '14

commit 1b5b984791be2690ea40e07f1580c83cefc6d093 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Feb 21 20:27:06 2014 +0000 Fix a typo in doc/DESCRIPTORS.md. --- doc/DESCRIPTORS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/DESCRIPTORS.md b/doc/DESCRIPTORS.md index 226f368..eea8ba8 100644 --- a/doc/DESCRIPTORS.md +++ b/doc/DESCRIPTORS.md @@ -38,7 +38,7 @@ fields. They look like this: VXAN4HoclQiNWdgZF3kAdCXW+8YR/rqyYtSOaLFOxgs= -----END SIGNATURE----- -As of tor-0.2.4.15, bridge router descriptors may be missig the 'opt ' prefix, +As of tor-0.2.4.15, bridge router descriptors may be missing the 'opt ' prefix, and thus appear like this: @purpose bridge @@ -182,7 +182,7 @@ obproxy pluggable transport types: D9FLbPlXw4NWy9B32IT/luOHsENaAJNvOv7ociMPnsM= -----END SIGNATURE----- -## Bridge router microdescriptors +## Bridge router microdescriptors / bridge networkstatus documents r Unnamed /wywABJee98ZPOiCGYM1dpgQc70 NpK1tsi97A+SH8s0evowXkRcyr8 2013-10-22 01:49:45 88.200.197.4 9001 0 a [6212:b13d:252e:479d:32b8:d713:3718:2fac]:9001

1 0