March 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Validate extrainfo descriptors by default.
by isis＠torproject.org 20 Mar '15

20 Mar '15

commit babbede52344e6ddf51dea2bcc61b688a7aa8c0f Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 5 01:45:29 2014 +0000 Validate extrainfo descriptors by default. Also update the documentation of `b.p.d.parseBridgeExtraInfoFiles` to mention a method for ``router-signature`` verification, and the reasons why we use Stem's ``RelayExtraInfoDescriptor`` class, rather than the ``BridgeExtraInfoDescriptor`` class. --- lib/bridgedb/parse/descriptors.py | 29 ++++++++++++++++++++++----- lib/bridgedb/test/test_parse_descriptors.py | 4 ++-- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index a7cb202..03b3f55 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -164,22 +164,41 @@ def deduplicate(descriptors): def parseBridgeExtraInfoFiles(*filenames, **kwargs): """Parse files which contain ``@type bridge-extrainfo-descriptor``s. + .. warning:: This function will *not* check that the ``router-signature`` + at the end of the extrainfo descriptor is valid. See + ``bridgedb.bridges.Bridge._verifyExtraInfoSignature`` for a method for + checking the signature. + .. note:: This function will call :func:`deduplicate` to deduplicate the extrainfo descriptors parsed from all **filenames**. :kwargs validate: If there is a ``'validate'`` keyword argument, its value will be passed along as the ``'validate'`` argument to :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`. + The ``'validate'`` keyword argument defaults to ``True``, meaning that + the hash digest stored in the ``router-digest`` line will be checked + against the actual contents of the descriptor and the extrainfo + document's signature will be verified. :rtype: dict :returns: A dictionary mapping bridge fingerprints to deduplicated - :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`s. + :api:`stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor`s. """ descriptors = [] - descriptorType = 'bridge-extra-info 1.1' - validate = False - if ('validate' in kwargs) and (kwargs['validate'] is True): - validate = True + # The ``stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`` + # class (with ``descriptorType = 'bridge-extra-info 1.1``) is unsuitable + # for our purposes for the following reasons: + # + # 1. It expects a ``router-digest`` line, which is only present in + # sanitised bridge extrainfo descriptors. + # + # 2. It doesn't check the ``router-signature`` (nor does it expect there + # to be a signature). + descriptorType = 'extra-info 1.0' + + validate = True + if ('validate' in kwargs) and (kwargs['validate'] is False): + validate = False for filename in filenames: logging.info("Parsing %s descriptors with Stem: %s" diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 953f371..c3772c7 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -21,7 +21,7 @@ HAS_STEM = False try: from stem.descriptor.server_descriptor import RelayDescriptor - from stem.descriptor.extrainfo_descriptor import BridgeExtraInfoDescriptor + from stem.descriptor.extrainfo_descriptor import RelayExtraInfoDescriptor from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb.parse import descriptors except (ImportError, NameError), error: @@ -252,7 +252,7 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) bridge = routers.values()[0] - self.assertIsInstance(bridge, BridgeExtraInfoDescriptor) + self.assertIsInstance(bridge, RelayExtraInfoDescriptor) def test_parse_descriptors_parseBridgeExtraInfoFiles_one_file(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with only one

1 0

[bridgedb/master] Replace descriptors in test_parse_descriptors with newer leekspin ones.
by isis＠torproject.org 20 Mar '15

20 Mar '15

commit 74f8a5548fb408c64169321789f7bdb718b3996b Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 5 00:55:55 2014 +0000 Replace descriptors in test_parse_descriptors with newer leekspin ones. The descriptors in this unittest file were created by an older version of Leekspin, which had a bugs in the way it created signatures and coverted between fingerprint formats. In addition, we're now able to test parsing of obfs4 `transport` lines. --- lib/bridgedb/parse/descriptors.py | 2 +- lib/bridgedb/test/test_parse_descriptors.py | 150 +++++++++++++++------------ 2 files changed, 82 insertions(+), 70 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index 6c6fd3e..a7cb202 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -66,7 +66,7 @@ def parseNetworkStatusFile(filename, validate=True, skipAnnotations=True, return routers -def parseServerDescriptorsFile(filename, validate=False): +def parseServerDescriptorsFile(filename, validate=True): """Parse a file which contains ``@type bridge-server-descriptor``s. .. note:: ``validate`` defaults to ``False`` because there appears to be a diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 1abd08f..953f371 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -31,111 +31,114 @@ else: BRIDGE_NETWORKSTATUS_0 = '''\ -r OutwitsPlod b6khbPOgbomgMSGswx9w+N/X3cw qBckYSWbwl/F/qzQWwMJWBxHZ+w 2014-03-12 16:07:08 152.78.9.20 17810 0 -a [bfbd:7a90:2347:cc4:e854:64b3:2c31:124f]:17810 +r MiserLandfalls 4IsyTSCtChPhFPAnq5rD8yymlqA /GMC4lz8RXT/62v6kZNdmzSmopk 2014-11-04 06:23:22 2.215.61.223 4056 0 +a [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055 s Fast Guard Running Stable Valid -w Bandwidth=1902273 +w Bandwidth=1678904 p reject 1-65535 ''' BRIDGE_NETWORKSTATUS_1 = '''\ -r Reestablishes jPqMRoqkH62eLMXl76DqIddlpto 2BETKn1sOghC6coUkCSq/9mvPNM 2014-08-20 19:52:41 25.178.4.186 32324 0 -a [d7b3:8c3e:186a:d65f:706:cbfd:8512:fd1]:32324 +r Unmentionable BgOrX0ViP5hNsK5ZvixAuPZ6EY0 NTg9NoE5ls9KjF96Dp/UdrabZ9Y 2014-11-04 12:23:37 80.44.173.87 51691 0 +a [da14:7d1e:ba8e:60d0:b078:3f88:382b:5c70]:51690 s Fast Guard Running Stable Valid -w Bandwidth=497963 +w Bandwidth=24361 p reject 1-65535 ''' BRIDGE_SERVER_DESCRIPTOR = '''\ -router OutwitsPlod 152.78.9.20 17810 0 0 -or-address [bfbd:7a90:2347:cc4:e854:64b3:2c31:124f]:17810 -platform Tor 0.2.4.16-rc on Linux -protocols Link 1 2 Circuit 1 -published 2014-03-12 16:07:08 -fingerprint 6FA9 216C F3A0 6E89 A031 21AC C31F 70F8 DFD7 DDCC -uptime 57032961 -bandwidth 2240117028 2532306205 1947927850 -extra-info-digest 069EBB610CD8B02BF1BB0CAB17B99DDA73CCC91A +@purpose bridge +router MiserLandfalls 2.215.61.223 4056 0 0 +or-address [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055 +platform Tor 0.2.2.39 on Linux +opt protocols Link 1 2 Circuit 1 +published 2014-11-04 06:23:22 +opt fingerprint E08B 324D 20AD 0A13 E114 F027 AB9A C3F3 2CA6 96A0 +uptime 24247659 +bandwidth 1977077890 2234957615 1719198165 +opt extra-info-digest 1CBBB3D6158F324476E6804B7EE25623899271CB onion-key -----BEGIN RSA PUBLIC KEY----- -MIGkAgEAMA0GCSqGSIb3DQEBAQUABIGPMIGMAgEAAoGBANI67YIwW8xF2v310PZt -Qc8jm0ptwLHmgBdhAzHAIGagqknjvukX5GTL0zie5covhxrQhZqjJm/gQ8inwkol -kZCue1ZQ9PHaTWjz58ESMQo41h+9Whfd8Egm2ev1+MwqlPy1Kr3rcPNIEetsmtil -DFNocpEfq1MC0tDG6qVO6/FNAgMBAAE= +MIGJAoGBAOm4NX2wi8JmgcAyvOyiAEfq9UkzaNHK+VnSZBiPIrb5GAKFibR7S+Bb +7+x7tsT8VBNbe9QmwML2GVah3xXg68gJAksMNIgFdpud+zMhduuGd0jr7V55aLmH +ePGJYCh78B9RqfvmeTridp3pljwcAheKKH/YKi3nv1fPY0BwahurAgMBAAE= -----END RSA PUBLIC KEY----- signing-key -----BEGIN RSA PUBLIC KEY----- -MIGkAgEAMA0GCSqGSIb3DQEBAQUABIGPMIGMAgEAAoGBALk0Ws5qPlgwKO6IQ1b/ -aamtEdXEPj2DrZTF3aGYR5zoZgw9gwmkbRHjrMQ/Wj+QHg0cTFY2DsYt81QXwiv+ -m1P9sshMZSZZz2P8Ld8WqUNuN7YIIJx/fj9Vy6LRFySzoyQ4FF/1Dio+JD0rvtyc -ZyRJl2aV5iYA9/TQY2zs2cxFAgMBAAE= +MIGJAoGBANd/JkrTZRT24EkK3DDc/E+Nj1QBnKIm/xXMyW0gkotFOVdewIWjwQ5z +Tn3YbDhrFN0aFYVdVwNbRhW83e+jZDkpIQuxlQOx6bT13vrzmg8ff1tH8I9EePl7 +MO4v0DLPIEcu7Zfz90oC1bl36oqNsD4h0v4yK/XjVwLutIGiy3gTAgMBAAE= -----END RSA PUBLIC KEY----- -hidden-service-dir contact Somebody <somebody(a)example.com> +ntor-onion-key NBsk2O6ks5qnxLhhhKPd59zi0IzfjnakoOJP+Cm8OAE reject *:* router-signature -----BEGIN SIGNATURE----- -i/nkrD4VxqWcnAlBS48hIilrE7C4DvRJhN4XWep7TXNbEC48IqFG+49xpKV6qkts -yKaUDBfD9Y1tMM0mrRjEWK0xYWX/4Ug9Xbbv2q1so4EuS35AF11d69Yf/2ppnCu7 -r+qtX7csROF4KyFJYFNJUKf/hroPHKWuTGCcqzb+D68= +YYA5wJTHcjqXk/QBaDXHX/4Fb8W2OctF4X4VHyxH9Hsou4Ip7nzdfWzbBTcBiIrt +ybaaMO15L9Ctkli/capN+nCw2jWgivgiPnAmJNmLGeN6skTKjLPAau+839hBuQxu +P2aB/+XQfzFBA5TaWF83coDng4OGodhwHaOx10Kn7Bg= -----END SIGNATURE----- ''' BRIDGE_EXTRA_INFO_DESCRIPTOR = '''\ -extra-info OutwitsPlod 6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC -published 2014-03-12 16:07:08 -write-history 2014-03-12 16:07:08 (900 s) 3188736,2226176,2866176 -read-history 2014-03-12 16:07:08 (900 s) 3891200,2483200,2698240 -dirreq-write-history 2014-03-12 16:07:08 (900 s) 1024,0,2048 -dirreq-read-history 2014-03-12 16:07:08 (900 s) 0,0,0 -geoip-db-digest AAF7B842E52974556F9969A62DF8D31F9D886A33 -geoip6-db-digest C2C80F4EF2908E55A764603B08A8CB99A681EA19 -dirreq-stats-end 2014-03-12 16:07:08 (86400 s) +extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 +published 2014-11-04 06:23:22 +write-history 2014-11-04 06:23:22 (900 s) 3188736,2226176,2866176 +read-history 2014-11-04 06:23:22 (900 s) 3891200,2483200,2698240 +dirreq-write-history 2014-11-04 06:23:22 (900 s) 1024,0,2048 +dirreq-read-history 2014-11-04 06:23:22 (900 s) 0,0,0 +geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F +geoip6-db-digest E983833985E4BCA34CEF611B2DF51942D188E638 +dirreq-stats-end 2014-11-04 06:23:22 (86400 s) dirreq-v3-ips dirreq-v3-reqs dirreq-v3-resp ok=16,not-enough-sigs=0,unavailable=0,not-found=0,not-modified=0,busy=0 dirreq-v3-direct-dl complete=0,timeout=0,running=0 dirreq-v3-tunneled-dl complete=12,timeout=0,running=0 -transport obfs3 152.78.9.20:17811 -transport obfs2 152.78.9.20:17812 -bridge-stats-end 2014-03-12 16:07:08 (86400 s) +transport obfs3 2.215.61.223:4057 +transport obfs2 2.215.61.223:4058 +transport scramblesuit 2.215.61.223:4059 password=ABCDEFGHIJKLMNOPQRSTUVWXYZ234567 +transport obfs4 2.215.61.223:4060 iat-mode=0,node-id=19a448c01aa2e7d55979473b647e282459995b85,public-key=7a61b53701befdae0eeeffaecc73f14e20b537bb0f8b91ad7c2936dc63562b25 +bridge-stats-end 2014-11-04 06:23:22 (86400 s) bridge-ips ca=8 bridge-ip-versions v4=8,v6=0 bridge-ip-transports <OR>=8 router-signature -----BEGIN SIGNATURE----- -aW1ZlxqeaGcbNSxVpMONU0ER4xgdihb9X2crguzKa/TYVweCZ2Ew7x3Rsg4cUNpr -Fb05F3Zxg6ZUTMC8gfh6leDGw5eSX7OGVaaJICTfeLbNopLVk+JKNGMJ32R/Zia0 -feWndKJk/zj5ZtkMND8VVbWuJE+R6Jh2Q3L0p8IZ6J4= +KOXNPCoe+Q+thFA/Lz7RTja2tWp4oC6SvyIooEZibHtEDgiXuU4sELWT4bSOk3np +RVmu7QPMmNybx4LHowq3pOeNLtJzpWg8Pfo+N6tR+K4nqPwBRmpsuDhCD/tIXJlP +U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE = '''\ -extra-info OutwitsPlod 6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC -published 2014-03-12 17:07:08 -write-history 2014-03-12 17:07:08 (900 s) 3188736,2226176,2866176 -read-history 2014-03-12 17:07:08 (900 s) 3891200,2483200,2698240 -dirreq-write-history 2014-03-12 17:07:08 (900 s) 1024,0,2048 -dirreq-read-history 2014-03-12 17:07:08 (900 s) 0,0,0 -geoip-db-digest AAF7B842E52974556F9969A62DF8D31F9D886A33 -geoip6-db-digest C2C80F4EF2908E55A764603B08A8CB99A681EA19 -dirreq-stats-end 2014-03-12 17:07:08 (86400 s) +extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 +published 2014-11-04 08:10:25 +write-history 2014-11-04 08:10:25 (900 s) 3188736,2226176,2866176,2226176 +read-history 2014-11-04 08:10:25 (900 s) 3891200,2483200,2698240,2483200 +dirreq-write-history 2014-11-04 08:10:25 (900 s) 1024,0,2048,3072 +dirreq-read-history 2014-11-04 08:10:25 (900 s) 0,0,0,0 +geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F +geoip6-db-digest E983833985E4BCA34CEF611B2DF51942D188E638 +dirreq-stats-end 2014-11-04 08:10:25 (86400 s) dirreq-v3-ips dirreq-v3-reqs dirreq-v3-resp ok=16,not-enough-sigs=0,unavailable=0,not-found=0,not-modified=0,busy=0 dirreq-v3-direct-dl complete=0,timeout=0,running=0 dirreq-v3-tunneled-dl complete=12,timeout=0,running=0 -transport obfs3 152.78.9.20:17811 -transport obfs2 152.78.9.20:17812 -bridge-stats-end 2014-03-12 17:07:08 (86400 s) +transport obfs3 2.215.61.223:4057 +transport obfs2 2.215.61.223:4058 +transport scramblesuit 2.215.61.223:4059 password=ABCDEFGHIJKLMNOPQRSTUVWXYZ234567 +transport obfs4 2.215.61.223:4060 iat-mode=0,node-id=19a448c01aa2e7d55979473b647e282459995b85,public-key=7a61b53701befdae0eeeffaecc73f14e20b537bb0f8b91ad7c2936dc63562b25 +bridge-stats-end 2014-11-04 08:10:25 (86400 s) bridge-ips ca=8 bridge-ip-versions v4=8,v6=0 bridge-ip-transports <OR>=8 router-signature -----BEGIN SIGNATURE----- -aW1ZlxqeaGcbNSxVpMONU0ER4xgdihb9X2crguzKa/TYVweCZ2Ew7x3Rsg4cUNpr -Fb05F3Zxg6ZUTMC8gfh6leDGw5eSX7OGVaaJICTfeLbNopLVk+JKNGMJ32R/Zia0 -feWndKJk/zj5ZtkMND8VVbWuJE+R6Jh2Q3L0p8IZ6J4= +KOXNPCoe+Q+thFA/Lz7RTja2tWp4oC6SvyIooEZibHtEDgiXuU4sELWT4bSOk3np +RVmu7QPMmNybx4LHowq3pOeNLtJzpWg8Pfo+N6tR+K4nqPwBRmpsuDhCD/tIXJlP +U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' @@ -147,6 +150,11 @@ class ParseDescriptorsTests(unittest.TestCase): def setUp(self): """Test if we have Stem installed. Skip these tests if it's missing.""" + self.expectedIPBridge0 = '2.215.61.223' + self.expectedIPBridge1 = '80.44.173.87' + + self.expectedFprBridge0 = 'E08B324D20AD0A13E114F027AB9AC3F32CA696A0' + if self.skip: raise unittest.SkipTest("Couldn't import Stem.") @@ -174,9 +182,8 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertIsInstance(routers, list) bridge = routers[0] self.assertIsInstance(bridge, RelayDescriptor) - self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + self.assertEqual(bridge.address, self.expectedIPBridge0) + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_parseBridgeNetworkStatusFile_return_type(self): """``b.p.descriptors.parseNetworkStatusFile`` should return a dict.""" @@ -210,14 +217,15 @@ class ParseDescriptorsTests(unittest.TestCase): BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] - self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + self.assertEqual(bridge.address, self.expectedIPBridge0) + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_parseBridgeNetworkStatusFile_2(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with two bridge networkstatus descriptors. """ + expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] + # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', @@ -225,9 +233,9 @@ class ParseDescriptorsTests(unittest.TestCase): BRIDGE_NETWORKSTATUS_1) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] - self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + + self.assertIn(bridge.address, expectedIPs) + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_parseBridgeExtraInfoFiles_return_type(self): """The return type of ``b.p.descriptors.parseBridgeExtraInfoFiles`` @@ -253,9 +261,13 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) bridge = routers.values()[0] - self.assertEqual(len(bridge.transport), 2) - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + + # The number of transports we parsed should be equal to the number of + # 'transport' lines in the descriptor: + self.assertEqual(len(bridge.transport), + BRIDGE_EXTRA_INFO_DESCRIPTOR.count('transport ')) + + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_parseBridgeExtraInfoFiles_two_files(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with two

1 0

[bridgedb/master] Add error handling for cases when Stem cannot parse an extrainfo file.
by isis＠torproject.org 20 Mar '15

20 Mar '15

commit 1f19568e61142521ddaccbc92a461ce831ecb255 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 5 02:35:51 2014 +0000 Add error handling for cases when Stem cannot parse an extrainfo file. If a file containing bridge-extrainfo descriptors cannot be parsed the first time, and validation was enabled, then we parse it again with Stem's validation functionality disabled. If we still can't parse the file, we save a timestamped copy of the file for later debugging an continue with the next file. --- lib/bridgedb/parse/descriptors.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index 63d9bf2..6eac164 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -238,10 +238,38 @@ def parseBridgeExtraInfoFiles(*filenames, **kwargs): validate = False for filename in filenames: + document = None + documentWasUnparseable = False + logging.info("Parsing %s descriptors with Stem: %s" % (descriptorType, filename)) - document = parse_file(filename, descriptorType, validate=validate) - descriptors.extend([router for router in document]) + try: + document = parse_file(filename, descriptorType, validate=validate) + except ValueError as error: + documentWasUnparseable = True + + if documentWasUnparseable: + logging.warn(("Stem ran into an exception while parsing extrainfo " + "file '%s'!") % filename) + logging.debug("Error while parsing extrainfo file:\n%s" + % str(error)) + + if documentWasUnparseable and (validate is True): + logging.info(("Retrying parsing of extrainfo file '%s' with " + "validation disabled...") % filename) + try: + document = parse_file(filename, descriptorType, validate=False) + except ValueError as another: + logging.critical(("We were still unable to parse extrainfo " + "file on the second attempt! Bailing!")) + else: + documentWasUnparseable = False + + if documentWasUnparseable: + _copyUnparseableDescriptorFile(filename) + + if document: + descriptors.extend([router for router in document]) routers = deduplicate(descriptors) return routers

1 0

[bridgedb/master] Add a utility function for saving copies of unparsable descriptor files.
by isis＠torproject.org 20 Mar '15

20 Mar '15

commit 905eeee999d5be4f68c5fb323e809c756d1d828f Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 5 02:30:55 2014 +0000 Add a utility function for saving copies of unparsable descriptor files. * ADD new function, ``b.p.descriptors._copyUnparseableDescriptorFile``, for saving descriptor files that were unparseable for later debugging. --- lib/bridgedb/parse/descriptors.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index 03b3f55..63d9bf2 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -14,6 +14,8 @@ from __future__ import print_function import datetime import logging +import os +import shutil from stem.descriptor import extrainfo_descriptor from stem.descriptor import server_descriptor @@ -24,6 +26,41 @@ from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb import safelog +def _copyUnparseableDescriptorFile(filename): + """Save a copy of the bad descriptor file for later debugging. + + If the old filename was ``'descriptors/cached-extrainfo.new'``, then the + name of the copy will be something like + ``'descriptors/2014-11-05-01:57:23_cached-extrainfo.new.unparseable'``. + + :param str filename: The path to the unparseable descriptor file that we + should save a copy of. + :rtype: bool + :returns: ``True`` if a copy of the file was saved successfully, and + ``False`` otherwise. + """ + timestamp = datetime.datetime.now() + timestamp = timestamp.isoformat(sep=chr(0x2d)) + timestamp = timestamp.rsplit('.', 1)[0] + + path, sep, fname = filename.rpartition(os.path.sep) + newfilename = "%s%s%s_%s%sunparseable" % (path, sep, timestamp, + fname, os.path.extsep) + + logging.info(("Unparseable descriptor file '%s' will be copied to '%s' " + "for debugging.") % (filename, newfilename)) + + try: + shutil.copyfile(filename, newfilename) + except Exception as error: # pragma: no cover + logging.error(("Could not save copy of unparseable descriptor file " + "in '%s': %s") % (newfilename, str(error))) + return False + else: + logging.debug(("Successfully finished saving a copy of an unparseable " + "descriptor file.")) + return True + def parseNetworkStatusFile(filename, validate=True, skipAnnotations=True, descriptorClass=RouterStatusEntryV3): """Parse a file which contains an ``@type bridge-networkstatus`` document.

1 0

[bridgedb/develop] Add link to leekspin in README.rst.
by isis＠torproject.org 20 Mar '15

20 Mar '15

commit a5b15094679a9898aea9868bed7f5085a584f4d2 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Mar 21 01:24:08 2015 +0000 Add link to leekspin in README.rst. --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b4472c7..155c4f2 100644 --- a/README.rst +++ b/README.rst @@ -309,8 +309,9 @@ To create a bunch of fake bridge descriptors to test BridgeDB, do:: bridgedb mock [-n NUMBER_OF_DESCRIPTORS] -Note that you will need to install ``leekspin`` in order to run the -``bridgedb mock``. See HACKING.md for details. +Note that you will need to install +`leekspin <https://pypi.python.org/pypi/leekspin>`__ in order to run the +``bridgedb mock`` command. See ``doc/HACKING.md`` for details. And finally, to run the test suites, do::

1 0

[bridgedb/master] Merge branch 'master' into develop
by isis＠torproject.org 20 Mar '15

20 Mar '15

commit 1343b4b7f100782338bcf3a00932043f692e5941 Merge: 9955bd5 e08b98e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 3 02:29:15 2015 +0000 Merge branch 'master' into develop

1 0

[bridgedb/develop] Fix typos in README.rst.
by isis＠torproject.org 20 Mar '15

20 Mar '15

commit 11217eee36553e8a4ba3911bc2d1d4c125067061 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Mar 21 01:24:24 2015 +0000 Fix typos in README.rst. --- README.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 155c4f2..a9a27fc 100644 --- a/README.rst +++ b/README.rst @@ -384,12 +384,13 @@ Using a BridgeDB Instance ========================= Obviously, you'll have to feed it bridge descriptor files from a -BridgeAuth. There's currently only one BridgeAuth in the entire world, but Tor -Project is, of course, very interested in adding support for multiple -BridgeAuths so that we can scale our own network, and make it easier for -individual and organisations who wish to run a lot of Tor bridge relays, and -distribute those themselves, have an easier time doing so. If you'd like to -fund our work on this, please contact tor-dev(a)lists.torproject.org! +BridgeAuthority. There's currently only one BridgeAuthority in the entire +world, but Tor Project is, of course, very interested in adding support for +multiple BridgeAuthorities so that we can scale our own network, and make it +easier for individual and organisations who wish to run a lot of Tor bridge +relays have an easier time distributing those bridges themselves (if they wish +to do so). If you'd like to fund our work on this, please contact +tor-dev(a)lists.torproject.org! ---------------------------------- Accessing the HTTPS User Interface

1 0

[bridgedb/master] Update CHANGELOG for bridgedb-0.2.4 release.
by isis＠torproject.org 20 Mar '15

20 Mar '15

commit e333d91c632ee2c8bb00250cbe29c02feb174e03 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 3 03:04:38 2015 +0000 Update CHANGELOG for bridgedb-0.2.4 release. --- CHANGELOG | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index c6a5c2c..8c21489 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,68 @@ -Changes in version 0.2.4 - 2014-08-XXX +Changes in version 0.2.4 - 2015-02-03 + + * HOTFIXES a UnicodeDecodeError resulting from patches for #12627. + https://bugs.torproject.org/12627 + * FIXES #9874 https://bugs.torproject.org/9874 BridgeDB now has integration tests for all bridge distributors. Thanks to trygve for the patches. + * FIXES #12871 https://bugs.torproject.org/12871 + Bridge Buckets now work, even if the code for calculating Bridge + stability is disabled. + Thanks to Matt Finkel for the patches. + + * FIXES part of #12029 https://bugs.torproject.org/12029 + Major sections of the bridgedb.Bridges module, which holds + BridgeDB's main data structures for storing and parsing Bridges, + have been refactored in preparation for upcoming changes to use + Stem's parsers (see #9380 https://bugs.torproject.org/9380) + + * FIXES #12932 https://bugs.torproject.org/12932 + Arguments for Pluggable Transports in the bridge lines which + BridgeDB distributes to users are now properly + space-separated. This issue was affecting the deployment of the + obfs4 PT (see #12130 https://bugs.torproject.org/12130) + + * FIXES #13123 https://bugs.torproject.org/13123 + Previously, there were two additional whitespace characters at the + beginning of bridge lines handed out by BridgeDB's HTTPS + distributor, which would be annoying copy+pasted into TorLauncher + and torrcs, etc. These are now gone. + + * FIXES #12664 https://bugs.torproject.org/12664 + Previously, for the bridge lines handed out by BridgeDB's HTTPS + distributor, the newlines were not properly pasted when a user + would copy+paste the lines. This is now fixed. Additionally, there + is now a "Select All" button (JS must be enabled) to select all + text for the bridge lines, to attempt to reduce user copy+paste + errors. If the display area which contains the bridge lines is + clicked, and JS is enabled, it has the same effect as clicking the + "Select All" button. + + * FIXES #14064 https://bugs.torproject.org/14064 + The bridge lines handed out by BridgeDB's HTTPS distributor are + now displayed with a horizontal scrollbar if they are too long to + fit into the display area. + + * FIXES #11345 https://bugs.torproject.org/11345 + BridgeDB now supports giving users QRCodes for their bridge lines, + to facilitate getting bridges into Tails and onto mobile devices. + + * FIXES #12130 https://bugs.torproject.org/12130 + BridgeDB's distributors now have options to distribute obfs4 bridges. + +And includes the following general changes: + * CHANGES the integration tests based on Mechanize to only run on + CI servers, not locally on developers laptops, since it requires + the running BridgeDB test/staging instance to offer a plaintext + HTTP interface. See commit 24acf6a72. + https://gitweb.torproject.org/bridgedb.git/commit/?id=24acf6a72931c602631c9… + + * ADDS better installation instructions in README.rst for + developers who wish to test their changes to BridgeDB. + Thanks to Alden Page for the patch. + Changes in version 0.2.3 - 2014-07-26 @@ -39,7 +99,7 @@ Changes in version 0.2.3 - 2014-07-26 speed up this process, so sometimes BridgeDB's UI falls back to English when it doesn't have new enough translations files. Sorry! -And include the following general changes: +And includes the following general changes: * UPDATE translation: Chinese - Taiwan (zh_TW) THANKS TO danfong.

1 0

[bridgedb/develop] Merge branch 'master' into develop
by isis＠torproject.org 20 Mar '15

20 Mar '15

commit a06688478f76fb1bcc6e5a0f4d4aa41720bc8eb0 Merge: cdc0ffa 6f0f536 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Mar 21 02:00:52 2015 +0000 Merge branch 'master' into develop

1 0

[bridgedb/develop] Merge branch 'release-0.3.0'
by isis＠torproject.org 20 Mar '15

20 Mar '15

commit 6f0f5365f92d3f39db6ee3d52b7a5eedc09176ab Merge: e08b98e cdc0ffa Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Mar 21 02:00:24 2015 +0000 Merge branch 'release-0.3.0' .gitignore | 6 +- .gnupg/TESTING.pub | Bin 0 -> 4622 bytes .gnupg/TESTING.subkeys.sec | Bin 0 -> 7212 bytes .gnupg/bridgedb-offline-key.pub.asc | 113 ++ .gnupg/bridgedb-online-key.pub.asc | 100 ++ .gnupg/gpg.conf | 96 ++ .gnupg/pubring.gpg | Bin 0 -> 4634 bytes .gnupg/secring.gpg | Bin 0 -> 7224 bytes .test.requirements.txt | 2 +- .travis.requirements.txt | 9 +- .travis.yml | 28 +- CHANGELOG | 285 +++- Makefile | 6 +- README.rst | 73 +- TODO | 58 - bridgedb.conf | 87 +- doc/sphinx/source/bridgedb.Tests.rst | 8 - doc/sphinx/source/bridgedb.bridges.rst | 8 + doc/sphinx/source/bridgedb.geo.rst | 8 + doc/sphinx/source/bridgedb.parse.rst | 4 +- doc/sphinx/source/bridgedb.proxy.rst | 8 + doc/sphinx/source/bridgedb.qrcodes.rst | 8 + doc/sphinx/source/bridgedb.rst | 5 +- doc/sphinx/source/conf.py | 11 +- gnupghome/TESTING.subkeys.sec | Bin 7212 -> 0 bytes gnupghome/TESTING.subkeys.sec.EXPIRED-2013-09-11 | Bin 6505 -> 0 bytes gnupghome/bridgedb-offline-key.pub.asc | 113 -- gnupghome/bridgedb-online-signing-key.pub.asc | 148 -- gnupghome/gpg.conf | 96 -- lib/bridgedb/Bridges.py | 686 +-------- lib/bridgedb/Filters.py | 89 +- lib/bridgedb/HTTPServer.py | 37 +- lib/bridgedb/Main.py | 352 ++--- lib/bridgedb/Stability.py | 2 +- lib/bridgedb/Storage.py | 5 +- lib/bridgedb/Tests.py | 780 ----------- lib/bridgedb/bridgerequest.py | 54 +- lib/bridgedb/bridges.py | 1603 ++++++++++++++++++++++ lib/bridgedb/configure.py | 22 +- lib/bridgedb/crypto.py | 328 ++--- lib/bridgedb/email/autoresponder.py | 44 +- lib/bridgedb/email/server.py | 12 +- lib/bridgedb/geo.py | 78 ++ lib/bridgedb/i18n/ar/LC_MESSAGES/bridgedb.po | 21 +- lib/bridgedb/i18n/az/LC_MESSAGES/bridgedb.po | 357 +++++ lib/bridgedb/i18n/bg/LC_MESSAGES/bridgedb.po | 357 +++++ lib/bridgedb/i18n/ca/LC_MESSAGES/bridgedb.po | 392 +++++- lib/bridgedb/i18n/cs/LC_MESSAGES/bridgedb.po | 367 ++++- lib/bridgedb/i18n/da/LC_MESSAGES/bridgedb.po | 169 ++- lib/bridgedb/i18n/de/LC_MESSAGES/bridgedb.po | 103 +- lib/bridgedb/i18n/el/LC_MESSAGES/bridgedb.po | 20 +- lib/bridgedb/i18n/en/LC_MESSAGES/bridgedb.po | 445 +++++- lib/bridgedb/i18n/en_GB/LC_MESSAGES/bridgedb.po | 387 +++++- lib/bridgedb/i18n/en_US/LC_MESSAGES/bridgedb.po | 447 +++++- lib/bridgedb/i18n/eo/LC_MESSAGES/bridgedb.po | 359 +++++ lib/bridgedb/i18n/es/LC_MESSAGES/bridgedb.po | 79 +- lib/bridgedb/i18n/fa/LC_MESSAGES/bridgedb.po | 63 +- lib/bridgedb/i18n/fi/LC_MESSAGES/bridgedb.po | 63 +- lib/bridgedb/i18n/fr/LC_MESSAGES/bridgedb.po | 61 +- lib/bridgedb/i18n/fr_CA/LC_MESSAGES/bridgedb.po | 64 +- lib/bridgedb/i18n/hr_HR/LC_MESSAGES/bridgedb.po | 72 +- lib/bridgedb/i18n/hu/LC_MESSAGES/bridgedb.po | 59 +- lib/bridgedb/i18n/it/LC_MESSAGES/bridgedb.po | 161 ++- lib/bridgedb/i18n/ja/LC_MESSAGES/bridgedb.po | 145 +- lib/bridgedb/i18n/km/LC_MESSAGES/bridgedb.po | 19 +- lib/bridgedb/i18n/lv/LC_MESSAGES/bridgedb.po | 20 +- lib/bridgedb/i18n/nb/LC_MESSAGES/bridgedb.po | 161 ++- lib/bridgedb/i18n/nl/LC_MESSAGES/bridgedb.po | 170 ++- lib/bridgedb/i18n/pl/LC_MESSAGES/bridgedb.po | 100 +- lib/bridgedb/i18n/pt/LC_MESSAGES/bridgedb.po | 64 +- lib/bridgedb/i18n/pt_BR/LC_MESSAGES/bridgedb.po | 53 +- lib/bridgedb/i18n/ro/LC_MESSAGES/bridgedb.po | 21 +- lib/bridgedb/i18n/ru/LC_MESSAGES/bridgedb.po | 65 +- lib/bridgedb/i18n/sk/LC_MESSAGES/bridgedb.po | 367 ++++- lib/bridgedb/i18n/sk_SK/LC_MESSAGES/bridgedb.po | 357 +++++ lib/bridgedb/i18n/sl_SI/LC_MESSAGES/bridgedb.po | 359 +++++ lib/bridgedb/i18n/sq/LC_MESSAGES/bridgedb.po | 380 +++++ lib/bridgedb/i18n/sv/LC_MESSAGES/bridgedb.po | 61 +- lib/bridgedb/i18n/ta/LC_MESSAGES/bridgedb.po | 357 +++++ lib/bridgedb/i18n/templates/bridgedb.pot | 90 +- lib/bridgedb/i18n/tr/LC_MESSAGES/bridgedb.po | 97 +- lib/bridgedb/i18n/uk/LC_MESSAGES/bridgedb.po | 184 ++- lib/bridgedb/i18n/zh_CN/LC_MESSAGES/bridgedb.po | 121 +- lib/bridgedb/i18n/zh_TW/LC_MESSAGES/bridgedb.po | 22 +- lib/bridgedb/parse/addr.py | 7 + lib/bridgedb/parse/descriptors.py | 274 ++++ lib/bridgedb/parse/fingerprint.py | 16 +- lib/bridgedb/parse/networkstatus.py | 280 ---- lib/bridgedb/parse/nickname.py | 58 + lib/bridgedb/persistent.py | 45 +- lib/bridgedb/proxy.py | 470 +++++++ lib/bridgedb/qrcodes.py | 8 +- lib/bridgedb/runner.py | 29 + lib/bridgedb/schedule.py | 6 +- lib/bridgedb/strings.py | 5 +- lib/bridgedb/templates/bridgedb.asc | 226 ++- lib/bridgedb/test/deprecated.py | 520 +++++++ lib/bridgedb/test/deprecated_networkstatus.py | 275 ++++ lib/bridgedb/test/email_helpers.py | 15 +- lib/bridgedb/test/legacy_Tests.py | 725 ++++++++++ lib/bridgedb/test/test_Bridges.py | 199 --- lib/bridgedb/test/test_Main.py | 432 ++++++ lib/bridgedb/test/test_Tests.py | 13 +- lib/bridgedb/test/test_bridgedb.py | 1 + lib/bridgedb/test/test_bridges.py | 1357 ++++++++++++++++++ lib/bridgedb/test/test_crypto.py | 364 ++--- lib/bridgedb/test/test_email_autoresponder.py | 2 +- lib/bridgedb/test/test_geo.py | 96 ++ lib/bridgedb/test/test_https.py | 10 +- lib/bridgedb/test/test_parse_descriptors.py | 653 +++++++++ lib/bridgedb/test/test_parse_networkstatus.py | 414 ------ lib/bridgedb/test/test_parse_nickname.py | 63 + lib/bridgedb/test/test_persistentSaveAndLoad.py | 68 +- lib/bridgedb/test/test_proxy.py | 590 ++++++++ lib/bridgedb/test/test_qrcodes.py | 73 + lib/bridgedb/test/test_smtp.py | 1 + lib/bridgedb/test/test_util.py | 42 + lib/bridgedb/util.py | 10 +- requirements.txt | 3 +- scripts/get-tor-exits | 199 +++ setup.cfg | 4 + setup.py | 5 +- 122 files changed, 15331 insertions(+), 4858 deletions(-)

1 0