March 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Change unittest to demand {(fpr, router), ...} is returned after deduplicate().
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 1650ff00c97f9e39afdcdfefa08741cffbb70163 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 03:55:55 2014 +0000 Change unittest to demand {(fpr, router),...} is returned after deduplicate(). --- lib/bridgedb/test/test_parse_descriptors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 6a44895..85863e4 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -231,11 +231,11 @@ class ParseDescriptorsTests(unittest.TestCase): def test_parse_descriptors_parseBridgeExtraInfoFiles_return_type(self): """The return type of ``b.p.descriptors.parseBridgeExtraInfoFiles`` - should be a list. + should be a dictionary (after deduplication). """ descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) - self.assertIsInstance(routers, list) + self.assertIsInstance(routers, dict) def test_parse_descriptors_parseBridgeExtraInfoFiles_has_BridgeExtraInfoDescriptor(self): """The return of ``b.p.descriptors.parseBridgeExtraInfoFiles`` should

1 0

[bridgedb/master] Fix faulty unittests which expected parsed extrainfos to have an `address`.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 41f0f5c255740d777764fa7a336769c317b0cc26 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 03:57:35 2014 +0000 Fix faulty unittests which expected parsed extrainfos to have an `address`. --- lib/bridgedb/test/test_parse_descriptors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 85863e4..2a1171e 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -253,7 +253,7 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) bridge = routers.values()[0] - self.assertEqual(bridge.address, u'152.78.9.20') + self.assertEqual(len(bridge.transport), 2) self.assertEqual(bridge.fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') @@ -266,6 +266,5 @@ class ParseDescriptorsTests(unittest.TestCase): descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE) routers = descriptors.parseBridgeExtraInfoFiles(descFileOne, descFileTwo) bridge = routers.values()[0] - self.assertEqual(bridge.address, u'152.78.9.20') self.assertEqual(bridge.fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC')

1 0

[bridgedb/master] Add unittest explicitly for return type of parseBridgeExtraInfoFiles().
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 75ab626e39474eba43a045a6110acf513bfe9410 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 02:01:18 2014 +0000 Add unittest explicitly for return type of parseBridgeExtraInfoFiles(). --- lib/bridgedb/test/test_parse_descriptors.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 0ad54aa..e38b9fe 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -218,12 +218,19 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertEqual(bridge.fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + def test_parse_descriptors_parseBridgeExtraInfoFiles_return_type(self): + """The return type of ``b.p.descriptors.parseBridgeExtraInfoFiles`` + should be a list. + """ + descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) + routers = descriptors.parseBridgeExtraInfoFiles(descFile) + self.assertIsInstance(routers, list) + def test_parse_descriptors_parseBridgeExtraInfoFiles_one_file(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with only one bridge extrainfo file.""" descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) - self.assertIsInstance(routers, list) bridge = routers[0] self.assertIsInstance(bridge, BridgeExtraInfoDescriptor) self.assertEqual(bridge.address, u'152.78.9.20') @@ -237,7 +244,6 @@ class ParseDescriptorsTests(unittest.TestCase): descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE) routers = descriptors.parseBridgeExtraInfoFiles(descFileOne, descFileTwo) - self.assertIsInstance(routers, list) bridge = routers[0] self.assertIsInstance(bridge, BridgeExtraInfoDescriptor) self.assertEqual(bridge.address, u'152.78.9.20')

1 0

[bridgedb/master] Add test to check return of parseNetworkStatusFile() has `RouterStatusEntryV2`s.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 41ff4999a08328d235e65b4c858c36db5b76598b Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 02:38:15 2014 +0000 Add test to check return of parseNetworkStatusFile() has `RouterStatusEntryV2`s. --- lib/bridgedb/test/test_parse_descriptors.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index e38b9fe..6076de0 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -187,7 +187,20 @@ class ParseDescriptorsTests(unittest.TestCase): routers = descriptors.parseNetworkStatusFile(descFile) self.assertIsInstance(routers, dict) - def test_parse_descriptors_parseBridgeNetworkStatusFile_one_file(self): + def test_parse_descriptors_parseBridgeNetworkStatusFile_has_RouterStatusEntryV2(self): + """The items in the dict returned from + ``b.p.descriptors.parseNetworkStatusFile`` should be + ``RouterStatusEntryV2``s. + """ + # Write the descriptor to a file for testing. This is necessary + # because the function opens the networkstatus file to read it. + descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', + BRIDGE_NETWORKSTATUS_0) + routers = descriptors.parseNetworkStatusFile(descFile) + fingerprint, bridge = routers.items()[0] + self.assertIsInstance(bridge, RouterStatusEntryV2) + + def test_parse_descriptors_parseBridgeNetworkStatusFile_1(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with one bridge networkstatus descriptor. """ @@ -202,7 +215,7 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertEqual(bridge.fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') - def test_parse_descriptors_parseBridgeNetworkStatusFile_two_files(self): + def test_parse_descriptors_parseBridgeNetworkStatusFile_2(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with two bridge networkstatus descriptors. """

1 0

[bridgedb/master] Add another unittest for parseBridgeExtraInfoFiles() return type.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 6700f6b7f15fd3d8fe964fd3a16f64b96d455d02 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 03:44:57 2014 +0000 Add another unittest for parseBridgeExtraInfoFiles() return type. --- lib/bridgedb/test/test_parse_descriptors.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index fae6441..9cb4b91 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -237,13 +237,21 @@ class ParseDescriptorsTests(unittest.TestCase): routers = descriptors.parseBridgeExtraInfoFiles(descFile) self.assertIsInstance(routers, list) + def test_parse_descriptors_parseBridgeExtraInfoFiles_has_BridgeExtraInfoDescriptor(self): + """The return of ``b.p.descriptors.parseBridgeExtraInfoFiles`` should + contain ``BridgeExtraInfoDescriptor``s. + """ + descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) + routers = descriptors.parseBridgeExtraInfoFiles(descFile) + bridge = routers.values()[0] + self.assertIsInstance(bridge, BridgeExtraInfoDescriptor) + def test_parse_descriptors_parseBridgeExtraInfoFiles_one_file(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with only one bridge extrainfo file.""" descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) - bridge = routers[0] - self.assertIsInstance(bridge, BridgeExtraInfoDescriptor) + bridge = routers.values()[0] self.assertEqual(bridge.address, u'152.78.9.20') self.assertEqual(bridge.fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') @@ -255,8 +263,7 @@ class ParseDescriptorsTests(unittest.TestCase): descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE) routers = descriptors.parseBridgeExtraInfoFiles(descFileOne, descFileTwo) - bridge = routers[0] - self.assertIsInstance(bridge, BridgeExtraInfoDescriptor) + bridge = routers.values()[0] self.assertEqual(bridge.address, u'152.78.9.20') self.assertEqual(bridge.fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC')

1 0

[bridgedb/master] Fix some docstring stylings in test_parse_descriptors.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 9611f5d08fa599c6e022afb010a1cf974f44adee Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 03:55:14 2014 +0000 Fix some docstring stylings in test_parse_descriptors. --- lib/bridgedb/test/test_parse_descriptors.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 9cb4b91..6a44895 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -248,7 +248,8 @@ class ParseDescriptorsTests(unittest.TestCase): def test_parse_descriptors_parseBridgeExtraInfoFiles_one_file(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with only one - bridge extrainfo file.""" + bridge extrainfo file. + """ descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) bridge = routers.values()[0] @@ -259,7 +260,8 @@ class ParseDescriptorsTests(unittest.TestCase): def test_parse_descriptors_parseBridgeExtraInfoFiles_two_files(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with two bridge extrainfo files, and check that only the newest extrainfo - descriptor is used.""" + descriptor is used. + """ descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE) routers = descriptors.parseBridgeExtraInfoFiles(descFileOne, descFileTwo)

1 0

[bridgedb/master] Fix fingerprints in two tests for parseNetworkStatusFile().
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 1cb15d01fa107467b811eca7e3ebd17f3a31ecf9 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 02:42:10 2014 +0000 Fix fingerprints in two tests for parseNetworkStatusFile(). --- lib/bridgedb/test/test_parse_descriptors.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 6076de0..fae6441 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -209,10 +209,9 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) - bridge = routers.items()[0] - self.assertIsInstance(bridge, RouterStatusEntryV2) + fingerprint, bridge = routers.items()[0] self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, + self.assertEqual(fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') def test_parse_descriptors_parseBridgeNetworkStatusFile_2(self): @@ -225,10 +224,9 @@ class ParseDescriptorsTests(unittest.TestCase): BRIDGE_NETWORKSTATUS_0, BRIDGE_NETWORKSTATUS_1) routers = descriptors.parseNetworkStatusFile(descFile) - bridge = routers.items()[0] - self.assertIsInstance(bridge, RouterStatusEntryV2) + fingerprint, bridge = routers.items()[0] self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, + self.assertEqual(fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') def test_parse_descriptors_parseBridgeExtraInfoFiles_return_type(self):

1 0

[bridgedb/master] This test was supposed to test deduplication. Now it does.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 65f18cd7c31a97274a8277688f0512b69ac1e3e3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 03:58:28 2014 +0000 This test was supposed to test deduplication. Now it does. --- lib/bridgedb/test/test_parse_descriptors.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 2a1171e..b947a39 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -265,6 +265,5 @@ class ParseDescriptorsTests(unittest.TestCase): descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE) routers = descriptors.parseBridgeExtraInfoFiles(descFileOne, descFileTwo) - bridge = routers.values()[0] - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + # We shouldn't have duplicates: + self.assertEqual(len(routers), 1)

1 0

[bridgedb/master] Add more info to docstring for `b.p.descriptors.parseBridgeExtraInfoFiles()`.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 1e4de24ac711dbcbe6b1992da951a90ee64c2e8e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 04:00:27 2014 +0000 Add more info to docstring for `b.p.descriptors.parseBridgeExtraInfoFiles()`. --- lib/bridgedb/parse/descriptors.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index 939ecb0..ab0d37f 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -141,9 +141,15 @@ def deduplicate(descriptors): def parseBridgeExtraInfoFiles(*filenames, **kwargs): """Parse files which contain ``@type bridge-extrainfo-descriptor``s. - :kwargs: If there is a ``'validate'`` keyword argument, its value will be - passed along as the ``'validate'`` argument to + .. note:: This function will call :func:`deduplicate` to deduplicate the + extrainfo descriptors parsed from all **filenames**. + + :kwargs validate: If there is a ``'validate'`` keyword argument, its value + will be passed along as the ``'validate'`` argument to :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`. + :rtype: dict + :returns: A dictionary mapping bridge fingerprints to deduplicated + :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`s. """ descriptors = [] descriptorType = 'bridge-extra-info 1.1'

1 0

[bridgedb/master] Rewrite b.p.d.parseNetworkStatusFile() to use RouterStatusEntryV3.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit aa2d411c046318d541ef62d68c7d3da70918902b Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Aug 29 05:31:24 2014 +0000 Rewrite b.p.d.parseNetworkStatusFile() to use RouterStatusEntryV3. See Damian's comments on how we were using a RouterStatusEntryV2 in https://trac.torproject.org/projects/tor/ticket/9380#comment:16. --- lib/bridgedb/parse/descriptors.py | 73 ++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index a0806e2..5028ad3 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -16,48 +16,55 @@ import datetime import logging from stem.descriptor import extrainfo_descriptor -from stem.descriptor import networkstatus from stem.descriptor import server_descriptor from stem.descriptor import parse_file +from stem.descriptor.router_status_entry import _parse_file as _parseNSFile +from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb import safelog -def parseNetworkStatusFile(filename, validate=True): +def parseNetworkStatusFile(filename, validate=True, skipHeaders=True, + descriptorClass=RouterStatusEntryV3): """Parse a file which contains an ``@type bridge-networkstatus`` document. - :rtype: dict - :returns: A dictionary fingerprints mapped to - :api:`stem.descriptor.router_status_entry.RouterStatusEntryV2`s. + See `ticket #12254 <https://bugs.torproject.org/12254>`__ for why + networkstatus-bridges documents don't look anything like the networkstatus + v2 documents that they are purported to look like. They are missing all + headers, and the entire footer including authority signatures. + + :param str filename: The location of the file containing bridge + networkstatus descriptors. + :param bool validate: Passed along to Stem's parsers. If ``True``, the + descriptors will raise exceptions if they do not meet some definition + of correctness. + :param bool skipHeaders: If ``True``, skip parsing everything before the + first ``r`` line. + :param descriptorClass: A class (probably from + :api:`stem.descriptors.router_status_entry`) which Stem will parse + each descriptor it reads from **filename** into. + :raises ValueError: if the contents of a descriptor are malformed and + **validate** is ``True``. + :raises IOError: if the file at **filename** can't be read. + :rtype: list + :returns: A list of + :api:`stem.descriptor.router_status_entry.RouterStatusEntryV#`s. """ - logging.info("Parsing networkstatus entries with Stem: %s" % filename) - - fh = open(filename) - descriptors = fh.read() - fh.close() - - # See ticket #12254 for why networkstatus-bridges documents don't look - # anything like the networkstatus v2 documents that they are purported to - # look like. They are missing all headers, and the entire footer including - # authority signatures. - # - # https://trac.torproject.org/projects/tor/ticket/12254 - # - # As such, they do not currently start with a "published" line with an - # ISO8601 timestamp, as stem expects them to: - # - if not descriptors.startswith("published"): - precise = datetime.datetime.now().isoformat(sep=chr(0x20)) - timestamp = precise.rsplit('.', 1)[0] - descriptors = "published {t}\n{d}".format(t=timestamp, d=descriptors) - else: - logging.warn( - ("Networkstatus file '%s' started with 'published' line! Please " - "revise this function!") % filename) - - document = networkstatus.BridgeNetworkStatusDocument(descriptors, - validate=validate) - return document.routers + routers = [] + + logging.info("Parsing networkstatus file: %s" % filename) + with open(filename) as fh: + position = fh.tell() + if skipHeaders: + while not fh.readline().startswith('r '): + position = fh.tell() + logging.debug("Skipping %d bytes of networkstatus file." % position) + document = _parseNSFile(fh, validate, entry_class=descriptorClass, + start_position=position) + routers.extend(list(document)) + logging.info("Closed networkstatus file: %s" % filename) + + return routers def parseServerDescriptorsFile(filename, validate=False): """Parse a file which contains ``@type bridge-server-descriptor``s.

1 0