commit e14d74473ff7319298a920129db6037079ab3652 Author: Damian Johnson atagar@torproject.org Date: Thu Aug 15 13:53:35 2019 -0700

Bridge descriptor support

Adding a 'bridge' argument to descriptor download methods to retrieve bridge counterparts instead. --- stem/descriptor/__init__.py | 2 + stem/descriptor/collector.py | 43 +++++++++-------- stem/descriptor/networkstatus.py | 7 ++- stem/descriptor/router_status_entry.py | 13 ++++++ test/unit/descriptor/collector.py | 52 ++++++++++++++++++++- .../bridge-extra-infos-2019-03-cropped.tar | Bin 0 -> 15872 bytes .../bridge-server-descriptors-2019-02-cropped.tar | Bin 0 -> 9216 bytes .../collector/bridge-statuses-2019-05-cropped.tar | Bin 0 -> 467456 bytes 8 files changed, 93 insertions(+), 24 deletions(-)

diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 2044c7bf..c099ca86 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -920,6 +920,8 @@ class Descriptor(object): :returns: :class:`~stem.descriptor.TypeAnnotation` with our type information """

+ # TODO: populate this from the archive instead if available (so we have correct version numbers) + if self.TYPE_ANNOTATION_NAME is not None: return TypeAnnotation(self.TYPE_ANNOTATION_NAME, 1, 0) else: diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 99d02242..b7982426 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -146,25 +146,25 @@ def get_instance(): return SINGLETON_COLLECTOR

-def get_server_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3): +def get_server_descriptors(start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3): """ Shorthand for :func:`~stem.descriptor.collector.CollecTor.get_server_descriptors` on our singleton instance. """

- for desc in get_instance().get_server_descriptors(start, end, cache_to, timeout, retries): + for desc in get_instance().get_server_descriptors(start, end, cache_to, bridge, timeout, retries): yield desc

-def get_extrainfo_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3): +def get_extrainfo_descriptors(start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3): """ Shorthand for :func:`~stem.descriptor.collector.CollecTor.get_extrainfo_descriptors` on our singleton instance. """

- for desc in get_instance().get_extrainfo_descriptors(start, end, cache_to, timeout, retries): + for desc in get_instance().get_extrainfo_descriptors(start, end, cache_to, bridge, timeout, retries): yield desc

@@ -179,14 +179,14 @@ def get_microdescriptors(start = None, end = None, cache_to = None, timeout = No yield desc

-def get_consensus(start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, timeout = None, retries = 3): +def get_consensus(start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, bridge = False, timeout = None, retries = 3): """ Shorthand for :func:`~stem.descriptor.collector.CollecTor.get_consensus` on our singleton instance. """

- for desc in get_instance().get_consensus(start, end, cache_to, document_handler, version, microdescriptor, timeout, retries): + for desc in get_instance().get_consensus(start, end, cache_to, document_handler, version, microdescriptor, bridge, timeout, retries): yield desc

@@ -413,7 +413,7 @@ class CollecTor(object): self._cached_files = None self._cached_index_at = 0

- def get_server_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + def get_server_descriptors(self, start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3): """ Provides server descriptors published during the given time range, sorted oldest to newest. @@ -422,6 +422,7 @@ class CollecTor(object): :param datetime.datetime end: time range to end with :param str cache_to: directory to cache archives into, if an archive is available here it is not downloaded + :param bool bridge: standard descriptors if **False**, bridge if **True** :param int timeout: timeout for downloading each individual archive when the connection becomes idle, no timeout applied if **None** :param int retires: maximum attempts to impose on a per-archive basis @@ -433,13 +434,13 @@ class CollecTor(object): :raises: :class:`~stem.DownloadFailed` if the download fails """

- # TODO: support bridge variants ('bridge-server-descriptor' type) + desc_type = 'server-descriptor' if not bridge else 'bridge-server-descriptor'

- for f in self.files('server-descriptor', start, end): - for desc in f.read(cache_to, 'server-descriptor', timeout = timeout, retries = retries): + for f in self.files(desc_type, start, end): + for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries): yield desc

- def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3): """ Provides extrainfo descriptors published during the given time range, sorted oldest to newest. @@ -448,6 +449,7 @@ class CollecTor(object): :param datetime.datetime end: time range to end with :param str cache_to: directory to cache archives into, if an archive is available here it is not downloaded + :param bool bridge: standard descriptors if **False**, bridge if **True** :param int timeout: timeout for downloading each individual archive when the connection becomes idle, no timeout applied if **None** :param int retires: maximum attempts to impose on a per-archive basis @@ -459,10 +461,10 @@ class CollecTor(object): :raises: :class:`~stem.DownloadFailed` if the download fails """

- # TODO: support bridge variants ('bridge-extra-info' type) + desc_type = 'extra-info' if not bridge else 'bridge-extra-info'

- for f in self.files('extra-info', start, end): - for desc in f.read(cache_to, 'extra-info', timeout = timeout, retries = retries): + for f in self.files(desc_type, start, end): + for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries): yield desc

def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): @@ -498,7 +500,7 @@ class CollecTor(object): for desc in f.read(cache_to, 'microdescriptor', timeout = timeout, retries = retries): yield desc

- def get_consensus(self, start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, timeout = None, retries = 3): + def get_consensus(self, start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, bridge = False, timeout = None, retries = 3): """ Provides consensus router status entries published during the given time range, sorted oldest to newest. @@ -512,6 +514,7 @@ class CollecTor(object): :param int version: consensus variant to retrieve (versions 2 or 3) :param bool microdescriptor: provides the microdescriptor consensus if **True**, standard consensus otherwise + :param bool bridge: standard descriptors if **False**, bridge if **True** :param int timeout: timeout for downloading each individual archive when the connection becomes idle, no timeout applied if **None** :param int retires: maximum attempts to impose on a per-archive basis @@ -523,20 +526,20 @@ class CollecTor(object): :raises: :class:`~stem.DownloadFailed` if the download fails """

- if version == 3 and not microdescriptor: + if version == 3 and not microdescriptor and not bridge: desc_type = 'network-status-consensus-3' - elif version == 3 and microdescriptor: + elif version == 3 and microdescriptor and not bridge: desc_type = 'network-status-microdesc-consensus-3' - elif version == 2 and not microdescriptor: + elif version == 2 and not microdescriptor and not bridge: desc_type = 'network-status-2' + elif bridge: + desc_type = 'bridge-network-status' else: if microdescriptor and version != 3: raise ValueError('Only v3 microdescriptors are available (not version %s)' % version) else: raise ValueError('Only v2 and v3 router status entries are available (not version %s)' % version)

- # TODO: support bridge variants ('bridge-network-status' type) - for f in self.files(desc_type, start, end): for desc in f.read(cache_to, desc_type, document_handler, timeout = timeout, retries = retries): yield desc diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index b0589f2a..5f542c2f 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -93,6 +93,7 @@ from stem.descriptor import (

from stem.descriptor.router_status_entry import ( RouterStatusEntryV2, + RouterStatusEntryBridgeV2, RouterStatusEntryV3, RouterStatusEntryMicroV3, ) @@ -322,7 +323,7 @@ def _parse_file(document_file, document_type = None, validate = False, is_microd elif document_type == NetworkStatusDocumentV3: router_type = RouterStatusEntryMicroV3 if is_microdescriptor else RouterStatusEntryV3 elif document_type == BridgeNetworkStatusDocument: - document_type, router_type = BridgeNetworkStatusDocument, RouterStatusEntryV2 + document_type, router_type = BridgeNetworkStatusDocument, RouterStatusEntryBridgeV2 elif document_type == DetachedSignature: yield document_type(document_file.read(), validate, **kwargs) return @@ -1228,7 +1229,9 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): self._footer(document_file, validate)

def type_annotation(self): - if not self.is_microdescriptor: + if isinstance(self, BridgeNetworkStatusDocument): + return TypeAnnotation('bridge-network-status', 1, 0) + elif not self.is_microdescriptor: return TypeAnnotation('network-status-consensus-3' if not self.is_vote else 'network-status-vote-3', 1, 0) else: # Directory authorities do not issue a 'microdescriptor consensus' vote, diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index 31913417..ce662b40 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -15,6 +15,8 @@ sources...

RouterStatusEntry - Common parent for router status entries |- RouterStatusEntryV2 - Entry for a network status v2 document + | +- RouterStatusEntryBridgeV2 - Entry for a bridge flavored v2 document + | |- RouterStatusEntryV3 - Entry for a network status v3 document +- RouterStatusEntryMicroV3 - Entry for a microdescriptor flavored v3 document """ @@ -540,6 +542,17 @@ class RouterStatusEntryV2(RouterStatusEntry): return ('r', 's', 'v')

+class RouterStatusEntryBridgeV2(RouterStatusEntryV2): + """ + Information about an individual router stored within a bridge flavored + version 2 network status document. + + .. versionadded:: 1.8.0 + """ + + TYPE_ANNOTATION_NAME = 'bridge-network-status' + + class RouterStatusEntryV3(RouterStatusEntry): """ Information about an individual router stored within a version 3 network diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 7d1f0205..3ab06f03 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -212,6 +212,21 @@ class TestCollector(unittest.TestCase):

@patch('stem.util.connection.download') @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_bridge_server_descriptors(self, files_mock, download_mock): + with open(get_resource('collector/bridge-server-descriptors-2019-02-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/server-descriptors/bridge-server-descriptors-2019-02.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_server_descriptors(bridge = True)) + self.assertEqual(4, len(descriptors)) + + f = descriptors[0] + self.assertEqual('BridgeDescriptor', type(f).__name__) + self.assertEqual('E90D1DE12B930DEC3F3E1127AAA25E47430CD3F4', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') def test_reading_extrainfo_descriptors(self, files_mock, download_mock): with open(get_resource('collector/extra-infos-2019-04-cropped.tar'), 'rb') as archive: download_mock.return_value = archive.read() @@ -227,6 +242,21 @@ class TestCollector(unittest.TestCase):

@patch('stem.util.connection.download') @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_bridge_extrainfo_descriptors(self, files_mock, download_mock): + with open(get_resource('collector/bridge-extra-infos-2019-03-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2019-03.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors(bridge = True)) + self.assertEqual(6, len(descriptors)) + + f = descriptors[0] + self.assertEqual('BridgeExtraInfoDescriptor', type(f).__name__) + self.assertEqual('A0187027648A392C6AC413B66F7CD25DD001BF76', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') def test_reading_microdescriptors(self, files_mock, download_mock): with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive: download_mock.return_value = archive.read() @@ -267,14 +297,32 @@ class TestCollector(unittest.TestCase): self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(version = 2)))) self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(microdescriptor = True))))

- # but the microdescriptor archive *does* have microdescriptor consensuses - + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_microdescriptor_consensus(self, files_mock, download_mock): with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive: download_mock.return_value = archive.read()

+ files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/microdescs/microdescs-2019-05.tar', 12345, '2016-09-04 09:21')] + descriptors = list(stem.descriptor.collector.get_consensus(microdescriptor = True)) self.assertEqual(556, len(descriptors))

f = descriptors[0] self.assertEqual('RouterStatusEntryMicroV3', type(f).__name__) self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_bridge_consensus(self, files_mock, download_mock): + with open(get_resource('collector/bridge-statuses-2019-05-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/microdescs/bridge-statuses-2019-05.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_consensus(bridge = True)) + self.assertEqual(2593, len(descriptors)) + + f = descriptors[0] + self.assertEqual('RouterStatusEntryBridgeV2', type(f).__name__) + self.assertEqual('0035EA2A61E28D395F080ACA2244539490E70950', f.fingerprint) diff --git a/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar b/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar new file mode 100644 index 00000000..db9f3f7b Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar differ diff --git a/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar b/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar new file mode 100644 index 00000000..4d0255f3 Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar differ diff --git a/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar b/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar new file mode 100644 index 00000000..953d2a86 Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar differ