commit e14d74473ff7319298a920129db6037079ab3652
Author: Damian Johnson <atagar(a)torproject.org>
Date: Thu Aug 15 13:53:35 2019 -0700
Bridge descriptor support
Adding a 'bridge' argument to descriptor download methods to retrieve bridge
counterparts instead.
---
stem/descriptor/__init__.py | 2 +
stem/descriptor/collector.py | 43 +++++++++--------
stem/descriptor/networkstatus.py | 7 ++-
stem/descriptor/router_status_entry.py | 13 ++++++
test/unit/descriptor/collector.py | 52 ++++++++++++++++++++-
.../bridge-extra-infos-2019-03-cropped.tar | Bin 0 -> 15872 bytes
.../bridge-server-descriptors-2019-02-cropped.tar | Bin 0 -> 9216 bytes
.../collector/bridge-statuses-2019-05-cropped.tar | Bin 0 -> 467456 bytes
8 files changed, 93 insertions(+), 24 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index 2044c7bf..c099ca86 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -920,6 +920,8 @@ class Descriptor(object):
:returns: :class:`~stem.descriptor.TypeAnnotation` with our type information
"""
+ # TODO: populate this from the archive instead if available (so we have correct version numbers)
+
if self.TYPE_ANNOTATION_NAME is not None:
return TypeAnnotation(self.TYPE_ANNOTATION_NAME, 1, 0)
else:
diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index 99d02242..b7982426 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -146,25 +146,25 @@ def get_instance():
return SINGLETON_COLLECTOR
-def get_server_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3):
+def get_server_descriptors(start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3):
"""
Shorthand for
:func:`~stem.descriptor.collector.CollecTor.get_server_descriptors`
on our singleton instance.
"""
- for desc in get_instance().get_server_descriptors(start, end, cache_to, timeout, retries):
+ for desc in get_instance().get_server_descriptors(start, end, cache_to, bridge, timeout, retries):
yield desc
-def get_extrainfo_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3):
+def get_extrainfo_descriptors(start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3):
"""
Shorthand for
:func:`~stem.descriptor.collector.CollecTor.get_extrainfo_descriptors`
on our singleton instance.
"""
- for desc in get_instance().get_extrainfo_descriptors(start, end, cache_to, timeout, retries):
+ for desc in get_instance().get_extrainfo_descriptors(start, end, cache_to, bridge, timeout, retries):
yield desc
@@ -179,14 +179,14 @@ def get_microdescriptors(start = None, end = None, cache_to = None, timeout = No
yield desc
-def get_consensus(start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, timeout = None, retries = 3):
+def get_consensus(start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, bridge = False, timeout = None, retries = 3):
"""
Shorthand for
:func:`~stem.descriptor.collector.CollecTor.get_consensus`
on our singleton instance.
"""
- for desc in get_instance().get_consensus(start, end, cache_to, document_handler, version, microdescriptor, timeout, retries):
+ for desc in get_instance().get_consensus(start, end, cache_to, document_handler, version, microdescriptor, bridge, timeout, retries):
yield desc
@@ -413,7 +413,7 @@ class CollecTor(object):
self._cached_files = None
self._cached_index_at = 0
- def get_server_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
+ def get_server_descriptors(self, start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3):
"""
Provides server descriptors published during the given time range, sorted
oldest to newest.
@@ -422,6 +422,7 @@ class CollecTor(object):
:param datetime.datetime end: time range to end with
:param str cache_to: directory to cache archives into, if an archive is
available here it is not downloaded
+ :param bool bridge: standard descriptors if **False**, bridge if **True**
:param int timeout: timeout for downloading each individual archive when
the connection becomes idle, no timeout applied if **None**
:param int retires: maximum attempts to impose on a per-archive basis
@@ -433,13 +434,13 @@ class CollecTor(object):
:raises: :class:`~stem.DownloadFailed` if the download fails
"""
- # TODO: support bridge variants ('bridge-server-descriptor' type)
+ desc_type = 'server-descriptor' if not bridge else 'bridge-server-descriptor'
- for f in self.files('server-descriptor', start, end):
- for desc in f.read(cache_to, 'server-descriptor', timeout = timeout, retries = retries):
+ for f in self.files(desc_type, start, end):
+ for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries):
yield desc
- def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
+ def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3):
"""
Provides extrainfo descriptors published during the given time range,
sorted oldest to newest.
@@ -448,6 +449,7 @@ class CollecTor(object):
:param datetime.datetime end: time range to end with
:param str cache_to: directory to cache archives into, if an archive is
available here it is not downloaded
+ :param bool bridge: standard descriptors if **False**, bridge if **True**
:param int timeout: timeout for downloading each individual archive when
the connection becomes idle, no timeout applied if **None**
:param int retires: maximum attempts to impose on a per-archive basis
@@ -459,10 +461,10 @@ class CollecTor(object):
:raises: :class:`~stem.DownloadFailed` if the download fails
"""
- # TODO: support bridge variants ('bridge-extra-info' type)
+ desc_type = 'extra-info' if not bridge else 'bridge-extra-info'
- for f in self.files('extra-info', start, end):
- for desc in f.read(cache_to, 'extra-info', timeout = timeout, retries = retries):
+ for f in self.files(desc_type, start, end):
+ for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries):
yield desc
def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
@@ -498,7 +500,7 @@ class CollecTor(object):
for desc in f.read(cache_to, 'microdescriptor', timeout = timeout, retries = retries):
yield desc
- def get_consensus(self, start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, timeout = None, retries = 3):
+ def get_consensus(self, start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, bridge = False, timeout = None, retries = 3):
"""
Provides consensus router status entries published during the given time
range, sorted oldest to newest.
@@ -512,6 +514,7 @@ class CollecTor(object):
:param int version: consensus variant to retrieve (versions 2 or 3)
:param bool microdescriptor: provides the microdescriptor consensus if
**True**, standard consensus otherwise
+ :param bool bridge: standard descriptors if **False**, bridge if **True**
:param int timeout: timeout for downloading each individual archive when
the connection becomes idle, no timeout applied if **None**
:param int retires: maximum attempts to impose on a per-archive basis
@@ -523,20 +526,20 @@ class CollecTor(object):
:raises: :class:`~stem.DownloadFailed` if the download fails
"""
- if version == 3 and not microdescriptor:
+ if version == 3 and not microdescriptor and not bridge:
desc_type = 'network-status-consensus-3'
- elif version == 3 and microdescriptor:
+ elif version == 3 and microdescriptor and not bridge:
desc_type = 'network-status-microdesc-consensus-3'
- elif version == 2 and not microdescriptor:
+ elif version == 2 and not microdescriptor and not bridge:
desc_type = 'network-status-2'
+ elif bridge:
+ desc_type = 'bridge-network-status'
else:
if microdescriptor and version != 3:
raise ValueError('Only v3 microdescriptors are available (not version %s)' % version)
else:
raise ValueError('Only v2 and v3 router status entries are available (not version %s)' % version)
- # TODO: support bridge variants ('bridge-network-status' type)
-
for f in self.files(desc_type, start, end):
for desc in f.read(cache_to, desc_type, document_handler, timeout = timeout, retries = retries):
yield desc
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index b0589f2a..5f542c2f 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -93,6 +93,7 @@ from stem.descriptor import (
from stem.descriptor.router_status_entry import (
RouterStatusEntryV2,
+ RouterStatusEntryBridgeV2,
RouterStatusEntryV3,
RouterStatusEntryMicroV3,
)
@@ -322,7 +323,7 @@ def _parse_file(document_file, document_type = None, validate = False, is_microd
elif document_type == NetworkStatusDocumentV3:
router_type = RouterStatusEntryMicroV3 if is_microdescriptor else RouterStatusEntryV3
elif document_type == BridgeNetworkStatusDocument:
- document_type, router_type = BridgeNetworkStatusDocument, RouterStatusEntryV2
+ document_type, router_type = BridgeNetworkStatusDocument, RouterStatusEntryBridgeV2
elif document_type == DetachedSignature:
yield document_type(document_file.read(), validate, **kwargs)
return
@@ -1228,7 +1229,9 @@ class NetworkStatusDocumentV3(NetworkStatusDocument):
self._footer(document_file, validate)
def type_annotation(self):
- if not self.is_microdescriptor:
+ if isinstance(self, BridgeNetworkStatusDocument):
+ return TypeAnnotation('bridge-network-status', 1, 0)
+ elif not self.is_microdescriptor:
return TypeAnnotation('network-status-consensus-3' if not self.is_vote else 'network-status-vote-3', 1, 0)
else:
# Directory authorities do not issue a 'microdescriptor consensus' vote,
diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py
index 31913417..ce662b40 100644
--- a/stem/descriptor/router_status_entry.py
+++ b/stem/descriptor/router_status_entry.py
@@ -15,6 +15,8 @@ sources...
RouterStatusEntry - Common parent for router status entries
|- RouterStatusEntryV2 - Entry for a network status v2 document
+ | +- RouterStatusEntryBridgeV2 - Entry for a bridge flavored v2 document
+ |
|- RouterStatusEntryV3 - Entry for a network status v3 document
+- RouterStatusEntryMicroV3 - Entry for a microdescriptor flavored v3 document
"""
@@ -540,6 +542,17 @@ class RouterStatusEntryV2(RouterStatusEntry):
return ('r', 's', 'v')
+class RouterStatusEntryBridgeV2(RouterStatusEntryV2):
+ """
+ Information about an individual router stored within a bridge flavored
+ version 2 network status document.
+
+ .. versionadded:: 1.8.0
+ """
+
+ TYPE_ANNOTATION_NAME = 'bridge-network-status'
+
+
class RouterStatusEntryV3(RouterStatusEntry):
"""
Information about an individual router stored within a version 3 network
diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py
index 7d1f0205..3ab06f03 100644
--- a/test/unit/descriptor/collector.py
+++ b/test/unit/descriptor/collector.py
@@ -212,6 +212,21 @@ class TestCollector(unittest.TestCase):
@patch('stem.util.connection.download')
@patch('stem.descriptor.collector.CollecTor.files')
+ def test_reading_bridge_server_descriptors(self, files_mock, download_mock):
+ with open(get_resource('collector/bridge-server-descriptors-2019-02-cropped.tar'), 'rb') as archive:
+ download_mock.return_value = archive.read()
+
+ files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/server-descriptors/bridge-server-descriptors-2019-02.tar', 12345, '2016-09-04 09:21')]
+
+ descriptors = list(stem.descriptor.collector.get_server_descriptors(bridge = True))
+ self.assertEqual(4, len(descriptors))
+
+ f = descriptors[0]
+ self.assertEqual('BridgeDescriptor', type(f).__name__)
+ self.assertEqual('E90D1DE12B930DEC3F3E1127AAA25E47430CD3F4', f.fingerprint)
+
+ @patch('stem.util.connection.download')
+ @patch('stem.descriptor.collector.CollecTor.files')
def test_reading_extrainfo_descriptors(self, files_mock, download_mock):
with open(get_resource('collector/extra-infos-2019-04-cropped.tar'), 'rb') as archive:
download_mock.return_value = archive.read()
@@ -227,6 +242,21 @@ class TestCollector(unittest.TestCase):
@patch('stem.util.connection.download')
@patch('stem.descriptor.collector.CollecTor.files')
+ def test_reading_bridge_extrainfo_descriptors(self, files_mock, download_mock):
+ with open(get_resource('collector/bridge-extra-infos-2019-03-cropped.tar'), 'rb') as archive:
+ download_mock.return_value = archive.read()
+
+ files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2019-03.tar', 12345, '2016-09-04 09:21')]
+
+ descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors(bridge = True))
+ self.assertEqual(6, len(descriptors))
+
+ f = descriptors[0]
+ self.assertEqual('BridgeExtraInfoDescriptor', type(f).__name__)
+ self.assertEqual('A0187027648A392C6AC413B66F7CD25DD001BF76', f.fingerprint)
+
+ @patch('stem.util.connection.download')
+ @patch('stem.descriptor.collector.CollecTor.files')
def test_reading_microdescriptors(self, files_mock, download_mock):
with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive:
download_mock.return_value = archive.read()
@@ -267,14 +297,32 @@ class TestCollector(unittest.TestCase):
self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(version = 2))))
self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(microdescriptor = True))))
- # but the microdescriptor archive *does* have microdescriptor consensuses
-
+ @patch('stem.util.connection.download')
+ @patch('stem.descriptor.collector.CollecTor.files')
+ def test_reading_microdescriptor_consensus(self, files_mock, download_mock):
with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive:
download_mock.return_value = archive.read()
+ files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/microdescs/microdescs-2019-05.tar', 12345, '2016-09-04 09:21')]
+
descriptors = list(stem.descriptor.collector.get_consensus(microdescriptor = True))
self.assertEqual(556, len(descriptors))
f = descriptors[0]
self.assertEqual('RouterStatusEntryMicroV3', type(f).__name__)
self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint)
+
+ @patch('stem.util.connection.download')
+ @patch('stem.descriptor.collector.CollecTor.files')
+ def test_reading_bridge_consensus(self, files_mock, download_mock):
+ with open(get_resource('collector/bridge-statuses-2019-05-cropped.tar'), 'rb') as archive:
+ download_mock.return_value = archive.read()
+
+ files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/microdescs/bridge-statuses-2019-05.tar', 12345, '2016-09-04 09:21')]
+
+ descriptors = list(stem.descriptor.collector.get_consensus(bridge = True))
+ self.assertEqual(2593, len(descriptors))
+
+ f = descriptors[0]
+ self.assertEqual('RouterStatusEntryBridgeV2', type(f).__name__)
+ self.assertEqual('0035EA2A61E28D395F080ACA2244539490E70950', f.fingerprint)
diff --git a/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar b/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar
new file mode 100644
index 00000000..db9f3f7b
Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar differ
diff --git a/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar b/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar
new file mode 100644
index 00000000..4d0255f3
Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar differ
diff --git a/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar b/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar
new file mode 100644
index 00000000..953d2a86
Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar differ
