August 2019 - tor-commits - lists.torproject.org

[stem/master] Use DownloadFailed in directory and manual modules
by atagar＠torproject.org 18 Aug '19

18 Aug '19

commit b0a1a878c34a45bd85f304a17e38a834ee074d41 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 4 17:38:40 2019 -0700 Use DownloadFailed in directory and manual modules Ah. No need to wait for stem 2.x since DownloadFailed is an IOError subclass. Our from_remote() functions raise errors aside from download failures so the overall function signatures are unchanged. --- stem/directory.py | 29 +++++++++++++++++------------ stem/manual.py | 8 ++++---- test/unit/directory/authority.py | 3 ++- test/unit/directory/fallback.py | 3 ++- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/stem/directory.py b/stem/directory.py index 075235c9..9e397d88 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -40,7 +40,9 @@ as follows... import os import re +import sys +import stem import stem.util import stem.util.conf @@ -215,8 +217,6 @@ class Directory(object): :raises: **IOError** if unable to retrieve the fallback directories """ - # TODO: change IOError to DownloadFailed in stem 2.x - raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') def __hash__(self): @@ -266,11 +266,13 @@ class Authority(Directory): def from_remote(timeout = 60): try: lines = str_tools._to_unicode(urllib.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() - except Exception as exc: - raise IOError("Unable to download tor's directory authorities from %s: %s" % (GITWEB_AUTHORITY_URL, exc)) - if not lines: - raise IOError('%s did not have any content' % GITWEB_AUTHORITY_URL) + if not lines: + raise IOError('no content') + except: + exc, stacktrace = sys.exc_info()[1:3] + message = "Unable to download tor's directory authorities from %s: %s" % (GITWEB_AUTHORITY_URL, exc) + raise stem.DownloadFailed(GITWEB_AUTHORITY_URL, exc, stacktrace, message) # Entries look like... # @@ -412,16 +414,19 @@ class Fallback(Directory): def from_remote(timeout = 60): try: lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() - except Exception as exc: - raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc)) - if not lines: - raise IOError('%s did not have any content' % GITWEB_FALLBACK_URL) - elif lines[0] != '/* type=fallback */': - raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_URL) + if not lines: + raise IOError('no content') + except: + exc, stacktrace = sys.exc_info()[1:3] + message = "Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc) + raise stem.DownloadFailed(GITWEB_FALLBACK_URL, exc, stacktrace, message) # header metadata + if lines[0] != '/* type=fallback */': + raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_URL) + header = {} for line in Fallback._pop_section(lines): diff --git a/stem/manual.py b/stem/manual.py index 729fb204..24596851 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -53,6 +53,7 @@ import shutil import sys import tempfile +import stem import stem.prereq import stem.util import stem.util.conf @@ -296,8 +297,6 @@ def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, :raises: **IOError** if unable to retrieve the manual """ - # TODO: change IOError to DownloadFailed in stem 2.x - if not path and not file_handle: raise ValueError("Either the path or file_handle we're saving to must be provided") elif not stem.util.system.is_available('a2x'): @@ -313,8 +312,9 @@ def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, request = urllib.urlopen(url, timeout = timeout) shutil.copyfileobj(request, asciidoc_file) except: - exc = sys.exc_info()[1] - raise IOError("Unable to download tor's manual from %s to %s: %s" % (url, asciidoc_path, exc)) + exc, stacktrace = sys.exc_info()[1:3] + message = "Unable to download tor's manual from %s to %s: %s" % (url, asciidoc_path, exc) + raise stem.DownloadFailed(url, exc, stacktrace, message) try: stem.util.system.call('a2x -f manpage %s' % asciidoc_path) diff --git a/test/unit/directory/authority.py b/test/unit/directory/authority.py index 56bd6ec8..1c2a86b4 100644 --- a/test/unit/directory/authority.py +++ b/test/unit/directory/authority.py @@ -5,6 +5,7 @@ Unit tests for stem.directory.Authority. import io import unittest +import stem import stem.directory import stem.prereq @@ -78,4 +79,4 @@ class TestAuthority(unittest.TestCase): @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) def test_from_remote_empty(self): - self.assertRaisesRegexp(IOError, 'did not have any content', stem.directory.Authority.from_remote) + self.assertRaisesRegexp(stem.DownloadFailed, 'no content', stem.directory.Authority.from_remote) diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py index 2423932f..06b4510d 100644 --- a/test/unit/directory/fallback.py +++ b/test/unit/directory/fallback.py @@ -7,6 +7,7 @@ import re import tempfile import unittest +import stem import stem.directory import stem.util.conf @@ -123,7 +124,7 @@ class TestFallback(unittest.TestCase): @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) def test_from_remote_empty(self): - self.assertRaisesRegexp(IOError, 'did not have any content', stem.directory.Fallback.from_remote) + self.assertRaisesRegexp(stem.DownloadFailed, 'no content', stem.directory.Fallback.from_remote) @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'\n'.join(FALLBACK_GITWEB_CONTENT.splitlines()[1:])))) def test_from_remote_no_header(self):

1 0

[stem/master] Pydoc class table
by atagar＠torproject.org 18 Aug '19

18 Aug '19

commit 587fbcf51d9a4f21db5a8ecaea3dcb094a2a4045 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Aug 5 22:15:48 2019 -0700 Pydoc class table --- stem/descriptor/collector.py | 19 +++++++++++++++++-- stem/descriptor/remote.py | 7 +++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 57f9ce0a..04f45bad 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -16,7 +16,7 @@ With this you can either download and read directly from CollecTor... import datetime import stem.descriptor.collector - yesterday = datetime.datetime.today() - datetime.timedelta(1) + yesterday = datetime.datetime.utcnow() - datetime.timedelta(days = 1) # provide yesterday's exits @@ -33,7 +33,7 @@ With this you can either download and read directly from CollecTor... import stem.descriptor import stem.descriptor.collector - yesterday = datetime.datetime.today() - datetime.timedelta(1) + yesterday = datetime.datetime.utcnow() - datetime.timedelta(days = 1) path = os.path.expanduser('~/descriptor_cache/server_desc_today') with open(path, 'wb') as cache_file: @@ -46,6 +46,21 @@ With this you can either download and read directly from CollecTor... if desc.exit_policy.is_exiting_allowed(): print(' %s (%s)' % (desc.nickname, desc.fingerprint)) +:: + + get_instance - Provides a singleton CollecTor used for... + +- get_server_descriptors - published server descriptors + + File - Individual file residing within CollecTor + |- read - provides descriptors from this file + +- download - download this file to disk + + CollecTor - Downloader for descriptors from CollecTor + |- get_server_descriptors - published server descriptors + | + |- index - metadata for content available from CollecTor + +- files - files available from CollecTor + .. versionadded:: 1.8.0 """ diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 4f29a2e5..82634513 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -48,7 +48,10 @@ content. For example... |- their_server_descriptor - provides the server descriptor of the relay we download from |- get_server_descriptors - provides present server descriptors |- get_extrainfo_descriptors - provides present extrainfo descriptors - +- get_consensus - provides the present consensus or router status entries + |- get_microdescriptors - provides present microdescriptors with the given digests + |- get_consensus - provides the present consensus or router status entries + |- get_bandwidth_file - provides bandwidth heuristics used to make the next consensus + +- get_detached_signatures - authority signatures used to make the next consensus Query - Asynchronous request to download tor descriptors |- start - issues the query if it isn't already running @@ -63,7 +66,7 @@ content. For example... |- get_consensus - provides the present consensus or router status entries |- get_vote - provides an authority's vote for the next consensus |- get_key_certificates - provides present authority key certificates - |- get_bandwidth_file - provies bandwidth heuristics used to make the next consensus + |- get_bandwidth_file - provides bandwidth heuristics used to make the next consensus |- get_detached_signatures - authority signatures used to make the next consensus +- query - request an arbitrary descriptor resource

1 0

[stem/master] Explain why archives contain few microdescriptors
by atagar＠torproject.org 18 Aug '19

18 Aug '19

commit e66d29ac772ae1143e832c0e38b96a968b80735f Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Aug 10 14:15:39 2019 -0700 Explain why archives contain few microdescriptors Thanks to Karsten for the explanation! I'm probably not the only person that will be confused why CollecTor contains so few microdescriptors on an hourly basis so explaining this in our pydocs. --- stem/descriptor/collector.py | 11 ++++++++++- test/integ/descriptor/collector.py | 3 --- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 3867fa3d..85eafec6 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -450,7 +450,16 @@ class CollecTor(object): def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): """ Provides microdescriptors published during the given time range, - sorted oldest to newest. + sorted oldest to newest. Unlike server/extrainfo descriptors, + microdescriptors change very infrequently... + + :: + + "Microdescriptors are expected to be relatively static and only change + about once per week." -dir-spec section 3.3 + + CollecTor archives only contain microdescriptors that *change*, so hourly + tarballs often contain very few. :param datetime.datetime start: time range to begin with :param datetime.datetime end: time range to end with diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py index db57df5e..d38ce7d1 100644 --- a/test/integ/descriptor/collector.py +++ b/test/integ/descriptor/collector.py @@ -56,9 +56,6 @@ class TestCollector(unittest.TestCase): def test_downloading_microdescriptors(self): recent_descriptors = list(stem.descriptor.collector.get_microdescriptors(start = RECENT)) - # TODO: I'm unsure why these counts differ so much from server/extrainfo - # descriptors. Checking with Karsten. - if not (300 < len(recent_descriptors) < 800): self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) # 23 on 8/7/19

1 0

[stem/master] Download method for key certificates
by atagar＠torproject.org 18 Aug '19

18 Aug '19

commit 33b9c79b8e9a7f30fc4c81af8dfc2bd2e1319cf3 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 15 14:14:37 2019 -0700 Download method for key certificates --- stem/descriptor/collector.py | 51 ++++++++++++++++++--- test/unit/descriptor/collector.py | 15 ++++++ .../descriptor/data/collector/certs-cropped.tar | Bin 0 -> 14336 bytes 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index b7982426..9a238ecf 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -52,7 +52,8 @@ With this you can either download and read directly from CollecTor... |- get_server_descriptors - published server descriptors |- get_extrainfo_descriptors - published extrainfo descriptors |- get_microdescriptors - published microdescriptors - +- get_consensus - published router status entries + |- get_consensus - published router status entries + +- get_key_certificates - authority key certificates File - Individual file residing within CollecTor |- read - provides descriptors from this file @@ -63,6 +64,7 @@ With this you can either download and read directly from CollecTor... |- get_extrainfo_descriptors - published extrainfo descriptors |- get_microdescriptors - published microdescriptors |- get_consensus - published router status entries + |- get_key_certificates - authority key certificates | |- index - metadata for content available from CollecTor +- files - files available from CollecTor @@ -190,6 +192,17 @@ def get_consensus(start = None, end = None, cache_to = None, document_handler = yield desc +def get_key_certificates(start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Shorthand for + :func:`~stem.descriptor.collector.CollecTor.get_key_certificates` + on our singleton instance. + """ + + for desc in get_instance().get_key_certificates(start, end, cache_to, timeout, retries): + yield desc + + class File(object): """ File within CollecTor. @@ -244,7 +257,7 @@ class File(object): which to parse a :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` :param int timeout: timeout when connection becomes idle, no timeout applied if **None** - :param int retires: maximum attempts to impose + :param int retries: maximum attempts to impose :returns: iterator for :class:`~stem.descriptor.__init__.Descriptor` instances in the file @@ -297,7 +310,7 @@ class File(object): :param bool decompress: decompress written file :param int timeout: timeout when connection becomes idle, no timeout applied if **None** - :param int retires: maximum attempts to impose + :param int retries: maximum attempts to impose :returns: **str** with the path we downloaded to @@ -425,7 +438,7 @@ class CollecTor(object): :param bool bridge: standard descriptors if **False**, bridge if **True** :param int timeout: timeout for downloading each individual archive when the connection becomes idle, no timeout applied if **None** - :param int retires: maximum attempts to impose on a per-archive basis + :param int retries: maximum attempts to impose on a per-archive basis :returns: **iterator** of :class:`~stem.descriptor.server_descriptor.ServerDescriptor` for the @@ -452,7 +465,7 @@ class CollecTor(object): :param bool bridge: standard descriptors if **False**, bridge if **True** :param int timeout: timeout for downloading each individual archive when the connection becomes idle, no timeout applied if **None** - :param int retires: maximum attempts to impose on a per-archive basis + :param int retries: maximum attempts to impose on a per-archive basis :returns: **iterator** of :class:`~stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor` @@ -487,7 +500,7 @@ class CollecTor(object): available here it is not downloaded :param int timeout: timeout for downloading each individual archive when the connection becomes idle, no timeout applied if **None** - :param int retires: maximum attempts to impose on a per-archive basis + :param int retries: maximum attempts to impose on a per-archive basis :returns: **iterator** of :class:`~stem.descriptor.microdescriptor.Microdescriptor @@ -517,7 +530,7 @@ class CollecTor(object): :param bool bridge: standard descriptors if **False**, bridge if **True** :param int timeout: timeout for downloading each individual archive when the connection becomes idle, no timeout applied if **None** - :param int retires: maximum attempts to impose on a per-archive basis + :param int retries: maximum attempts to impose on a per-archive basis :returns: **iterator** of :class:`~stem.descriptor.router_status_entry.RouterStatusEntry` @@ -544,6 +557,30 @@ class CollecTor(object): for desc in f.read(cache_to, desc_type, document_handler, timeout = timeout, retries = retries): yield desc + def get_key_certificates(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Directory authority key certificates for the given time range, + sorted oldest to newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int timeout: timeout for downloading each individual archive when + the connection becomes idle, no timeout applied if **None** + :param int retries: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.networkstatus.KeyCertificate + for the given time range + + :raises: :class:`~stem.DownloadFailed` if the download fails + """ + + for f in self.files('dir-key-certificate-3', start, end): + for desc in f.read(cache_to, 'dir-key-certificate-3', timeout = timeout, retries = retries): + yield desc + def index(self, compression = 'best'): """ Provides the archives available in CollecTor. diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 3ab06f03..d6a3aee8 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -326,3 +326,18 @@ class TestCollector(unittest.TestCase): f = descriptors[0] self.assertEqual('RouterStatusEntryBridgeV2', type(f).__name__) self.assertEqual('0035EA2A61E28D395F080ACA2244539490E70950', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_key_certificates(self, files_mock, download_mock): + with open(get_resource('collector/certs-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/certs.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_key_certificates()) + self.assertEqual(5, len(descriptors)) + + f = descriptors[0] + self.assertEqual('KeyCertificate', type(f).__name__) + self.assertEqual('14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', f.fingerprint) diff --git a/test/unit/descriptor/data/collector/certs-cropped.tar b/test/unit/descriptor/data/collector/certs-cropped.tar new file mode 100644 index 00000000..9320b65f Binary files /dev/null and b/test/unit/descriptor/data/collector/certs-cropped.tar differ

1 0

[stem/master] Include both method/function for get_server_descriptors
by atagar＠torproject.org 18 Aug '19

18 Aug '19

commit 916f492b3bf4bba2903eeaa203fb073f85efaa96 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Aug 5 22:16:27 2019 -0700 Include both method/function for get_server_descriptors Like stem.descriptor.remote planning to include both function and method counterparts for descriptor fetching. --- stem/descriptor/collector.py | 48 ++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 04f45bad..e6e605f0 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -141,27 +141,13 @@ def get_instance(): def get_server_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3): """ - Provides server descriptors for the given time range, sorted oldest to - newest. - - :param datetime.datetime start: time range to begin with - :param datetime.datetime end: time range to end with - :param str cache_to: directory to cache archives into, if an archive is - available here it is not downloaded - :param int timeout: timeout for downloading each individual archive when the - connection becomes idle, no timeout applied if **None** - :param int retires: maximum attempts to impose on a per-archive basis - - :returns: **iterator** of - :class:`~stem.descriptor.server_descriptor.ServerDescriptor` for the given - time range - - :raises: :class:`~stem.DownloadFailed` if the download fails + Shorthand for + :func:`~stem.descriptor.collector.CollecTor.get_server_descriptors` + on our singleton instance. """ - for f in get_instance().files('server-descriptor', start, end): - for desc in f.read(cache_to, timeout = timeout, retries = retries): - yield desc + for desc in get_instance().get_server_descriptors(start, end, cache_to, timeout, retries): + yield desc class File(object): @@ -383,6 +369,30 @@ class CollecTor(object): self._cached_files = None self._cached_index_at = 0 + def get_server_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Provides server descriptors published during the given time range, sorted + oldest to newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int timeout: timeout for downloading each individual archive when + the connection becomes idle, no timeout applied if **None** + :param int retires: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.server_descriptor.ServerDescriptor` for the + given time range + + :raises: :class:`~stem.DownloadFailed` if the download fails + """ + + for f in self.files('server-descriptor', start, end): + for desc in f.read(cache_to, timeout = timeout, retries = retries): + yield desc + def index(self, compression = 'best'): """ Provides the archives available in CollecTor.

1 0

[stem/master] Integ test server descriptor downloading
by atagar＠torproject.org 18 Aug '19

18 Aug '19

commit cb7147a502d6b2eb51b65540eaeab062bcfdf013 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Aug 5 22:18:07 2019 -0700 Integ test server descriptor downloading --- test/integ/descriptor/collector.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py index 6a0ec5ac..040c6b72 100644 --- a/test/integ/descriptor/collector.py +++ b/test/integ/descriptor/collector.py @@ -2,12 +2,16 @@ Integration tests for stem.descriptor.collector. """ +import datetime import unittest import test.require +import stem.descriptor.collector + from stem.descriptor import Compression -from stem.descriptor.collector import CollecTor + +RECENT = datetime.datetime.utcnow() - datetime.timedelta(minutes = 60) class TestCollector(unittest.TestCase): @@ -31,11 +35,20 @@ class TestCollector(unittest.TestCase): def test_index_lzma(self): self._test_index(Compression.LZMA) + @test.require.only_run_once + @test.require.online + def test_downloading_server_descriptors(self): + recent_descriptors = list(stem.descriptor.collector.get_server_descriptors(start = RECENT)) + + if not (300 < len(recent_descriptors) < 800): + self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) # 584 on 8/5/19 + def _test_index(self, compression): if compression and not compression.available: self.skipTest('(%s unavailable)' % compression) + return - collector = CollecTor() + collector = stem.descriptor.collector.CollecTor() index = collector.index(compression = compression) self.assertEqual('https://collector.torproject.org', index['path'])

1 0

[stem/master] Bridge descriptor support
by atagar＠torproject.org 18 Aug '19

18 Aug '19

commit e14d74473ff7319298a920129db6037079ab3652 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 15 13:53:35 2019 -0700 Bridge descriptor support Adding a 'bridge' argument to descriptor download methods to retrieve bridge counterparts instead. --- stem/descriptor/__init__.py | 2 + stem/descriptor/collector.py | 43 +++++++++-------- stem/descriptor/networkstatus.py | 7 ++- stem/descriptor/router_status_entry.py | 13 ++++++ test/unit/descriptor/collector.py | 52 ++++++++++++++++++++- .../bridge-extra-infos-2019-03-cropped.tar | Bin 0 -> 15872 bytes .../bridge-server-descriptors-2019-02-cropped.tar | Bin 0 -> 9216 bytes .../collector/bridge-statuses-2019-05-cropped.tar | Bin 0 -> 467456 bytes 8 files changed, 93 insertions(+), 24 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 2044c7bf..c099ca86 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -920,6 +920,8 @@ class Descriptor(object): :returns: :class:`~stem.descriptor.TypeAnnotation` with our type information """ + # TODO: populate this from the archive instead if available (so we have correct version numbers) + if self.TYPE_ANNOTATION_NAME is not None: return TypeAnnotation(self.TYPE_ANNOTATION_NAME, 1, 0) else: diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 99d02242..b7982426 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -146,25 +146,25 @@ def get_instance(): return SINGLETON_COLLECTOR -def get_server_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3): +def get_server_descriptors(start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3): """ Shorthand for :func:`~stem.descriptor.collector.CollecTor.get_server_descriptors` on our singleton instance. """ - for desc in get_instance().get_server_descriptors(start, end, cache_to, timeout, retries): + for desc in get_instance().get_server_descriptors(start, end, cache_to, bridge, timeout, retries): yield desc -def get_extrainfo_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3): +def get_extrainfo_descriptors(start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3): """ Shorthand for :func:`~stem.descriptor.collector.CollecTor.get_extrainfo_descriptors` on our singleton instance. """ - for desc in get_instance().get_extrainfo_descriptors(start, end, cache_to, timeout, retries): + for desc in get_instance().get_extrainfo_descriptors(start, end, cache_to, bridge, timeout, retries): yield desc @@ -179,14 +179,14 @@ def get_microdescriptors(start = None, end = None, cache_to = None, timeout = No yield desc -def get_consensus(start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, timeout = None, retries = 3): +def get_consensus(start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, bridge = False, timeout = None, retries = 3): """ Shorthand for :func:`~stem.descriptor.collector.CollecTor.get_consensus` on our singleton instance. """ - for desc in get_instance().get_consensus(start, end, cache_to, document_handler, version, microdescriptor, timeout, retries): + for desc in get_instance().get_consensus(start, end, cache_to, document_handler, version, microdescriptor, bridge, timeout, retries): yield desc @@ -413,7 +413,7 @@ class CollecTor(object): self._cached_files = None self._cached_index_at = 0 - def get_server_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + def get_server_descriptors(self, start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3): """ Provides server descriptors published during the given time range, sorted oldest to newest. @@ -422,6 +422,7 @@ class CollecTor(object): :param datetime.datetime end: time range to end with :param str cache_to: directory to cache archives into, if an archive is available here it is not downloaded + :param bool bridge: standard descriptors if **False**, bridge if **True** :param int timeout: timeout for downloading each individual archive when the connection becomes idle, no timeout applied if **None** :param int retires: maximum attempts to impose on a per-archive basis @@ -433,13 +434,13 @@ class CollecTor(object): :raises: :class:`~stem.DownloadFailed` if the download fails """ - # TODO: support bridge variants ('bridge-server-descriptor' type) + desc_type = 'server-descriptor' if not bridge else 'bridge-server-descriptor' - for f in self.files('server-descriptor', start, end): - for desc in f.read(cache_to, 'server-descriptor', timeout = timeout, retries = retries): + for f in self.files(desc_type, start, end): + for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries): yield desc - def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3): """ Provides extrainfo descriptors published during the given time range, sorted oldest to newest. @@ -448,6 +449,7 @@ class CollecTor(object): :param datetime.datetime end: time range to end with :param str cache_to: directory to cache archives into, if an archive is available here it is not downloaded + :param bool bridge: standard descriptors if **False**, bridge if **True** :param int timeout: timeout for downloading each individual archive when the connection becomes idle, no timeout applied if **None** :param int retires: maximum attempts to impose on a per-archive basis @@ -459,10 +461,10 @@ class CollecTor(object): :raises: :class:`~stem.DownloadFailed` if the download fails """ - # TODO: support bridge variants ('bridge-extra-info' type) + desc_type = 'extra-info' if not bridge else 'bridge-extra-info' - for f in self.files('extra-info', start, end): - for desc in f.read(cache_to, 'extra-info', timeout = timeout, retries = retries): + for f in self.files(desc_type, start, end): + for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries): yield desc def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): @@ -498,7 +500,7 @@ class CollecTor(object): for desc in f.read(cache_to, 'microdescriptor', timeout = timeout, retries = retries): yield desc - def get_consensus(self, start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, timeout = None, retries = 3): + def get_consensus(self, start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, bridge = False, timeout = None, retries = 3): """ Provides consensus router status entries published during the given time range, sorted oldest to newest. @@ -512,6 +514,7 @@ class CollecTor(object): :param int version: consensus variant to retrieve (versions 2 or 3) :param bool microdescriptor: provides the microdescriptor consensus if **True**, standard consensus otherwise + :param bool bridge: standard descriptors if **False**, bridge if **True** :param int timeout: timeout for downloading each individual archive when the connection becomes idle, no timeout applied if **None** :param int retires: maximum attempts to impose on a per-archive basis @@ -523,20 +526,20 @@ class CollecTor(object): :raises: :class:`~stem.DownloadFailed` if the download fails """ - if version == 3 and not microdescriptor: + if version == 3 and not microdescriptor and not bridge: desc_type = 'network-status-consensus-3' - elif version == 3 and microdescriptor: + elif version == 3 and microdescriptor and not bridge: desc_type = 'network-status-microdesc-consensus-3' - elif version == 2 and not microdescriptor: + elif version == 2 and not microdescriptor and not bridge: desc_type = 'network-status-2' + elif bridge: + desc_type = 'bridge-network-status' else: if microdescriptor and version != 3: raise ValueError('Only v3 microdescriptors are available (not version %s)' % version) else: raise ValueError('Only v2 and v3 router status entries are available (not version %s)' % version) - # TODO: support bridge variants ('bridge-network-status' type) - for f in self.files(desc_type, start, end): for desc in f.read(cache_to, desc_type, document_handler, timeout = timeout, retries = retries): yield desc diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index b0589f2a..5f542c2f 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -93,6 +93,7 @@ from stem.descriptor import ( from stem.descriptor.router_status_entry import ( RouterStatusEntryV2, + RouterStatusEntryBridgeV2, RouterStatusEntryV3, RouterStatusEntryMicroV3, ) @@ -322,7 +323,7 @@ def _parse_file(document_file, document_type = None, validate = False, is_microd elif document_type == NetworkStatusDocumentV3: router_type = RouterStatusEntryMicroV3 if is_microdescriptor else RouterStatusEntryV3 elif document_type == BridgeNetworkStatusDocument: - document_type, router_type = BridgeNetworkStatusDocument, RouterStatusEntryV2 + document_type, router_type = BridgeNetworkStatusDocument, RouterStatusEntryBridgeV2 elif document_type == DetachedSignature: yield document_type(document_file.read(), validate, **kwargs) return @@ -1228,7 +1229,9 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): self._footer(document_file, validate) def type_annotation(self): - if not self.is_microdescriptor: + if isinstance(self, BridgeNetworkStatusDocument): + return TypeAnnotation('bridge-network-status', 1, 0) + elif not self.is_microdescriptor: return TypeAnnotation('network-status-consensus-3' if not self.is_vote else 'network-status-vote-3', 1, 0) else: # Directory authorities do not issue a 'microdescriptor consensus' vote, diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index 31913417..ce662b40 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -15,6 +15,8 @@ sources... RouterStatusEntry - Common parent for router status entries |- RouterStatusEntryV2 - Entry for a network status v2 document + | +- RouterStatusEntryBridgeV2 - Entry for a bridge flavored v2 document + | |- RouterStatusEntryV3 - Entry for a network status v3 document +- RouterStatusEntryMicroV3 - Entry for a microdescriptor flavored v3 document """ @@ -540,6 +542,17 @@ class RouterStatusEntryV2(RouterStatusEntry): return ('r', 's', 'v') +class RouterStatusEntryBridgeV2(RouterStatusEntryV2): + """ + Information about an individual router stored within a bridge flavored + version 2 network status document. + + .. versionadded:: 1.8.0 + """ + + TYPE_ANNOTATION_NAME = 'bridge-network-status' + + class RouterStatusEntryV3(RouterStatusEntry): """ Information about an individual router stored within a version 3 network diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 7d1f0205..3ab06f03 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -212,6 +212,21 @@ class TestCollector(unittest.TestCase): @patch('stem.util.connection.download') @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_bridge_server_descriptors(self, files_mock, download_mock): + with open(get_resource('collector/bridge-server-descriptors-2019-02-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/server-descriptors/bridge-server-descriptors-2019-02.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_server_descriptors(bridge = True)) + self.assertEqual(4, len(descriptors)) + + f = descriptors[0] + self.assertEqual('BridgeDescriptor', type(f).__name__) + self.assertEqual('E90D1DE12B930DEC3F3E1127AAA25E47430CD3F4', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') def test_reading_extrainfo_descriptors(self, files_mock, download_mock): with open(get_resource('collector/extra-infos-2019-04-cropped.tar'), 'rb') as archive: download_mock.return_value = archive.read() @@ -227,6 +242,21 @@ class TestCollector(unittest.TestCase): @patch('stem.util.connection.download') @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_bridge_extrainfo_descriptors(self, files_mock, download_mock): + with open(get_resource('collector/bridge-extra-infos-2019-03-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2019-03.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors(bridge = True)) + self.assertEqual(6, len(descriptors)) + + f = descriptors[0] + self.assertEqual('BridgeExtraInfoDescriptor', type(f).__name__) + self.assertEqual('A0187027648A392C6AC413B66F7CD25DD001BF76', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') def test_reading_microdescriptors(self, files_mock, download_mock): with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive: download_mock.return_value = archive.read() @@ -267,14 +297,32 @@ class TestCollector(unittest.TestCase): self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(version = 2)))) self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(microdescriptor = True)))) - # but the microdescriptor archive *does* have microdescriptor consensuses - + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_microdescriptor_consensus(self, files_mock, download_mock): with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive: download_mock.return_value = archive.read() + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/microdescs/microdescs-2019-05.tar', 12345, '2016-09-04 09:21')] + descriptors = list(stem.descriptor.collector.get_consensus(microdescriptor = True)) self.assertEqual(556, len(descriptors)) f = descriptors[0] self.assertEqual('RouterStatusEntryMicroV3', type(f).__name__) self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_bridge_consensus(self, files_mock, download_mock): + with open(get_resource('collector/bridge-statuses-2019-05-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/microdescs/bridge-statuses-2019-05.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_consensus(bridge = True)) + self.assertEqual(2593, len(descriptors)) + + f = descriptors[0] + self.assertEqual('RouterStatusEntryBridgeV2', type(f).__name__) + self.assertEqual('0035EA2A61E28D395F080ACA2244539490E70950', f.fingerprint) diff --git a/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar b/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar new file mode 100644 index 00000000..db9f3f7b Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar differ diff --git a/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar b/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar new file mode 100644 index 00000000..4d0255f3 Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar differ diff --git a/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar b/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar new file mode 100644 index 00000000..953d2a86 Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar differ

1 0

[stem/master] Plan to rename hidden_service_descriptor module
by atagar＠torproject.org 18 Aug '19

18 Aug '19

commit f13b5da021ed2caee99b249e20a01e66f9b359aa Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Aug 9 16:31:27 2019 -0700 Plan to rename hidden_service_descriptor module The 'descriptor' suffix is redundant with the directory. I try to keep module names short due to import statements, so this is a no-brainer. --- stem/descriptor/hidden_service_descriptor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stem/descriptor/hidden_service_descriptor.py b/stem/descriptor/hidden_service_descriptor.py index 1b7f2cc3..99d6414e 100644 --- a/stem/descriptor/hidden_service_descriptor.py +++ b/stem/descriptor/hidden_service_descriptor.py @@ -21,6 +21,9 @@ These are only available through the Controller's .. versionadded:: 1.4.0 """ +# TODO: In stem 2.x rename this module to 'hidden_service' (ie, drop the +# redundant '_descriptor' suffix). + import base64 import binascii import collections

1 0

[stem/master] Download method for microdescriptors
by atagar＠torproject.org 18 Aug '19

18 Aug '19

commit 60d714a33ca055fcefdad26493b4787483b1d71a Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Aug 7 18:36:53 2019 -0700 Download method for microdescriptors --- stem/descriptor/collector.py | 39 +++++++++++++++++++++++++++++++++++++- test/integ/descriptor/collector.py | 13 ++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 82a213f3..c8c4a1b7 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -50,7 +50,8 @@ With this you can either download and read directly from CollecTor... get_instance - Provides a singleton CollecTor used for... |- get_server_descriptors - published server descriptors - +- get_extrainfo_descriptors - published extrainfo descriptors + |- get_extrainfo_descriptors - published extrainfo descriptors + +- get_microdescriptors - published microdescriptors File - Individual file residing within CollecTor |- read - provides descriptors from this file @@ -59,6 +60,7 @@ With this you can either download and read directly from CollecTor... CollecTor - Downloader for descriptors from CollecTor |- get_server_descriptors - published server descriptors |- get_extrainfo_descriptors - published extrainfo descriptors + |- get_microdescriptors - published microdescriptors | |- index - metadata for content available from CollecTor +- files - files available from CollecTor @@ -163,6 +165,17 @@ def get_extrainfo_descriptors(start = None, end = None, cache_to = None, timeout yield desc +def get_microdescriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Shorthand for + :func:`~stem.descriptor.collector.CollecTor.get_microdescriptors` + on our singleton instance. + """ + + for desc in get_instance().get_microdescriptors(start, end, cache_to, timeout, retries): + yield desc + + class File(object): """ File within CollecTor. @@ -430,6 +443,30 @@ class CollecTor(object): for desc in f.read(cache_to, timeout = timeout, retries = retries): yield desc + def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Provides microdescriptors published during the given time range, + sorted oldest to newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int timeout: timeout for downloading each individual archive when + the connection becomes idle, no timeout applied if **None** + :param int retires: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.microdescriptor.Microdescriptor + for the given time range + + :raises: :class:`~stem.DownloadFailed` if the download fails + """ + + for f in self.files('microdescriptor', start, end): + for desc in f.read(cache_to, timeout = timeout, retries = retries): + yield desc + def index(self, compression = 'best'): """ Provides the archives available in CollecTor. diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py index 1a2e1e32..db57df5e 100644 --- a/test/integ/descriptor/collector.py +++ b/test/integ/descriptor/collector.py @@ -49,7 +49,18 @@ class TestCollector(unittest.TestCase): recent_descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors(start = RECENT)) if not (300 < len(recent_descriptors) < 800): - self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) + self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) # 583 on 8/7/19 + + @test.require.only_run_once + @test.require.online + def test_downloading_microdescriptors(self): + recent_descriptors = list(stem.descriptor.collector.get_microdescriptors(start = RECENT)) + + # TODO: I'm unsure why these counts differ so much from server/extrainfo + # descriptors. Checking with Karsten. + + if not (300 < len(recent_descriptors) < 800): + self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) # 23 on 8/7/19 def _test_index(self, compression): if compression and not compression.available:

1 0

[stem/master] Correct caching example
by atagar＠torproject.org 18 Aug '19

18 Aug '19

commit 39c82a7c88480f2da22a7b766c58998d112c08a4 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 15 15:01:48 2019 -0700 Correct caching example I whipped this up prior to writing the module. While developing it I paid special care to this use case so it's actually more elegant than the demo I anticipated. Running this example now works. --- stem/descriptor/collector.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index e8e798b3..20eb6872 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -29,22 +29,23 @@ With this you can either download and read directly from CollecTor... :: import datetime - import os import stem.descriptor import stem.descriptor.collector yesterday = datetime.datetime.utcnow() - datetime.timedelta(days = 1) - path = os.path.expanduser('~/descriptor_cache/server_desc_today') + cache_dir = '~/descriptor_cache/server_desc_today' - with open(path, 'wb') as cache_file: - for desc in stem.descriptor.collector.get_server_descriptors(start = yesterday): - cache_file.write(desc.get_bytes()) + collector = stem.descriptor.collector.CollecTor() + + for f in collector.files('server-descriptor', start = yesterday): + f.download(cache_dir) # then later... - for desc in stem.descriptor.parse_file(path, descriptor_type = 'server-descriptor 1.0'): - if desc.exit_policy.is_exiting_allowed(): - print(' %s (%s)' % (desc.nickname, desc.fingerprint)) + for f in collector.files('server-descriptor', start = yesterday): + for desc in f.read(cache_dir): + if desc.exit_policy.is_exiting_allowed(): + print(' %s (%s)' % (desc.nickname, desc.fingerprint)) :: @@ -355,6 +356,8 @@ class File(object): if self.compression != Compression.PLAINTEXT and decompress: filename = filename.rsplit('.', 1)[0] + directory = os.path.expanduser(directory) + path = os.path.join(directory, filename) if not os.path.exists(directory):

1 0