tor-commits August 2019

tor-commits@lists.torproject.org

19 participants
2737 discussions

[stem/master] Move DownloadFailed into base module
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 0e32da680122a669958adb8f18b660e5d3e3c498 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Aug 2 12:46:47 2019 -0700 Move DownloadFailed into base module While I love colocating this with the download() helper, when it comes to practical usage keeping module names of common exceptions like this short are important for usability. --- stem/__init__.py | 66 ++++++++++++++++++++++++++++++++++++++++ stem/descriptor/collector.py | 10 +++---- stem/util/connection.py | 70 +++---------------------------------------- test/integ/util/connection.py | 5 ++-- test/unit/util/connection.py | 5 ++-- 5 files changed, 81 insertions(+), 75 deletions(-) diff --git a/stem/__init__.py b/stem/__init__.py index 70870e8e..dc8ae2d2 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -28,6 +28,9 @@ Library for working with the tor process. +- SocketError - Communication with the socket failed. +- SocketClosed - Socket has been shut down. + DownloadFailed - Inability to download a resource. + +- DownloadTimeout - Download timeout reached. + .. data:: Runlevel (enum) Rating of importance used for event logging. @@ -499,6 +502,8 @@ Library for working with the tor process. ================= =========== """ +import traceback + import stem.util import stem.util.enum @@ -532,6 +537,8 @@ __all__ = [ 'InvalidArguments', 'SocketError', 'SocketClosed', + 'DownloadFailed', + 'DownloadTimeout', 'Runlevel', 'Signal', 'Flag', @@ -712,6 +719,65 @@ class SocketClosed(SocketError): 'Control socket was closed before completing the message.' +class DownloadFailed(IOError): + """ + Inability to download a resource. Python's urllib module raises + a wide variety of undocumented exceptions (urllib2.URLError, + socket.timeout, and others). + + This wraps lower level failures in a common exception type that + retains their exception and `stacktrace + <https://docs.python.org/3/library/traceback.html>`_. + + .. versionadded:: 1.8.0 + + :var str url: url we failed to download from + :var Exception error: original urllib exception + :var traceback stacktrace: original stacktrace + :var str stacktrace_str: string representation of the stacktrace + """ + + def __init__(self, url, error, stacktrace, message = None): + if message is None: + # The string representation of exceptions can reside in several places. + # urllib.URLError use a 'reason' attribute that in turn may referrence + # low level structures such as socket.gaierror. Whereas most exceptions + # use a 'message' attribute. + + reason = str(error) + + all_str_repr = ( + getattr(getattr(error, 'reason', None), 'strerror', None), + getattr(error, 'reason', None), + getattr(error, 'message', None), + ) + + for str_repr in all_str_repr: + if str_repr and isinstance(str_repr, str): + reason = str_repr + break + + message = 'Failed to download from %s (%s): %s' % (url, type(error).__name__, reason) + + super(DownloadFailed, self).__init__(message) + + self.url = url + self.error = error + self.stacktrace = stacktrace + self.stacktrace_str = ''.join(traceback.format_tb(stacktrace)) + + +class DownloadTimeout(DownloadFailed): + """ + Timeout reached while downloading this resource. + + .. versionadded:: 1.8.0 + """ + + def __init__(self, url, error, stacktrace, timeout): + super(DownloadTimeout, self).__init__('Failed to download from %s: %0.1f second timeout reached' % (url, timeout)) + + Runlevel = stem.util.enum.UppercaseEnum( 'DEBUG', 'INFO', diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index d9f159e1..57f9ce0a 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -141,7 +141,7 @@ def get_server_descriptors(start = None, end = None, cache_to = None, timeout = :class:`~stem.descriptor.server_descriptor.ServerDescriptor` for the given time range - :raises: :class:`~stem.util.connection.DownloadFailed` if the download fails + :raises: :class:`~stem.DownloadFailed` if the download fails """ for f in get_instance().files('server-descriptor', start, end): @@ -209,7 +209,7 @@ class File(object): :raises: * **ValueError** if unable to determine the descirptor type * **TypeError** if we cannot parse this descriptor type - * :class:`~stem.util.connection.DownloadFailed` if the download fails + * :class:`~stem.DownloadFailed` if the download fails """ if descriptor_type is None: @@ -256,7 +256,7 @@ class File(object): :returns: **str** with the path we downloaded to - :raises: :class:`~stem.util.connection.DownloadFailed` if the download fails + :raises: :class:`~stem.DownloadFailed` if the download fails """ # TODO: If checksums get added to the index we should replace @@ -383,7 +383,7 @@ class CollecTor(object): * **ValueError** if json is malformed * **IOError** if unable to decompress - * :class:`~stem.util.connection.DownloadFailed` if the download fails + * :class:`~stem.DownloadFailed` if the download fails """ if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: @@ -419,7 +419,7 @@ class CollecTor(object): * **ValueError** if json is malformed * **IOError** if unable to decompress - * :class:`~stem.util.connection.DownloadFailed` if the download fails + * :class:`~stem.DownloadFailed` if the download fails """ if not self._cached_files or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: diff --git a/stem/util/connection.py b/stem/util/connection.py index 7be7fe09..a1f93625 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -8,9 +8,6 @@ Connection and networking based utility functions. :: - DownloadFailed - Inability to download a resource. - +- DownloadTimeout - Download timeout reached. - download - download from a given url get_connections - quieries the connections belonging to a given process system_resolvers - provides connection resolution methods that are likely to be available @@ -65,8 +62,8 @@ import re import socket import sys import time -import traceback +import stem import stem.util import stem.util.proc import stem.util.system @@ -176,65 +173,6 @@ class Connection(collections.namedtuple('Connection', ['local_address', 'local_p """ -class DownloadFailed(IOError): - """ - Inability to download a resource. Python's urllib module raises - a wide variety of undocumented exceptions (urllib2.URLError, - socket.timeout, and others). - - This wraps lower level failures in a common exception type that - retains their exception and `stacktrace - <https://docs.python.org/3/library/traceback.html>`_. - - .. versionadded:: 1.8.0 - - :var str url: url we failed to download from - :var Exception error: original urllib exception - :var traceback stacktrace: original stacktrace - :var str stacktrace_str: string representation of the stacktrace - """ - - def __init__(self, url, error, stacktrace, message = None): - if message is None: - # The string representation of exceptions can reside in several places. - # urllib.URLError use a 'reason' attribute that in turn may referrence - # low level structures such as socket.gaierror. Whereas most exceptions - # use a 'message' attribute. - - reason = str(error) - - all_str_repr = ( - getattr(getattr(error, 'reason', None), 'strerror', None), - getattr(error, 'reason', None), - getattr(error, 'message', None), - ) - - for str_repr in all_str_repr: - if str_repr and isinstance(str_repr, str): - reason = str_repr - break - - message = 'Failed to download from %s (%s): %s' % (url, type(error).__name__, reason) - - super(DownloadFailed, self).__init__(message) - - self.url = url - self.error = error - self.stacktrace = stacktrace - self.stacktrace_str = ''.join(traceback.format_tb(stacktrace)) - - -class DownloadTimeout(DownloadFailed): - """ - Timeout reached while downloading this resource. - - .. versionadded:: 1.8.0 - """ - - def __init__(self, url, error, stacktrace, timeout): - super(DownloadTimeout, self).__init__('Failed to download from %s: %0.1f second timeout reached' % (url, timeout)) - - def download(url, timeout = None, retries = None): """ Download from the given url. @@ -248,7 +186,7 @@ def download(url, timeout = None, retries = None): :returns: **bytes** content of the given url - :raises: :class:`~stem.util.connection.DownloadFailed` if the download fails + :raises: :class:`~stem.DownloadFailed` if the download fails """ if retries is None: @@ -259,7 +197,7 @@ def download(url, timeout = None, retries = None): try: return urllib.urlopen(url, timeout = timeout).read() except socket.timeout as exc: - raise DownloadTimeout(url, exc, sys.exc_info()[2], timeout) + raise stem.DownloadTimeout(url, exc, sys.exc_info()[2], timeout) except: exc, stacktrace = sys.exc_info()[1:3] @@ -271,7 +209,7 @@ def download(url, timeout = None, retries = None): return download(url, timeout, retries - 1) else: log.debug('Failed to download from %s: %s' % (url, exc)) - raise DownloadFailed(url, exc, stacktrace) + raise stem.DownloadFailed(url, exc, stacktrace) def get_connections(resolver = None, process_pid = None, process_name = None): diff --git a/test/integ/util/connection.py b/test/integ/util/connection.py index 4617fe56..ed55ad89 100644 --- a/test/integ/util/connection.py +++ b/test/integ/util/connection.py @@ -5,6 +5,7 @@ that we're running. import unittest +import stem import stem.util.connection import stem.util.system import test.require @@ -54,8 +55,8 @@ class TestConnection(unittest.TestCase): def test_download_failure(self): try: stem.util.connection.download('https://no.such.testing.url') - self.fail('expected a stem.util.connection.DownloadFailed to be raised') - except stem.util.connection.DownloadFailed as exc: + self.fail('expected a stem.DownloadFailed to be raised') + except stem.DownloadFailed as exc: self.assertEqual('Failed to download from https://no.such.testing.url (URLError): Name or service not known', str(exc)) self.assertEqual('https://no.such.testing.url', exc.url) self.assertEqual('Name or service not known', exc.error.reason.strerror) diff --git a/test/unit/util/connection.py b/test/unit/util/connection.py index 57718446..8721ff6d 100644 --- a/test/unit/util/connection.py +++ b/test/unit/util/connection.py @@ -6,6 +6,7 @@ import io import platform import unittest +import stem import stem.util.connection from stem.util.connection import Resolver, Connection @@ -189,8 +190,8 @@ class TestConnection(unittest.TestCase): try: stem.util.connection.download(URL) - self.fail('expected a stem.util.connection.DownloadFailed to be raised') - except stem.util.connection.DownloadFailed as exc: + self.fail('expected a stem.DownloadFailed to be raised') + except stem.DownloadFailed as exc: self.assertEqual('Failed to download from https://example.unit.test.url (URLError): boom', str(exc)) self.assertEqual(URL, exc.url) self.assertEqual('boom', exc.error.reason)

1 0

[stem/master] Raise DownloadFailed from stem.descriptor.remote
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit d52bbea9d3d370e122a4da35a94362ec17523c51 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 4 14:21:21 2019 -0700 Raise DownloadFailed from stem.descriptor.remote Ok, now that we have decent test coverage time to use this exception type where it matters most: in our remote module. This will greately simplify exception catchability and provide callers with the underlying stacktrace (which at present is getting obscured). I attempted to provide this back in commit aebf3e4, which got reverted in 8c61fc8. --- stem/__init__.py | 3 ++- stem/descriptor/remote.py | 44 +++++++++++++++++++++++------------------- stem/directory.py | 2 ++ stem/manual.py | 2 ++ stem/util/connection.py | 4 +++- test/unit/descriptor/remote.py | 2 +- 6 files changed, 34 insertions(+), 23 deletions(-) diff --git a/stem/__init__.py b/stem/__init__.py index dc8ae2d2..293b2b94 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -775,7 +775,8 @@ class DownloadTimeout(DownloadFailed): """ def __init__(self, url, error, stacktrace, timeout): - super(DownloadTimeout, self).__init__('Failed to download from %s: %0.1f second timeout reached' % (url, timeout)) + message = 'Failed to download from %s: %0.1f second timeout reached' % (url, timeout) + super(DownloadTimeout, self).__init__(url, error, stacktrace, message) Runlevel = stem.util.enum.UppercaseEnum( diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index af24f624..4f29a2e5 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -97,6 +97,7 @@ content. For example... import io import random +import socket import sys import threading import time @@ -525,11 +526,8 @@ class Query(object): **False**... * **ValueError** if the descriptor contents is malformed - * **socket.timeout** if our request timed out - * **urllib2.URLError** for most request failures - - Note that the urllib2 module may fail with other exception types, in - which case we'll pass it along. + * :class:`~stem.DownloadTimeout` if our request timed out + * :class:`~stem.DownloadFailed` if our request fails """ return list(self._run(suppress)) @@ -951,11 +949,11 @@ class DescriptorDownloader(object): Traceback (most recent call last): File "demo.py", line 3, in detached_sigs = stem.descriptor.remote.get_detached_signatures().run()[0] - File "/home/atagar/Desktop/stem/stem/descriptor/remote.py", line 476, in run + File "/home/atagar/Desktop/stem/stem/descriptor/remote.py", line 533, in run return list(self._run(suppress)) - File "/home/atagar/Desktop/stem/stem/descriptor/remote.py", line 487, in _run + File "/home/atagar/Desktop/stem/stem/descriptor/remote.py", line 544, in _run raise self.error - urllib2.HTTPError: HTTP Error 404: Not found + stem.DownloadFailed: Failed to download from http://154.35.175.225:80/tor/status-vote/next/consensus-signatures (HTTPError): Not found .. versionadded:: 1.8.0 @@ -1064,20 +1062,26 @@ def _download_from_dirport(url, compression, timeout): :returns: two value tuple of the form (data, reply_headers) :raises: - * **socket.timeout** if our request timed out - * **urllib2.URLError** for most request failures + * :class:`~stem.DownloadTimeout` if our request timed out + * :class:`~stem.DownloadFailed` if our request fails """ - response = urllib.urlopen( - urllib.Request( - url, - headers = { - 'Accept-Encoding': ', '.join(map(lambda c: c.encoding, compression)), - 'User-Agent': stem.USER_AGENT, - } - ), - timeout = timeout, - ) + try: + response = urllib.urlopen( + urllib.Request( + url, + headers = { + 'Accept-Encoding': ', '.join(map(lambda c: c.encoding, compression)), + 'User-Agent': stem.USER_AGENT, + } + ), + timeout = timeout, + ) + except socket.timeout as exc: + raise stem.DownloadTimeout(url, exc, sys.exc_info()[2], timeout) + except: + exc, stacktrace = sys.exc_info()[1:3] + raise stem.DownloadFailed(url, exc, stacktrace) return _decompress(response.read(), response.headers.get('Content-Encoding')), response.headers diff --git a/stem/directory.py b/stem/directory.py index 01eca70b..075235c9 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -215,6 +215,8 @@ class Directory(object): :raises: **IOError** if unable to retrieve the fallback directories """ + # TODO: change IOError to DownloadFailed in stem 2.x + raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') def __hash__(self): diff --git a/stem/manual.py b/stem/manual.py index 0aef8a48..729fb204 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -296,6 +296,8 @@ def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, :raises: **IOError** if unable to retrieve the manual """ + # TODO: change IOError to DownloadFailed in stem 2.x + if not path and not file_handle: raise ValueError("Either the path or file_handle we're saving to must be provided") elif not stem.util.system.is_available('a2x'): diff --git a/stem/util/connection.py b/stem/util/connection.py index a1f93625..421eb9a3 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -186,7 +186,9 @@ def download(url, timeout = None, retries = None): :returns: **bytes** content of the given url - :raises: :class:`~stem.DownloadFailed` if the download fails + :raises: + * :class:`~stem.DownloadTimeout` if our request timed out + * :class:`~stem.DownloadFailed` if our request fails """ if retries is None: diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 6dbaf43e..8a35216d 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -383,7 +383,7 @@ class TestDescriptorDownloader(unittest.TestCase): # After two requests we'll have reached our total permissable timeout. # Check that we don't make a third. - self.assertRaises(socket.timeout, query.run) + self.assertRaises(stem.DownloadTimeout, query.run) self.assertEqual(2, dirport_mock.call_count) def test_query_with_invalid_endpoints(self):

1 0

[stem/master] Download method for extrainfo descriptors
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit c109fda630132f6981d4afbba0a4e5bb878fa6ee Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Aug 6 12:18:38 2019 -0700 Download method for extrainfo descriptors --- stem/descriptor/collector.py | 39 +++++++++++++++++++++++++++++++++++++- test/integ/descriptor/collector.py | 8 ++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index e6e605f0..82a213f3 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -49,7 +49,8 @@ With this you can either download and read directly from CollecTor... :: get_instance - Provides a singleton CollecTor used for... - +- get_server_descriptors - published server descriptors + |- get_server_descriptors - published server descriptors + +- get_extrainfo_descriptors - published extrainfo descriptors File - Individual file residing within CollecTor |- read - provides descriptors from this file @@ -57,6 +58,7 @@ With this you can either download and read directly from CollecTor... CollecTor - Downloader for descriptors from CollecTor |- get_server_descriptors - published server descriptors + |- get_extrainfo_descriptors - published extrainfo descriptors | |- index - metadata for content available from CollecTor +- files - files available from CollecTor @@ -150,6 +152,17 @@ def get_server_descriptors(start = None, end = None, cache_to = None, timeout = yield desc +def get_extrainfo_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Shorthand for + :func:`~stem.descriptor.collector.CollecTor.get_extrainfo_descriptors` + on our singleton instance. + """ + + for desc in get_instance().get_extrainfo_descriptors(start, end, cache_to, timeout, retries): + yield desc + + class File(object): """ File within CollecTor. @@ -393,6 +406,30 @@ class CollecTor(object): for desc in f.read(cache_to, timeout = timeout, retries = retries): yield desc + def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Provides extrainfo descriptors published during the given time range, + sorted oldest to newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int timeout: timeout for downloading each individual archive when + the connection becomes idle, no timeout applied if **None** + :param int retires: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor` + for the given time range + + :raises: :class:`~stem.DownloadFailed` if the download fails + """ + + for f in self.files('extra-info', start, end): + for desc in f.read(cache_to, timeout = timeout, retries = retries): + yield desc + def index(self, compression = 'best'): """ Provides the archives available in CollecTor. diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py index 040c6b72..1a2e1e32 100644 --- a/test/integ/descriptor/collector.py +++ b/test/integ/descriptor/collector.py @@ -43,6 +43,14 @@ class TestCollector(unittest.TestCase): if not (300 < len(recent_descriptors) < 800): self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) # 584 on 8/5/19 + @test.require.only_run_once + @test.require.online + def test_downloading_extrainfo_descriptors(self): + recent_descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors(start = RECENT)) + + if not (300 < len(recent_descriptors) < 800): + self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) + def _test_index(self, compression): if compression and not compression.available: self.skipTest('(%s unavailable)' % compression)

1 0

[stem/master] Download method for bandwidth files
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 16ac148a98b19f6c9649ab1f548bd3c5410428fc Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 15 14:51:30 2019 -0700 Download method for bandwidth files --- stem/descriptor/collector.py | 53 ++++++++++++++++++--- test/unit/descriptor/collector.py | 19 +++++++- .../data/collector/bandwidths-2019-05-cropped.tar | Bin 0 -> 32768 bytes 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 6a4c700f..e8e798b3 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -53,8 +53,10 @@ With this you can either download and read directly from CollecTor... |- get_extrainfo_descriptors - published extrainfo descriptors |- get_microdescriptors - published microdescriptors |- get_consensus - published router status entries + | |- get_key_certificates - authority key certificates - +- get_exit_list - TorDNSEL exit list + |- get_bandwidth_files - bandwidth authority heuristics + +- get_exit_lists - TorDNSEL exit list File - Individual file residing within CollecTor |- read - provides descriptors from this file @@ -65,8 +67,10 @@ With this you can either download and read directly from CollecTor... |- get_extrainfo_descriptors - published extrainfo descriptors |- get_microdescriptors - published microdescriptors |- get_consensus - published router status entries + | |- get_key_certificates - authority key certificates - |- get_exit_list - TorDNSEL exit list + |- get_bandwidth_files - bandwidth authority heuristics + |- get_exit_lists - TorDNSEL exit list | |- index - metadata for content available from CollecTor +- files - files available from CollecTor @@ -205,14 +209,25 @@ def get_key_certificates(start = None, end = None, cache_to = None, timeout = No yield desc -def get_exit_list(start = None, end = None, cache_to = None, timeout = None, retries = 3): +def get_bandwidth_files(start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Shorthand for + :func:`~stem.descriptor.collector.CollecTor.get_bandwidth_files` + on our singleton instance. + """ + + for desc in get_instance().get_bandwidth_files(start, end, cache_to, timeout, retries): + yield desc + + +def get_exit_lists(start = None, end = None, cache_to = None, timeout = None, retries = 3): """ Shorthand for - :func:`~stem.descriptor.collector.CollecTor.get_exit_list` + :func:`~stem.descriptor.collector.CollecTor.get_exit_lists` on our singleton instance. """ - for desc in get_instance().get_exit_list(start, end, cache_to, timeout, retries): + for desc in get_instance().get_exit_lists(start, end, cache_to, timeout, retries): yield desc @@ -594,9 +609,33 @@ class CollecTor(object): for desc in f.read(cache_to, 'dir-key-certificate-3', timeout = timeout, retries = retries): yield desc - def get_exit_list(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + def get_bandwidth_files(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Bandwidth authority heuristics for the given time range, sorted oldest to + newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int timeout: timeout for downloading each individual archive when + the connection becomes idle, no timeout applied if **None** + :param int retries: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.bandwidth_file.BandwidthFile + for the given time range + + :raises: :class:`~stem.DownloadFailed` if the download fails + """ + + for f in self.files('bandwidth-file', start, end): + for desc in f.read(cache_to, 'bandwidth-file', timeout = timeout, retries = retries): + yield desc + + def get_exit_lists(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): """ - `TorDNSEL exit list <https://www.torproject.org/projects/tordnsel.html.en>`_ + `TorDNSEL exit lists <https://www.torproject.org/projects/tordnsel.html.en>`_ for the given time range, sorted oldest to newest. :param datetime.datetime start: time range to begin with diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index e3ba1f40..44893fab 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -344,13 +344,28 @@ class TestCollector(unittest.TestCase): @patch('stem.util.connection.download') @patch('stem.descriptor.collector.CollecTor.files') - def test_reading_exit_list(self, files_mock, download_mock): + def test_reading_bandwidth_files(self, files_mock, download_mock): + with open(get_resource('collector/bandwidths-2019-05-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/bandwidths/bandwidths-2019-05.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_bandwidth_files()) + self.assertEqual(2, len(descriptors)) + + f = descriptors[0] + self.assertEqual('BandwidthFile', type(f).__name__) + self.assertEqual(22, len(f.measurements)) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_exit_lists(self, files_mock, download_mock): with open(get_resource('collector/exit-list-2018-11-cropped.tar'), 'rb') as archive: download_mock.return_value = archive.read() files_mock.return_value = [stem.descriptor.collector.File('archive/exit-lists/exit-list-2018-11.tar', 12345, '2016-09-04 09:21')] - descriptors = list(stem.descriptor.collector.get_exit_list()) + descriptors = list(stem.descriptor.collector.get_exit_lists()) self.assertEqual(3713, len(descriptors)) f = descriptors[0] diff --git a/test/unit/descriptor/data/collector/bandwidths-2019-05-cropped.tar b/test/unit/descriptor/data/collector/bandwidths-2019-05-cropped.tar new file mode 100644 index 00000000..4f3c8724 Binary files /dev/null and b/test/unit/descriptor/data/collector/bandwidths-2019-05-cropped.tar differ

1 0

[stem/master] Download method for TorDNSEL exit lists
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 466f319f3bb0a40317bd30dfd3fee487d7bb3432 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 15 14:29:04 2019 -0700 Download method for TorDNSEL exit lists --- stem/descriptor/collector.py | 39 ++++++++++++++++++++- test/unit/descriptor/collector.py | 15 ++++++++ .../data/collector/exit-list-2018-11-cropped.tar | Bin 0 -> 590336 bytes 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 9a238ecf..6a4c700f 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -53,7 +53,8 @@ With this you can either download and read directly from CollecTor... |- get_extrainfo_descriptors - published extrainfo descriptors |- get_microdescriptors - published microdescriptors |- get_consensus - published router status entries - +- get_key_certificates - authority key certificates + |- get_key_certificates - authority key certificates + +- get_exit_list - TorDNSEL exit list File - Individual file residing within CollecTor |- read - provides descriptors from this file @@ -65,6 +66,7 @@ With this you can either download and read directly from CollecTor... |- get_microdescriptors - published microdescriptors |- get_consensus - published router status entries |- get_key_certificates - authority key certificates + |- get_exit_list - TorDNSEL exit list | |- index - metadata for content available from CollecTor +- files - files available from CollecTor @@ -203,6 +205,17 @@ def get_key_certificates(start = None, end = None, cache_to = None, timeout = No yield desc +def get_exit_list(start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Shorthand for + :func:`~stem.descriptor.collector.CollecTor.get_exit_list` + on our singleton instance. + """ + + for desc in get_instance().get_exit_list(start, end, cache_to, timeout, retries): + yield desc + + class File(object): """ File within CollecTor. @@ -581,6 +594,30 @@ class CollecTor(object): for desc in f.read(cache_to, 'dir-key-certificate-3', timeout = timeout, retries = retries): yield desc + def get_exit_list(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + `TorDNSEL exit list <https://www.torproject.org/projects/tordnsel.html.en>`_ + for the given time range, sorted oldest to newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int timeout: timeout for downloading each individual archive when + the connection becomes idle, no timeout applied if **None** + :param int retries: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.tordnsel.TorDNSEL + for the given time range + + :raises: :class:`~stem.DownloadFailed` if the download fails + """ + + for f in self.files('tordnsel', start, end): + for desc in f.read(cache_to, 'tordnsel', timeout = timeout, retries = retries): + yield desc + def index(self, compression = 'best'): """ Provides the archives available in CollecTor. diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index d6a3aee8..e3ba1f40 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -341,3 +341,18 @@ class TestCollector(unittest.TestCase): f = descriptors[0] self.assertEqual('KeyCertificate', type(f).__name__) self.assertEqual('14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_exit_list(self, files_mock, download_mock): + with open(get_resource('collector/exit-list-2018-11-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/exit-lists/exit-list-2018-11.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_exit_list()) + self.assertEqual(3713, len(descriptors)) + + f = descriptors[0] + self.assertEqual('TorDNSEL', type(f).__name__) + self.assertEqual('0011BD2485AD45D984EC4159C88FC066E5E3300E', f.fingerprint) diff --git a/test/unit/descriptor/data/collector/exit-list-2018-11-cropped.tar b/test/unit/descriptor/data/collector/exit-list-2018-11-cropped.tar new file mode 100644 index 00000000..2f86f8de Binary files /dev/null and b/test/unit/descriptor/data/collector/exit-list-2018-11-cropped.tar differ

1 0

[stem/master] Support a DocumentHandler when reading consensus documents
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 20c36088b536022995d9f10475a6386dbd5d4fdb Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Aug 14 14:04:23 2019 -0700 Support a DocumentHandler when reading consensus documents When reading a consensus our callers need the ability to specify what form they wish to receive the document (just the document, a document with the relays populated, or just the relays). Unfortunately CollecTor has separate annotations for v3 consensus and vote documents, whereas all other classes have at most one. This requires rethinking the annotation constants but for the moment moving on to bigger fish. --- stem/descriptor/collector.py | 21 +++++++++++++-------- stem/descriptor/networkstatus.py | 2 ++ test/unit/descriptor/collector.py | 9 ++++++++- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 09e94e74..7c2447a2 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -78,10 +78,11 @@ import shutil import tempfile import time +import stem.descriptor import stem.util.connection import stem.util.str_tools -from stem.descriptor import Compression, parse_file +from stem.descriptor import Compression, DocumentHandler COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old @@ -178,14 +179,14 @@ def get_microdescriptors(start = None, end = None, cache_to = None, timeout = No yield desc -def get_consensus(start = None, end = None, cache_to = None, version = 3, microdescriptor = False, timeout = None, retries = 3): +def get_consensus(start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, timeout = None, retries = 3): """ Shorthand for :func:`~stem.descriptor.collector.CollecTor.get_consensus` on our singleton instance. """ - for desc in get_instance().get_consensus(start, end, cache_to, version, microdescriptor, timeout, retries): + for desc in get_instance().get_consensus(start, end, cache_to, document_handler, version, microdescriptor, timeout, retries): yield desc @@ -216,7 +217,7 @@ class File(object): self._guessed_type = File._guess_descriptor_types(path) self._downloaded_to = None # location we last downloaded to - def read(self, directory = None, descriptor_type = None, timeout = None, retries = 3): + def read(self, directory = None, descriptor_type = None, document_handler = DocumentHandler.ENTRIES, timeout = None, retries = 3): """ Provides descriptors from this archive. Descriptors are downloaded or read from disk as follows... @@ -239,6 +240,8 @@ class File(object): :param str descriptor_type: `descriptor type <https://metrics.torproject.org/collector.html#data-formats>`_, this is guessed if not provided + :var stem.descriptor.__init__.DocumentHandler document_handler: method in + which to parse a :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` :param int timeout: timeout when connection becomes idle, no timeout applied if **None** :param int retires: maximum attempts to impose @@ -269,7 +272,7 @@ class File(object): tmp_directory = tempfile.mkdtemp() - for desc in self.read(tmp_directory, descriptor_type, timeout, retries): + for desc in self.read(tmp_directory, descriptor_type, document_handler, timeout, retries): yield desc shutil.rmtree(tmp_directory) @@ -281,7 +284,7 @@ class File(object): # Archives can contain multiple descriptor types, so parsing everything and # filtering to what we're after. - for desc in parse_file(path): + for desc in stem.descriptor.parse_file(path, document_handler = document_handler): desc_annotation = type(desc).TYPE_ANNOTATION_NAME if descriptor_type is None or (desc_annotation and descriptor_type.startswith(desc_annotation)): @@ -497,7 +500,7 @@ class CollecTor(object): for desc in f.read(cache_to, 'microdescriptor', timeout = timeout, retries = retries): yield desc - def get_consensus(self, start = None, end = None, cache_to = None, version = 3, microdescriptor = False, timeout = None, retries = 3): + def get_consensus(self, start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, timeout = None, retries = 3): """ Provides consensus router status entries published during the given time range, sorted oldest to newest. @@ -506,6 +509,8 @@ class CollecTor(object): :param datetime.datetime end: time range to end with :param str cache_to: directory to cache archives into, if an archive is available here it is not downloaded + :var stem.descriptor.__init__.DocumentHandler document_handler: method in + which to parse a :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` :param int version: consensus variant to retrieve (versions 2 or 3) :param bool microdescriptor: provides the microdescriptor consensus if **True**, standard consensus otherwise @@ -536,7 +541,7 @@ class CollecTor(object): # TODO: document vs router status entries (ie. DocumentType)? for f in self.files(desc_type, start, end): - for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries): + for desc in f.read(cache_to, desc_type, document_handler, timeout = timeout, retries = retries): yield desc def index(self, compression = 'best'): diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index b0589f2a..dd7759f5 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -1032,6 +1032,8 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): Added the bandwidth_file_digest attributbute. """ + TYPE_ANNOTATION_NAME = 'network-status-consensus-3' # TODO: can also be network-status-vote-3 + ATTRIBUTES = { 'version': (None, _parse_header_network_status_version_line), 'version_flavor': ('ns', _parse_header_network_status_version_line), diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index f5bed87e..7d1f0205 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -8,7 +8,7 @@ import unittest import stem.prereq -from stem.descriptor import Compression +from stem.descriptor import Compression, DocumentHandler from stem.descriptor.collector import CollecTor, File from test.unit.descriptor import get_resource from test.unit.descriptor.data.collector.index import EXAMPLE_INDEX @@ -255,6 +255,13 @@ class TestCollector(unittest.TestCase): self.assertEqual('RouterStatusEntryV3', type(f).__name__) self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint) + descriptors = list(stem.descriptor.collector.get_consensus(document_handler = DocumentHandler.DOCUMENT)) + self.assertEqual(2, len(descriptors)) + + f = descriptors[0] + self.assertEqual('NetworkStatusDocumentV3', type(f).__name__) + self.assertEqual(35, len(f.routers)) + # this archive shouldn't have any v2 or microdescriptor consensus data self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(version = 2))))

1 0

[stem/master] Unit test collector downloading
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 4a319f1794e54fc216d84ac192cc3efc1ce15ec2 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 11 05:14:49 2019 -0700 Unit test collector downloading On reflection there's no need for us to rely exclusively on integ tests for download coverage. Those certainly do a great job, but simple to get unit test coverage of this as well. --- stem/descriptor/collector.py | 8 +- test/unit/descriptor/collector.py | 81 ++++++++++++++++++++- test/unit/descriptor/data/__init__.py | 2 +- test/unit/descriptor/data/collector/__init__.py | 7 ++ .../data/collector/consensuses-2018-06-cropped.tar | Bin 0 -> 100864 bytes .../data/collector/extra-infos-2019-04-cropped.tar | Bin 0 -> 22528 bytes .../{collector_index.json => collector/index.json} | 0 .../{collector_index.py => collector/index.py} | 0 .../data/collector/microdescs-2019-05-cropped.tar | Bin 0 -> 193024 bytes .../server-descriptors-2005-12-cropped.tar | Bin 0 -> 22528 bytes 10 files changed, 91 insertions(+), 7 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index c0c97832..09e94e74 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -435,7 +435,7 @@ class CollecTor(object): # TODO: support bridge variants ('bridge-server-descriptor' type) for f in self.files('server-descriptor', start, end): - for desc in f.read(cache_to, timeout = timeout, retries = retries): + for desc in f.read(cache_to, 'server-descriptor', timeout = timeout, retries = retries): yield desc def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): @@ -461,7 +461,7 @@ class CollecTor(object): # TODO: support bridge variants ('bridge-extra-info' type) for f in self.files('extra-info', start, end): - for desc in f.read(cache_to, timeout = timeout, retries = retries): + for desc in f.read(cache_to, 'extra-info', timeout = timeout, retries = retries): yield desc def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): @@ -494,7 +494,7 @@ class CollecTor(object): """ for f in self.files('microdescriptor', start, end): - for desc in f.read(cache_to, timeout = timeout, retries = retries): + for desc in f.read(cache_to, 'microdescriptor', timeout = timeout, retries = retries): yield desc def get_consensus(self, start = None, end = None, cache_to = None, version = 3, microdescriptor = False, timeout = None, retries = 3): @@ -536,7 +536,7 @@ class CollecTor(object): # TODO: document vs router status entries (ie. DocumentType)? for f in self.files(desc_type, start, end): - for desc in f.read(cache_to, timeout = timeout, retries = retries): + for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries): yield desc def index(self, compression = 'best'): diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 77c1c460..f5bed87e 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -11,7 +11,7 @@ import stem.prereq from stem.descriptor import Compression from stem.descriptor.collector import CollecTor, File from test.unit.descriptor import get_resource -from test.unit.descriptor.data.collector_index import EXAMPLE_INDEX +from test.unit.descriptor.data.collector.index import EXAMPLE_INDEX try: # added in python 3.3 @@ -22,7 +22,7 @@ except ImportError: URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' -with open(get_resource('collector_index.json'), 'rb') as index_file: +with open(get_resource('collector/index.json'), 'rb') as index_file: EXAMPLE_INDEX_JSON = index_file.read() @@ -194,3 +194,80 @@ class TestCollector(unittest.TestCase): self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', ], [f.path for f in collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1))]) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_server_descriptors(self, files_mock, download_mock): + with open(get_resource('collector/server-descriptors-2005-12-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_server_descriptors()) + self.assertEqual(5, len(descriptors)) + + f = descriptors[0] + self.assertEqual('RelayDescriptor', type(f).__name__) + self.assertEqual('3E2F63E2356F52318B536A12B6445373808A5D6C', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_extrainfo_descriptors(self, files_mock, download_mock): + with open(get_resource('collector/extra-infos-2019-04-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/extra-infos/extra-infos-2019-04.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors()) + self.assertEqual(7, len(descriptors)) + + f = descriptors[0] + self.assertEqual('RelayExtraInfoDescriptor', type(f).__name__) + self.assertEqual('170EF19C0FA0491DFCEA6E1FB0941670B80506E1', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_microdescriptors(self, files_mock, download_mock): + with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/microdescs/microdescs-2019-05.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_microdescriptors()) + self.assertEqual(3, len(descriptors)) + + f = descriptors[0] + self.assertEqual('Microdescriptor', type(f).__name__) + self.assertEqual(['ed25519'], list(f.identifiers.keys())) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_consensus(self, files_mock, download_mock): + with open(get_resource('collector/consensuses-2018-06-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/consensuses/consensuses-2018-06.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_consensus()) + self.assertEqual(243, len(descriptors)) + + f = descriptors[0] + self.assertEqual('RouterStatusEntryV3', type(f).__name__) + self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint) + + # this archive shouldn't have any v2 or microdescriptor consensus data + + self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(version = 2)))) + self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(microdescriptor = True)))) + + # but the microdescriptor archive *does* have microdescriptor consensuses + + with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + descriptors = list(stem.descriptor.collector.get_consensus(microdescriptor = True)) + self.assertEqual(556, len(descriptors)) + + f = descriptors[0] + self.assertEqual('RouterStatusEntryMicroV3', type(f).__name__) + self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint) diff --git a/test/unit/descriptor/data/__init__.py b/test/unit/descriptor/data/__init__.py index 376864fd..8bc640c6 100644 --- a/test/unit/descriptor/data/__init__.py +++ b/test/unit/descriptor/data/__init__.py @@ -3,5 +3,5 @@ Test data for test.unit.descriptor tests. """ __all__ = [ - 'collector_index', + 'collector', ] diff --git a/test/unit/descriptor/data/collector/__init__.py b/test/unit/descriptor/data/collector/__init__.py new file mode 100644 index 00000000..fe90c9fb --- /dev/null +++ b/test/unit/descriptor/data/collector/__init__.py @@ -0,0 +1,7 @@ +""" +Test data for test.unit.descriptor's collector tests. +""" + +__all__ = [ + 'index', +] diff --git a/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar b/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar new file mode 100644 index 00000000..0728e595 Binary files /dev/null and b/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar differ diff --git a/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar b/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar new file mode 100644 index 00000000..21227286 Binary files /dev/null and b/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar differ diff --git a/test/unit/descriptor/data/collector_index.json b/test/unit/descriptor/data/collector/index.json similarity index 100% rename from test/unit/descriptor/data/collector_index.json rename to test/unit/descriptor/data/collector/index.json diff --git a/test/unit/descriptor/data/collector_index.py b/test/unit/descriptor/data/collector/index.py similarity index 100% rename from test/unit/descriptor/data/collector_index.py rename to test/unit/descriptor/data/collector/index.py diff --git a/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar b/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar new file mode 100644 index 00000000..789774c2 Binary files /dev/null and b/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar differ diff --git a/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar b/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar new file mode 100644 index 00000000..4ffe1434 Binary files /dev/null and b/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar differ

1 0

[stem/master] Download method for the consensus
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 69a2df402790653dadd26b301acc2bf1db96214e Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Aug 10 15:40:13 2019 -0700 Download method for the consensus --- stem/descriptor/collector.py | 61 +++++++++++++++++++++++++++++++++++++- test/integ/descriptor/collector.py | 31 +++++++++++++++++-- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 85eafec6..c0c97832 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -51,7 +51,8 @@ With this you can either download and read directly from CollecTor... get_instance - Provides a singleton CollecTor used for... |- get_server_descriptors - published server descriptors |- get_extrainfo_descriptors - published extrainfo descriptors - +- get_microdescriptors - published microdescriptors + |- get_microdescriptors - published microdescriptors + +- get_consensus - published router status entries File - Individual file residing within CollecTor |- read - provides descriptors from this file @@ -61,6 +62,7 @@ With this you can either download and read directly from CollecTor... |- get_server_descriptors - published server descriptors |- get_extrainfo_descriptors - published extrainfo descriptors |- get_microdescriptors - published microdescriptors + |- get_consensus - published router status entries | |- index - metadata for content available from CollecTor +- files - files available from CollecTor @@ -176,6 +178,17 @@ def get_microdescriptors(start = None, end = None, cache_to = None, timeout = No yield desc +def get_consensus(start = None, end = None, cache_to = None, version = 3, microdescriptor = False, timeout = None, retries = 3): + """ + Shorthand for + :func:`~stem.descriptor.collector.CollecTor.get_consensus` + on our singleton instance. + """ + + for desc in get_instance().get_consensus(start, end, cache_to, version, microdescriptor, timeout, retries): + yield desc + + class File(object): """ File within CollecTor. @@ -419,6 +432,8 @@ class CollecTor(object): :raises: :class:`~stem.DownloadFailed` if the download fails """ + # TODO: support bridge variants ('bridge-server-descriptor' type) + for f in self.files('server-descriptor', start, end): for desc in f.read(cache_to, timeout = timeout, retries = retries): yield desc @@ -443,6 +458,8 @@ class CollecTor(object): :raises: :class:`~stem.DownloadFailed` if the download fails """ + # TODO: support bridge variants ('bridge-extra-info' type) + for f in self.files('extra-info', start, end): for desc in f.read(cache_to, timeout = timeout, retries = retries): yield desc @@ -480,6 +497,48 @@ class CollecTor(object): for desc in f.read(cache_to, timeout = timeout, retries = retries): yield desc + def get_consensus(self, start = None, end = None, cache_to = None, version = 3, microdescriptor = False, timeout = None, retries = 3): + """ + Provides consensus router status entries published during the given time + range, sorted oldest to newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int version: consensus variant to retrieve (versions 2 or 3) + :param bool microdescriptor: provides the microdescriptor consensus if + **True**, standard consensus otherwise + :param int timeout: timeout for downloading each individual archive when + the connection becomes idle, no timeout applied if **None** + :param int retires: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.router_status_entry.RouterStatusEntry` + for the given time range + + :raises: :class:`~stem.DownloadFailed` if the download fails + """ + + if version == 3 and not microdescriptor: + desc_type = 'network-status-consensus-3' + elif version == 3 and microdescriptor: + desc_type = 'network-status-microdesc-consensus-3' + elif version == 2 and not microdescriptor: + desc_type = 'network-status-2' + else: + if microdescriptor and version != 3: + raise ValueError('Only v3 microdescriptors are available (not version %s)' % version) + else: + raise ValueError('Only v2 and v3 router status entries are available (not version %s)' % version) + + # TODO: support bridge variants ('bridge-network-status' type) + # TODO: document vs router status entries (ie. DocumentType)? + + for f in self.files(desc_type, start, end): + for desc in f.read(cache_to, timeout = timeout, retries = retries): + yield desc + def index(self, compression = 'best'): """ Provides the archives available in CollecTor. diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py index d38ce7d1..f1336c53 100644 --- a/test/integ/descriptor/collector.py +++ b/test/integ/descriptor/collector.py @@ -3,6 +3,7 @@ Integration tests for stem.descriptor.collector. """ import datetime +import re import unittest import test.require @@ -56,8 +57,34 @@ class TestCollector(unittest.TestCase): def test_downloading_microdescriptors(self): recent_descriptors = list(stem.descriptor.collector.get_microdescriptors(start = RECENT)) - if not (300 < len(recent_descriptors) < 800): - self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) # 23 on 8/7/19 + if not (10 < len(recent_descriptors) < 100): + self.fail('Downloaded %i descriptors, expected 10-100' % len(recent_descriptors)) # 23 on 8/7/19 + + @test.require.only_run_once + @test.require.online + def test_downloading_consensus_v3(self): + recent_descriptors = list(stem.descriptor.collector.get_consensus(start = RECENT)) + + if not (3000 < len(recent_descriptors) < 10000): + self.fail('Downloaded %i descriptors, expected 3000-10000' % len(recent_descriptors)) # 6554 on 8/10/19 + + @test.require.only_run_once + @test.require.online + def test_downloading_consensus_micro(self): + recent_descriptors = list(stem.descriptor.collector.get_consensus(start = RECENT, microdescriptor = True)) + + if not (3000 < len(recent_descriptors) < 10000): + self.fail('Downloaded %i descriptors, expected 3000-10000' % len(recent_descriptors)) + + def test_downloading_consensus_invalid_type(self): + test_values = ( + ({'version': 2, 'microdescriptor': True}, 'Only v3 microdescriptors are available (not version 2)'), + ({'version': 1}, 'Only v2 and v3 router status entries are available (not version 1)'), + ({'version': 4}, 'Only v2 and v3 router status entries are available (not version 4)'), + ) + + for args, expected_msg in test_values: + self.assertRaisesRegexp(ValueError, re.escape(expected_msg), list, stem.descriptor.collector.get_consensus(**args)) def _test_index(self, compression): if compression and not compression.available:

1 0

[stem/master] Filter archives with multiple descriptor types
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit c54a66b92c63e9156b884c1dedcc3bd2da443f38 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Aug 9 16:40:07 2019 -0700 Filter archives with multiple descriptor types One CollecTor archive type contains multiple descriptor types, so filtering out unrequested descriptors. --- stem/descriptor/collector.py | 12 ++++++++---- stem/descriptor/router_status_entry.py | 6 ++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index c8c4a1b7..3867fa3d 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -263,12 +263,16 @@ class File(object): return - # TODO: the following will not work if the tar contains multiple types or a type we do not support - path = self.download(directory, True, timeout, retries) - for desc in parse_file(path, descriptor_type): - yield desc + # Archives can contain multiple descriptor types, so parsing everything and + # filtering to what we're after. + + for desc in parse_file(path): + desc_annotation = type(desc).TYPE_ANNOTATION_NAME + + if descriptor_type is None or (desc_annotation and descriptor_type.startswith(desc_annotation)): + yield desc def download(self, directory, decompress = True, timeout = None, retries = 3): """ diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index 344c5697..31913417 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -515,6 +515,8 @@ class RouterStatusEntryV2(RouterStatusEntry): a default value, others are left as **None** if undefined """ + TYPE_ANNOTATION_NAME = 'network-status-consensus-2' + ATTRIBUTES = dict(RouterStatusEntry.ATTRIBUTES, **{ 'digest': (None, _parse_r_line), }) @@ -575,6 +577,8 @@ class RouterStatusEntryV3(RouterStatusEntry): Added the protocols attribute. """ + TYPE_ANNOTATION_NAME = 'network-status-consensus-3' + ATTRIBUTES = dict(RouterStatusEntry.ATTRIBUTES, **{ 'digest': (None, _parse_r_line), 'or_addresses': ([], _parse_a_line), @@ -652,6 +656,8 @@ class RouterStatusEntryMicroV3(RouterStatusEntry): a default value, others are left as **None** if undefined """ + TYPE_ANNOTATION_NAME = 'network-status-microdesc-consensus-3' + ATTRIBUTES = dict(RouterStatusEntry.ATTRIBUTES, **{ 'or_addresses': ([], _parse_a_line), 'bandwidth': (None, _parse_w_line),

1 0

[stem/master] Use DownloadFailed in directory and manual modules
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit b0a1a878c34a45bd85f304a17e38a834ee074d41 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 4 17:38:40 2019 -0700 Use DownloadFailed in directory and manual modules Ah. No need to wait for stem 2.x since DownloadFailed is an IOError subclass. Our from_remote() functions raise errors aside from download failures so the overall function signatures are unchanged. --- stem/directory.py | 29 +++++++++++++++++------------ stem/manual.py | 8 ++++---- test/unit/directory/authority.py | 3 ++- test/unit/directory/fallback.py | 3 ++- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/stem/directory.py b/stem/directory.py index 075235c9..9e397d88 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -40,7 +40,9 @@ as follows... import os import re +import sys +import stem import stem.util import stem.util.conf @@ -215,8 +217,6 @@ class Directory(object): :raises: **IOError** if unable to retrieve the fallback directories """ - # TODO: change IOError to DownloadFailed in stem 2.x - raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') def __hash__(self): @@ -266,11 +266,13 @@ class Authority(Directory): def from_remote(timeout = 60): try: lines = str_tools._to_unicode(urllib.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() - except Exception as exc: - raise IOError("Unable to download tor's directory authorities from %s: %s" % (GITWEB_AUTHORITY_URL, exc)) - if not lines: - raise IOError('%s did not have any content' % GITWEB_AUTHORITY_URL) + if not lines: + raise IOError('no content') + except: + exc, stacktrace = sys.exc_info()[1:3] + message = "Unable to download tor's directory authorities from %s: %s" % (GITWEB_AUTHORITY_URL, exc) + raise stem.DownloadFailed(GITWEB_AUTHORITY_URL, exc, stacktrace, message) # Entries look like... # @@ -412,16 +414,19 @@ class Fallback(Directory): def from_remote(timeout = 60): try: lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() - except Exception as exc: - raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc)) - if not lines: - raise IOError('%s did not have any content' % GITWEB_FALLBACK_URL) - elif lines[0] != '/* type=fallback */': - raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_URL) + if not lines: + raise IOError('no content') + except: + exc, stacktrace = sys.exc_info()[1:3] + message = "Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc) + raise stem.DownloadFailed(GITWEB_FALLBACK_URL, exc, stacktrace, message) # header metadata + if lines[0] != '/* type=fallback */': + raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_URL) + header = {} for line in Fallback._pop_section(lines): diff --git a/stem/manual.py b/stem/manual.py index 729fb204..24596851 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -53,6 +53,7 @@ import shutil import sys import tempfile +import stem import stem.prereq import stem.util import stem.util.conf @@ -296,8 +297,6 @@ def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, :raises: **IOError** if unable to retrieve the manual """ - # TODO: change IOError to DownloadFailed in stem 2.x - if not path and not file_handle: raise ValueError("Either the path or file_handle we're saving to must be provided") elif not stem.util.system.is_available('a2x'): @@ -313,8 +312,9 @@ def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, request = urllib.urlopen(url, timeout = timeout) shutil.copyfileobj(request, asciidoc_file) except: - exc = sys.exc_info()[1] - raise IOError("Unable to download tor's manual from %s to %s: %s" % (url, asciidoc_path, exc)) + exc, stacktrace = sys.exc_info()[1:3] + message = "Unable to download tor's manual from %s to %s: %s" % (url, asciidoc_path, exc) + raise stem.DownloadFailed(url, exc, stacktrace, message) try: stem.util.system.call('a2x -f manpage %s' % asciidoc_path) diff --git a/test/unit/directory/authority.py b/test/unit/directory/authority.py index 56bd6ec8..1c2a86b4 100644 --- a/test/unit/directory/authority.py +++ b/test/unit/directory/authority.py @@ -5,6 +5,7 @@ Unit tests for stem.directory.Authority. import io import unittest +import stem import stem.directory import stem.prereq @@ -78,4 +79,4 @@ class TestAuthority(unittest.TestCase): @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) def test_from_remote_empty(self): - self.assertRaisesRegexp(IOError, 'did not have any content', stem.directory.Authority.from_remote) + self.assertRaisesRegexp(stem.DownloadFailed, 'no content', stem.directory.Authority.from_remote) diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py index 2423932f..06b4510d 100644 --- a/test/unit/directory/fallback.py +++ b/test/unit/directory/fallback.py @@ -7,6 +7,7 @@ import re import tempfile import unittest +import stem import stem.directory import stem.util.conf @@ -123,7 +124,7 @@ class TestFallback(unittest.TestCase): @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) def test_from_remote_empty(self): - self.assertRaisesRegexp(IOError, 'did not have any content', stem.directory.Fallback.from_remote) + self.assertRaisesRegexp(stem.DownloadFailed, 'no content', stem.directory.Fallback.from_remote) @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'\n'.join(FALLBACK_GITWEB_CONTENT.splitlines()[1:])))) def test_from_remote_no_header(self):

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits August 2019