commit 16ac148a98b19f6c9649ab1f548bd3c5410428fc Author: Damian Johnson atagar@torproject.org Date: Thu Aug 15 14:51:30 2019 -0700

Download method for bandwidth files --- stem/descriptor/collector.py | 53 ++++++++++++++++++--- test/unit/descriptor/collector.py | 19 +++++++- .../data/collector/bandwidths-2019-05-cropped.tar | Bin 0 -> 32768 bytes 3 files changed, 63 insertions(+), 9 deletions(-)

diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 6a4c700f..e8e798b3 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -53,8 +53,10 @@ With this you can either download and read directly from CollecTor... |- get_extrainfo_descriptors - published extrainfo descriptors |- get_microdescriptors - published microdescriptors |- get_consensus - published router status entries + | |- get_key_certificates - authority key certificates - +- get_exit_list - TorDNSEL exit list + |- get_bandwidth_files - bandwidth authority heuristics + +- get_exit_lists - TorDNSEL exit list

File - Individual file residing within CollecTor |- read - provides descriptors from this file @@ -65,8 +67,10 @@ With this you can either download and read directly from CollecTor... |- get_extrainfo_descriptors - published extrainfo descriptors |- get_microdescriptors - published microdescriptors |- get_consensus - published router status entries + | |- get_key_certificates - authority key certificates - |- get_exit_list - TorDNSEL exit list + |- get_bandwidth_files - bandwidth authority heuristics + |- get_exit_lists - TorDNSEL exit list | |- index - metadata for content available from CollecTor +- files - files available from CollecTor @@ -205,14 +209,25 @@ def get_key_certificates(start = None, end = None, cache_to = None, timeout = No yield desc

-def get_exit_list(start = None, end = None, cache_to = None, timeout = None, retries = 3): +def get_bandwidth_files(start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Shorthand for + :func:`~stem.descriptor.collector.CollecTor.get_bandwidth_files` + on our singleton instance. + """ + + for desc in get_instance().get_bandwidth_files(start, end, cache_to, timeout, retries): + yield desc + + +def get_exit_lists(start = None, end = None, cache_to = None, timeout = None, retries = 3): """ Shorthand for - :func:`~stem.descriptor.collector.CollecTor.get_exit_list` + :func:`~stem.descriptor.collector.CollecTor.get_exit_lists` on our singleton instance. """

- for desc in get_instance().get_exit_list(start, end, cache_to, timeout, retries): + for desc in get_instance().get_exit_lists(start, end, cache_to, timeout, retries): yield desc

@@ -594,9 +609,33 @@ class CollecTor(object): for desc in f.read(cache_to, 'dir-key-certificate-3', timeout = timeout, retries = retries): yield desc

- def get_exit_list(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + def get_bandwidth_files(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Bandwidth authority heuristics for the given time range, sorted oldest to + newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int timeout: timeout for downloading each individual archive when + the connection becomes idle, no timeout applied if **None** + :param int retries: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.bandwidth_file.BandwidthFile + for the given time range + + :raises: :class:`~stem.DownloadFailed` if the download fails + """ + + for f in self.files('bandwidth-file', start, end): + for desc in f.read(cache_to, 'bandwidth-file', timeout = timeout, retries = retries): + yield desc + + def get_exit_lists(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): """ - `TorDNSEL exit list https://www.torproject.org/projects/tordnsel.html.en`_ + `TorDNSEL exit lists https://www.torproject.org/projects/tordnsel.html.en`_ for the given time range, sorted oldest to newest.

:param datetime.datetime start: time range to begin with diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index e3ba1f40..44893fab 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -344,13 +344,28 @@ class TestCollector(unittest.TestCase):

@patch('stem.util.connection.download') @patch('stem.descriptor.collector.CollecTor.files') - def test_reading_exit_list(self, files_mock, download_mock): + def test_reading_bandwidth_files(self, files_mock, download_mock): + with open(get_resource('collector/bandwidths-2019-05-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/bandwidths/bandwidths-2019-05.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_bandwidth_files()) + self.assertEqual(2, len(descriptors)) + + f = descriptors[0] + self.assertEqual('BandwidthFile', type(f).__name__) + self.assertEqual(22, len(f.measurements)) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_exit_lists(self, files_mock, download_mock): with open(get_resource('collector/exit-list-2018-11-cropped.tar'), 'rb') as archive: download_mock.return_value = archive.read()

files_mock.return_value = [stem.descriptor.collector.File('archive/exit-lists/exit-list-2018-11.tar', 12345, '2016-09-04 09:21')]

- descriptors = list(stem.descriptor.collector.get_exit_list()) + descriptors = list(stem.descriptor.collector.get_exit_lists()) self.assertEqual(3713, len(descriptors))

f = descriptors[0] diff --git a/test/unit/descriptor/data/collector/bandwidths-2019-05-cropped.tar b/test/unit/descriptor/data/collector/bandwidths-2019-05-cropped.tar new file mode 100644 index 00000000..4f3c8724 Binary files /dev/null and b/test/unit/descriptor/data/collector/bandwidths-2019-05-cropped.tar differ