commit 60d714a33ca055fcefdad26493b4787483b1d71a Author: Damian Johnson atagar@torproject.org Date: Wed Aug 7 18:36:53 2019 -0700
Download method for microdescriptors --- stem/descriptor/collector.py | 39 +++++++++++++++++++++++++++++++++++++- test/integ/descriptor/collector.py | 13 ++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-)
diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 82a213f3..c8c4a1b7 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -50,7 +50,8 @@ With this you can either download and read directly from CollecTor...
get_instance - Provides a singleton CollecTor used for... |- get_server_descriptors - published server descriptors - +- get_extrainfo_descriptors - published extrainfo descriptors + |- get_extrainfo_descriptors - published extrainfo descriptors + +- get_microdescriptors - published microdescriptors
File - Individual file residing within CollecTor |- read - provides descriptors from this file @@ -59,6 +60,7 @@ With this you can either download and read directly from CollecTor... CollecTor - Downloader for descriptors from CollecTor |- get_server_descriptors - published server descriptors |- get_extrainfo_descriptors - published extrainfo descriptors + |- get_microdescriptors - published microdescriptors | |- index - metadata for content available from CollecTor +- files - files available from CollecTor @@ -163,6 +165,17 @@ def get_extrainfo_descriptors(start = None, end = None, cache_to = None, timeout yield desc
+def get_microdescriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Shorthand for + :func:`~stem.descriptor.collector.CollecTor.get_microdescriptors` + on our singleton instance. + """ + + for desc in get_instance().get_microdescriptors(start, end, cache_to, timeout, retries): + yield desc + + class File(object): """ File within CollecTor. @@ -430,6 +443,30 @@ class CollecTor(object): for desc in f.read(cache_to, timeout = timeout, retries = retries): yield desc
+ def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Provides microdescriptors published during the given time range, + sorted oldest to newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int timeout: timeout for downloading each individual archive when + the connection becomes idle, no timeout applied if **None** + :param int retires: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.microdescriptor.Microdescriptor + for the given time range + + :raises: :class:`~stem.DownloadFailed` if the download fails + """ + + for f in self.files('microdescriptor', start, end): + for desc in f.read(cache_to, timeout = timeout, retries = retries): + yield desc + def index(self, compression = 'best'): """ Provides the archives available in CollecTor. diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py index 1a2e1e32..db57df5e 100644 --- a/test/integ/descriptor/collector.py +++ b/test/integ/descriptor/collector.py @@ -49,7 +49,18 @@ class TestCollector(unittest.TestCase): recent_descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors(start = RECENT))
if not (300 < len(recent_descriptors) < 800): - self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) + self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) # 583 on 8/7/19 + + @test.require.only_run_once + @test.require.online + def test_downloading_microdescriptors(self): + recent_descriptors = list(stem.descriptor.collector.get_microdescriptors(start = RECENT)) + + # TODO: I'm unsure why these counts differ so much from server/extrainfo + # descriptors. Checking with Karsten. + + if not (300 < len(recent_descriptors) < 800): + self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors)) # 23 on 8/7/19
def _test_index(self, compression): if compression and not compression.available: