August 2019 - tor-commits - lists.torproject.org

[stem/master] Drop zstd from CollecTor unit tests
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit ae1970e2d8f25c097a61a48347a520d25314a5bc Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 5 14:49:15 2019 -0700 Drop zstd from CollecTor unit tests Oops! Just remembered why I didn't originally include zstd tests. CollecTor doesn't do zstd compression (just tor's DirPort). --- test/unit/descriptor/collector.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index cfb57c3d..260e246b 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -76,20 +76,6 @@ class TestCollector(unittest.TestCase): urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.lzma', timeout = None) @patch(URL_OPEN) - def test_download_zstd(self, urlopen_mock): - if not Compression.ZSTD.available: - self.skipTest('(zstd compression unavailable)') - return - - import zstd - compressor = zstd.ZstdCompressor() - urlopen_mock.return_value = io.BytesIO(compressor.compress(MINIMAL_INDEX_JSON)) - - collector = CollecTor(compression = Compression.ZSTD) - self.assertEqual(MINIMAL_INDEX, collector.index()) - urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.zst', timeout = None) - - @patch(URL_OPEN) def test_download_retries(self, urlopen_mock): urlopen_mock.side_effect = IOError('boom') @@ -118,7 +104,7 @@ class TestCollector(unittest.TestCase): self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index) def test_index_malformed_compression(self): - for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA, Compression.ZSTD): + for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA): if not compression.available: next

1 0

[stem/master] Python3 unit test fixes
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 2f0dc11f7e9bb4ee11e3c543ed993f4e7361987d Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 26 14:50:45 2019 -0700 Python3 unit test fixes Handful of tweaks to fix python3 compatability for our unit tests. Non-test changes are just non-behavioral (just renaming and such). --- stem/descriptor/collector.py | 32 +++++++++++++++++--------------- test/unit/descriptor/collector.py | 12 ++++++------ 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 1f7ddb96..69425dd5 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -432,21 +432,21 @@ class CollecTor(object): matches = [] - for entry in self._cached_files: - if start and (entry.start is None or entry.start < start): + for f in self._cached_files: + if start and (f.start is None or f.start < start): continue - elif end and (entry.end is None or entry.end > end): + elif end and (f.end is None or f.end > end): continue - if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in entry._guess_descriptor_types()]): - matches.append(entry) + if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in f._guess_descriptor_types()]): + matches.append(f) return matches @staticmethod def _files(val, path): """ - Provies a mapping of paths to files within the index. + Recursively provies files within the index. :param dict val: index hash :param list path: path we've transversed into @@ -454,16 +454,18 @@ class CollecTor(object): :returns: **list** of :class:`~stem.descriptor.collector.File` """ + if not isinstance(val, dict): + return [] # leaf node without any files + files = [] - if isinstance(val, dict): - for k, v in val.items(): - if k == 'files': - for attr in v: - file_path = '/'.join(path + [attr.get('path')]) - files.append(File(file_path, attr.get('size'), attr.get('last_modified'))) - elif k == 'directories': - for attr in v: - files.extend(CollecTor._files(attr, path + [attr.get('path')])) + for k, v in val.items(): + if k == 'files': + for attr in v: + file_path = '/'.join(path + [attr.get('path')]) + files.append(File(file_path, attr.get('size'), attr.get('last_modified'))) + elif k == 'directories': + for attr in v: + files.extend(CollecTor._files(attr, path + [attr.get('path')])) return files diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 914b52b9..ad0087dd 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -79,7 +79,7 @@ class TestCollector(unittest.TestCase): collector = CollecTor() self.assertEqual(MINIMAL_INDEX, collector.index(Compression.LZMA)) - urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.lzma', timeout = None) + urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.xz', timeout = None) @patch(URL_OPEN) def test_download_retries(self, urlopen_mock): @@ -131,7 +131,7 @@ class TestCollector(unittest.TestCase): self.assertEqual(85, len(files)) test_path = 'archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz' - extrainfo_file = filter(lambda x: x.path == test_path, files)[0] + extrainfo_file = list(filter(lambda x: x.path == test_path, files))[0] self.assertEqual(test_path, extrainfo_file.path) self.assertEqual(Compression.LZMA, extrainfo_file.compression) self.assertEqual(True, extrainfo_file.tar) @@ -185,7 +185,7 @@ class TestCollector(unittest.TestCase): 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', ] - self.assertEqual(expected, map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor'))) + self.assertEqual(expected, list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor')))) @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) def test_file_query_by_date(self): @@ -195,17 +195,17 @@ class TestCollector(unittest.TestCase): 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', - ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1)))) + ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1))))) self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', - ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', end = datetime.datetime(2007, 1, 1)))) + ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', end = datetime.datetime(2007, 1, 1))))) self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', - ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1)))) + ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1))))) def test_guess_descriptor_types(self): f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21')

1 0

[stem/master] Drop unused error_attr attribute
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit d2c6adf33a3d087a19d469ead0719cb8c3389f27 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 26 10:27:50 2019 -0700 Drop unused error_attr attribute Removing an unused attribute that is no longer set nor read. This was added in commit aebf3e4 to retain original stacktraces, then effectively reverted in commit 8c61fc8 without removing this attribute. --- stem/descriptor/remote.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index f715d743..251fc26f 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -477,8 +477,7 @@ class Query(object): self.fall_back_to_authority = fall_back_to_authority self.content = None - self.error = None # TODO: maybe remove in favor of error_attr in stem 2.x - self._error_attr = None + self.error = None self.is_done = False self.download_url = None

1 0

[stem/master] Note compression of files
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 9d9d635669126bf0f183cfe4d961dd9933508865 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 12 11:14:19 2019 -0700 Note compression of files Attribute on the File class we'll want for decompression. --- stem/descriptor/collector.py | 12 ++++++++++++ test/unit/descriptor/collector.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index d94d5871..a47612cb 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -161,16 +161,28 @@ class File(object): File within CollecTor. :var str path: file path within collector + :var stem.descriptor.Compression compression: file compression, **None** if + this cannot be determined + :var bool tar: **True** if a tarball, **False** otherwise :var int size: size of the file :var datetime last_modified: when the file was last modified """ def __init__(self, path, size, last_modified): self.path = path + self.compression = None + self.tar = path.endswith('.tar') or '.tar.' in path self.size = size self.last_modified = datetime.datetime.strptime(last_modified, '%Y-%m-%d %H:%M') self._guessed_type = None + if '.' not in self.path or self.path.endswith('.tar'): + self.compression = Compression.PLAINTEXT + else: + for compression in (Compression.LZMA, Compression.BZ2, Compression.GZIP): + if self.path.endswith(compression.extension): + self.compression = compression + def guess_descriptor_types(self): """ Descriptor @type this file is expected to have based on its path. If unable diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 609a4929..3c4d39a0 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -133,9 +133,24 @@ class TestCollector(unittest.TestCase): extrainfo_file = files[test_path] self.assertEqual(test_path, extrainfo_file.path) + self.assertEqual(Compression.LZMA, extrainfo_file.compression) + self.assertEqual(True, extrainfo_file.tar) self.assertEqual(6459884, extrainfo_file.size) self.assertEqual(datetime.datetime(2016, 6, 23, 9, 54), extrainfo_file.last_modified) + def test_file_compression_attributes(self): + f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') + self.assertEqual(Compression.LZMA, f.compression) + self.assertEqual(True, f.tar) + + f = File('archive/webstats/webstats-2015-03.tar', 20480, '2018-03-19 16:07') + self.assertEqual(Compression.PLAINTEXT, f.compression) + self.assertEqual(True, f.tar) + + f = File('recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos', 1162899, '2019-07-03 02:05') + self.assertEqual(Compression.PLAINTEXT, f.compression) + self.assertEqual(False, f.tar) + def test_guess_descriptor_types(self): f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21') self.assertEqual(('bridge-extra-info 1.3',), f.guess_descriptor_types())

1 0

[stem/master] Move compression argument into index method
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 601266ef7576de121d12c6b275df9b2103e336d7 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 23 14:27:09 2019 -0700 Move compression argument into index method Originally I thought that CollecTor offered each file with all compression types (like DirPorts) but on reflection that's wrong. The index is the only file that's compressed in multiple ways so moving the 'compression' argument from CollecTor's constructor to the index() method. Overall this makes things quite a bit cleaner. --- stem/descriptor/__init__.py | 9 ++++--- stem/descriptor/collector.py | 54 +++++++++++++++------------------------ test/unit/descriptor/collector.py | 48 +++++++++++++++++----------------- 3 files changed, 50 insertions(+), 61 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 2c69aef0..2044c7bf 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -225,14 +225,17 @@ class _Compression(object): """ if not self.available: - if self.name == 'zstd': + if self._name == 'zstd': raise ImportError('Decompressing zstd data requires https://pypi.org/project/zstandard/') - elif self.name == 'lzma': + elif self._name == 'lzma': raise ImportError('Decompressing lzma data requires https://docs.python.org/3/library/lzma.html') else: raise ImportError("'%s' decompression module is unavailable" % self._module_name) - return self._decompression_func(self._module, content) + try: + return self._decompression_func(self._module, content) + except Exception as exc: + raise IOError('Failed to decompress as %s: %s' % (self, exc)) def __str__(self): return self._name diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index bb9f1925..754a96b9 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -64,10 +64,6 @@ try: except ImportError: import urllib2 as urllib -import stem.prereq -import stem.util.enum -import stem.util.str_tools - COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old @@ -113,12 +109,11 @@ COLLECTOR_DESC_TYPES = { } -def _download(url, compression, timeout, retries): +def _download(url, timeout, retries): """ Download from the given url. :param str url: uncompressed url to download from - :param descriptor.Compression compression: decompression type :param int timeout: timeout when connection becomes idle, no timeout applied if **None** :param int retires: maximum attempts to impose @@ -135,13 +130,9 @@ def _download(url, compression, timeout, retries): """ start_time = time.time() - extension = compression.extension if compression not in (None, Compression.PLAINTEXT) else '' - - if not url.endswith(extension): - url += extension try: - response = urllib.urlopen(url, timeout = timeout).read() + return urllib.urlopen(url, timeout = timeout).read() except: exc = sys.exc_info()[1] @@ -150,19 +141,11 @@ def _download(url, compression, timeout, retries): if retries > 0 and (timeout is None or timeout > 0): log.debug("Failed to download from CollecTor at '%s' (%i retries remaining): %s" % (url, retries, exc)) - return _download(url, compression, timeout, retries - 1) + return _download(url, timeout, retries - 1) else: log.debug("Failed to download from CollecTor at '%s': %s" % (url, exc)) raise - if compression not in (None, Compression.PLAINTEXT): - try: - response = compression.decompress(response) - except Exception as exc: - raise IOError('Unable to decompress %s response from %s: %s' % (compression, url, exc)) - - return stem.util.str_tools._to_unicode(response) - class File(object): """ @@ -262,15 +245,12 @@ class CollecTor(object): provided in `an index <https://collector.torproject.org/index/index.json>`_ that's fetched as required. - :var descriptor.Compression compression: compression type to - download from, if undefiled we'll use the best decompression available :var int retries: number of times to attempt the request if downloading it fails :var float timeout: duration before we'll time out our request """ - def __init__(self, compression = 'best', retries = 2, timeout = None): - self.compression = Compression.PLAINTEXT + def __init__(self, retries = 2, timeout = None): self.retries = retries self.timeout = timeout @@ -278,18 +258,13 @@ class CollecTor(object): self._cached_files = None self._cached_index_at = 0 - if compression == 'best': - for option in (Compression.LZMA, Compression.BZ2, Compression.GZIP): - if option.available: - self.compression = option - break - elif compression is not None: - self.compression = compression - - def index(self): + def index(self, compression = 'best'): """ Provides the archives available in CollecTor. + :param descriptor.Compression compression: compression type to + download from, if undefiled we'll use the best decompression available + :returns: :class:`~stem.descriptor.collector.Index` with the archive contents @@ -303,7 +278,18 @@ class CollecTor(object): """ if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: - response = _download(COLLECTOR_URL + 'index/index.json', self.compression, self.timeout, self.retries) + if compression == 'best': + for option in (Compression.LZMA, Compression.BZ2, Compression.GZIP, Compression.PLAINTEXT): + if option.available: + compression = option + break + elif compression is None: + compression = Compression.PLAINTEXT + + extension = compression.extension if compression != Compression.PLAINTEXT else '' + url = COLLECTOR_URL + 'index/index.json' + extension + response = compression.decompress(_download(url, self.timeout, self.retries)) + self._cached_index = json.loads(response) self._cached_index_at = time.time() diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 9d442deb..86641f32 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -38,8 +38,8 @@ class TestCollector(unittest.TestCase): def test_download_plaintext(self, urlopen_mock): urlopen_mock.return_value = io.BytesIO(MINIMAL_INDEX_JSON) - collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(MINIMAL_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(MINIMAL_INDEX, collector.index(Compression.PLAINTEXT)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json', timeout = None) @patch(URL_OPEN) @@ -51,8 +51,8 @@ class TestCollector(unittest.TestCase): import zlib urlopen_mock.return_value = io.BytesIO(zlib.compress(MINIMAL_INDEX_JSON)) - collector = CollecTor(compression = Compression.GZIP) - self.assertEqual(MINIMAL_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(MINIMAL_INDEX, collector.index(Compression.GZIP)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None) @patch(URL_OPEN) @@ -64,8 +64,8 @@ class TestCollector(unittest.TestCase): import bz2 urlopen_mock.return_value = io.BytesIO(bz2.compress(MINIMAL_INDEX_JSON)) - collector = CollecTor(compression = Compression.BZ2) - self.assertEqual(MINIMAL_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(MINIMAL_INDEX, collector.index(Compression.BZ2)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None) @patch(URL_OPEN) @@ -77,8 +77,8 @@ class TestCollector(unittest.TestCase): import lzma urlopen_mock.return_value = io.BytesIO(lzma.compress(MINIMAL_INDEX_JSON)) - collector = CollecTor(compression = Compression.LZMA) - self.assertEqual(MINIMAL_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(MINIMAL_INDEX, collector.index(Compression.LZMA)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.lzma', timeout = None) @patch(URL_OPEN) @@ -97,35 +97,35 @@ class TestCollector(unittest.TestCase): @patch(URL_OPEN, Mock(return_value = io.BytesIO(MINIMAL_INDEX_JSON))) def test_index(self): - collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(MINIMAL_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(MINIMAL_INDEX, collector.index(Compression.PLAINTEXT)) @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json'))) def test_index_malformed_json(self): - collector = CollecTor(compression = Compression.PLAINTEXT) + collector = CollecTor() if stem.prereq.is_python_3(): - self.assertRaisesRegexp(ValueError, 'Expecting value: line 1 column 1', collector.index) + self.assertRaisesRegexp(ValueError, 'Expecting value: line 1 column 1', collector.index, Compression.PLAINTEXT) else: - self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index) + self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index, Compression.PLAINTEXT) def test_index_malformed_compression(self): for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA): if not compression.available: - next + continue with patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not compressed'))): - collector = CollecTor(compression = compression) - self.assertRaisesRegexp(IOError, 'Unable to decompress %s response' % compression, collector.index) + collector = CollecTor() + self.assertRaisesRegexp(IOError, 'Failed to decompress as %s' % compression, collector.index, compression) @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) def test_real_index(self): - collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(EXAMPLE_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(EXAMPLE_INDEX, collector.index(compression = Compression.PLAINTEXT)) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) def test_contents(self): - collector = CollecTor(compression = Compression.PLAINTEXT) + collector = CollecTor() files = collector.files() self.assertEqual(85, len(files)) @@ -172,9 +172,9 @@ class TestCollector(unittest.TestCase): f = File('recent/relay-descriptors/extra-infos/2019-07-03-23-05-00-extra-infos', 1162899, '2019-07-03 02:05') self.assertEqual(datetime.datetime(2019, 7, 4, 0, 5, 0), f.end) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) def test_file_query_by_type(self): - collector = CollecTor(compression = Compression.PLAINTEXT) + collector = CollecTor() expected = [ 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', @@ -187,9 +187,9 @@ class TestCollector(unittest.TestCase): self.assertEqual(expected, map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor'))) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) def test_file_query_by_date(self): - collector = CollecTor(compression = Compression.PLAINTEXT) + collector = CollecTor() self.assertEqual([ 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors',

1 0

[stem/master] Helper to guess descriptor type
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit bee688e60c19c5ce8879ffbcc1e0c6b6c2e8597c Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jul 11 14:35:40 2019 -0700 Helper to guess descriptor type Ideally our index would include descriptor types, but without that next best thing is to guess based on file paths. --- stem/descriptor/collector.py | 55 +++++++++++++++++++++++++++++++++++++++ test/unit/descriptor/collector.py | 15 ++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 1742c596..d94d5871 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -70,6 +70,40 @@ import stem.util.str_tools COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old +# mapping of path prefixes to their descriptor type (sampled 7/11/19) + +COLLECTOR_DESC_TYPES = { + 'archive/bridge-descriptors/server-descriptors/': 'bridge-server-descriptor 1.2', + 'archive/bridge-descriptors/extra-infos/': 'bridge-extra-info 1.3', + 'archive/bridge-descriptors/statuses/': 'bridge-network-status 1.1', + 'archive/bridge-pool-assignments/': 'bridge-pool-assignment 1.0', + 'archive/exit-lists/': 'tordnsel 1.0', + 'archive/relay-descriptors/bandwidths/': 'bandwidth-file 1.0', + 'archive/relay-descriptors/certs': 'dir-key-certificate-3 1.0', + 'archive/relay-descriptors/consensuses/': 'network-status-consensus-3 1.0', + 'archive/relay-descriptors/extra-infos/': 'extra-info 1.0', + 'archive/relay-descriptors/microdescs/': ('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'), + 'archive/relay-descriptors/server-descriptors/': 'server-descriptor 1.0', + 'archive/relay-descriptors/statuses/': 'network-status-2 1.0', + 'archive/relay-descriptors/tor/': 'directory 1.0', + 'archive/relay-descriptors/votes/': 'network-status-vote-3 1.0', + 'archive/torperf/': 'torperf 1.0', + 'archive/webstats/': (), + 'recent/bridge-descriptors/extra-infos/': 'bridge-extra-info 1.3', + 'recent/bridge-descriptors/server-descriptors/': 'bridge-server-descriptor 1.2', + 'recent/bridge-descriptors/statuses/': 'bridge-network-status 1.2', + 'recent/exit-lists/': 'tordnsel 1.0', + 'recent/relay-descriptors/bandwidths/': 'bandwidth-file 1.0', + 'recent/relay-descriptors/consensuses/': 'network-status-consensus-3 1.0', + 'recent/relay-descriptors/extra-infos/': 'extra-info 1.0', + 'recent/relay-descriptors/microdescs/consensus-microdesc/': 'network-status-microdesc-consensus-3 1.0', + 'recent/relay-descriptors/microdescs/micro/': 'microdescriptor 1.0', + 'recent/relay-descriptors/server-descriptors/': 'server-descriptor 1.0', + 'recent/relay-descriptors/votes/': 'network-status-vote-3 1.0', + 'recent/torperf/': 'torperf 1.1', + 'recent/webstats/': (), +} + def _download(url, compression, timeout, retries): """ @@ -135,6 +169,27 @@ class File(object): self.path = path self.size = size self.last_modified = datetime.datetime.strptime(last_modified, '%Y-%m-%d %H:%M') + self._guessed_type = None + + def guess_descriptor_types(self): + """ + Descriptor @type this file is expected to have based on its path. If unable + to determine any this tuple is empty. + + :returns: **tuple** with the descriptor types this file is expected to have + """ + + if self._guessed_type is None: + guessed_type = () + + for path_prefix, types in COLLECTOR_DESC_TYPES.items(): + if self.path.startswith(path_prefix): + guessed_type = (types,) if isinstance(types, str) else types + break + + self._guessed_type = guessed_type + + return self._guessed_type class CollecTor(object): diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 3403ee50..609a4929 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -9,7 +9,7 @@ import unittest import stem.prereq from stem.descriptor import Compression -from stem.descriptor.collector import CollecTor +from stem.descriptor.collector import CollecTor, File from test.unit.descriptor import get_resource from test.unit.descriptor.data.collector_index import EXAMPLE_INDEX @@ -135,3 +135,16 @@ class TestCollector(unittest.TestCase): self.assertEqual(test_path, extrainfo_file.path) self.assertEqual(6459884, extrainfo_file.size) self.assertEqual(datetime.datetime(2016, 6, 23, 9, 54), extrainfo_file.last_modified) + + def test_guess_descriptor_types(self): + f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21') + self.assertEqual(('bridge-extra-info 1.3',), f.guess_descriptor_types()) + + f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') + self.assertEqual(('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'), f.guess_descriptor_types()) + + f = File('archive/webstats/webstats-2015-03.tar', 20480, '2018-03-19 16:07') + self.assertEqual((), f.guess_descriptor_types()) + + f = File('archive/no_such_file.tar', 20480, '2018-03-19 16:07') + self.assertEqual((), f.guess_descriptor_types())

1 0

[stem/master] Initial download function
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 4196e37f0f43d033b23f740c1e3e3d9612ba4655 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jul 29 19:49:37 2019 -0700 Initial download function Few rough edges, but a copy-paste of the initial pydoc demo now works. --- stem/descriptor/collector.py | 69 +++++++++++++++++++++++++++++++++++--- stem/descriptor/remote.py | 6 +--- test/integ/descriptor/collector.py | 4 +-- 3 files changed, 67 insertions(+), 12 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index f76fa225..a78d60c4 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -16,7 +16,7 @@ With this you can either download and read directly from CollecTor... import datetime import stem.descriptor.collector - yesterday = datetime.date.today() - datetime.timedelta(1) + yesterday = datetime.datetime.today() - datetime.timedelta(1) # provide yesterday's exits @@ -33,7 +33,7 @@ With this you can either download and read directly from CollecTor... import stem.descriptor import stem.descriptor.collector - yesterday = datetime.date.today() - datetime.timedelta(1) + yesterday = datetime.datetime.today() - datetime.timedelta(1) path = os.path.expanduser('~/descriptor_cache/server_desc_today') with open(path, 'wb') as cache_file: @@ -53,6 +53,7 @@ import datetime import json import os import re +import shutil import sys import tempfile import time @@ -70,6 +71,7 @@ except ImportError: COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old +SINGLETON_COLLECTOR = None YEAR_DATE = re.compile('-(\\d{4})-(\\d{2})\\.') SEC_DATE = re.compile('(\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2})') @@ -113,6 +115,52 @@ COLLECTOR_DESC_TYPES = { } +def get_instance(): + """ + Provides the singleton :class:`~stem.descriptor.collector.CollecTor` + used for this module's shorthand functions. + + :returns: singleton :class:`~stem.descriptor.collector.CollecTor` instance + """ + + global SINGLETON_COLLECTOR + + if SINGLETON_COLLECTOR is None: + SINGLETON_COLLECTOR = CollecTor() + + return SINGLETON_COLLECTOR + + +def get_server_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Provides server descriptors for the given time range, sorted oldest to + newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int timeout: timeout for downloading each individual archive when the + connection becomes idle, no timeout applied if **None** + :param int retires: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.server_descriptor.ServerDescriptor` for the given + time range + + :raises: + * **socket.timeout** if our request timed out + * **urllib2.URLError** for most request failures + + Note that the urllib2 module may fail with other exception types, in + which case we'll pass it along. + """ + + for f in get_instance().files('server-descriptor', start, end): + for desc in f.read(cache_to, timeout = timeout, retries = retries): + yield desc + + def _download(url, timeout, retries): """ Download from the given url. @@ -229,13 +277,24 @@ class File(object): if self._downloaded_to and os.path.exists(self._downloaded_to): directory = os.path.dirname(self._downloaded_to) else: - with tempfile.TemporaryDirectory() as tmp_directory: - return self.read(tmp_directory, descriptor_type, timeout, retries) + # TODO: The following can be replaced with simpler usage of + # tempfile.TemporaryDirectory when we drop python 2.x support. + + tmp_directory = tempfile.mkdtemp() + + for desc in self.read(tmp_directory, descriptor_type, timeout, retries): + yield desc + + shutil.rmtree(tmp_directory) + + return # TODO: the following will not work if the tar contains multiple types or a type we do not support path = self.download(directory, True, timeout, retries) - return parse_file(path, descriptor_type) + + for desc in parse_file(path, descriptor_type): + yield desc def download(self, directory, decompress = True, timeout = None, retries = 3): """ diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 251fc26f..af24f624 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -164,11 +164,7 @@ DIR_PORT_BLACKLIST = ('tor26', 'Serge') def get_instance(): """ Provides the singleton :class:`~stem.descriptor.remote.DescriptorDownloader` - used for the following functions... - - * :func:`stem.descriptor.remote.get_server_descriptors` - * :func:`stem.descriptor.remote.get_extrainfo_descriptors` - * :func:`stem.descriptor.remote.get_consensus` + used for this module's shorthand functions. .. versionadded:: 1.5.0 diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py index dbb09d5a..6a0ec5ac 100644 --- a/test/integ/descriptor/collector.py +++ b/test/integ/descriptor/collector.py @@ -35,8 +35,8 @@ class TestCollector(unittest.TestCase): if compression and not compression.available: self.skipTest('(%s unavailable)' % compression) - collector = CollecTor(compression = compression) - index = collector.index() + collector = CollecTor() + index = collector.index(compression = compression) self.assertEqual('https://collector.torproject.org', index['path']) self.assertEqual(['archive', 'contrib', 'recent'], [entry['path'] for entry in index['directories']])

1 0

[stem/master] Adjust CollecTor File class
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 4c744badc658d3d93d113972bfbf7cb463298ee4 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jul 29 17:35:32 2019 -0700 Adjust CollecTor File class Handful of very tiny adjustments. Dropping the unused tar attribute, making descriptor guessing a static function (like the others), fixing some minor edge cases, etc. --- stem/descriptor/collector.py | 51 ++++++------ test/unit/descriptor/collector.py | 161 ++++++++++++++++---------------------- 2 files changed, 92 insertions(+), 120 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index d1c90e0e..f76fa225 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -157,7 +157,6 @@ class File(object): :var str path: file path within collector :var stem.descriptor.Compression compression: file compression, **None** if this cannot be determined - :var bool tar: **True** if a tarball, **False** otherwise :var int size: size of the file :var datetime start: beginning of the time range descriptors are for, @@ -170,13 +169,12 @@ class File(object): def __init__(self, path, size, last_modified): self.path = path self.compression = File._guess_compression(path) - self.tar = path.endswith('.tar') or '.tar.' in path self.size = size self.start, self.end = File._guess_time_range(path) self.last_modified = datetime.datetime.strptime(last_modified, '%Y-%m-%d %H:%M') - self._guessed_type = None + self._guessed_type = File._guess_descriptor_types(path) self._downloaded_to = None # location we last downloaded to def read(self, directory = None, descriptor_type = None, timeout = None, retries = 3): @@ -220,21 +218,21 @@ class File(object): """ if descriptor_type is None: - descriptor_types = self._guess_descriptor_types() - - if not descriptor_types: + if not self._guessed_type: raise ValueError("Unable to determine this file's descriptor type") - elif len(descriptor_types) > 1: - raise ValueError("Unable to determine disambiguate file's descriptor type from %s" % ', '.join(descriptor_types)) + elif len(self._guessed_type) > 1: + raise ValueError("Unable to determine disambiguate file's descriptor type from %s" % ', '.join(self._guessed_type)) - descriptor_type = descriptor_types[0] + descriptor_type = self._guessed_type[0] if directory is None: if self._downloaded_to and os.path.exists(self._downloaded_to): directory = os.path.dirname(self._downloaded_to) else: with tempfile.TemporaryDirectory() as tmp_directory: - return self.read(tmp_directory, timeout, retries) + return self.read(tmp_directory, descriptor_type, timeout, retries) + + # TODO: the following will not work if the tar contains multiple types or a type we do not support path = self.download(directory, True, timeout, retries) return parse_file(path, descriptor_type) @@ -267,7 +265,7 @@ class File(object): filename = self.path.split('/')[-1] - if decompress: + if self.compression != Compression.PLAINTEXT and decompress: filename = filename.rsplit('.', 1)[0] path = os.path.join(directory, filename) @@ -277,36 +275,35 @@ class File(object): elif os.path.exists(path): return path # file already exists - with open(path, 'wb') as output_file: - response = _download(COLLECTOR_URL + self.path, timeout, retries) + response = _download(COLLECTOR_URL + self.path, timeout, retries) - if decompress: - response = self.compression.decompress(response) + if decompress: + response = self.compression.decompress(response) + with open(path, 'wb') as output_file: output_file.write(response) self._downloaded_to = path return path - def _guess_descriptor_types(self): + @staticmethod + def _guess_descriptor_types(path): """ Descriptor @type this file is expected to have based on its path. If unable to determine any this tuple is empty. - :returns: **tuple** with the descriptor types this file is expected to have - """ + Hopefully this will be replaced with an explicit value in the future: - if self._guessed_type is None: - guessed_type = () + https://trac.torproject.org/projects/tor/ticket/31204 - for path_prefix, types in COLLECTOR_DESC_TYPES.items(): - if self.path.startswith(path_prefix): - guessed_type = (types,) if isinstance(types, str) else types - break + :returns: **tuple** with the descriptor types this file is expected to have + """ - self._guessed_type = guessed_type + for path_prefix, types in COLLECTOR_DESC_TYPES.items(): + if path.startswith(path_prefix): + return (types,) if isinstance(types, str) else types - return self._guessed_type + return () @staticmethod def _guess_compression(path): @@ -437,7 +434,7 @@ class CollecTor(object): elif end and (f.end is None or f.end > end): continue - if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in f._guess_descriptor_types()]): + if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in f._guessed_type]): matches.append(f) return matches diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index ad0087dd..77c1c460 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -21,68 +21,105 @@ except ImportError: URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' -MINIMAL_INDEX = { - 'index_created': '2017-12-25 21:06', - 'build_revision': '56a303e', - 'path': 'https://collector.torproject.org' -} - -MINIMAL_INDEX_JSON = b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org"}' with open(get_resource('collector_index.json'), 'rb') as index_file: - EXAMPLE_INDEX_CONTENT = index_file.read() + EXAMPLE_INDEX_JSON = index_file.read() class TestCollector(unittest.TestCase): + # tests for the File class + + def test_file_guess_descriptor_types(self): + test_values = { + 'archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz': ('bridge-extra-info 1.3',), + 'archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz': ('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'), + 'archive/webstats/webstats-2015-03.tar': (), + 'archive/no_such_file.tar': (), + } + + for path, expected in test_values.items(): + self.assertEqual(expected, File._guess_descriptor_types(path)) + + def test_file_guess_compression(self): + test_values = { + 'archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz': Compression.LZMA, + 'archive/webstats/webstats-2015-03.tar': Compression.PLAINTEXT, + 'recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos': Compression.PLAINTEXT, + } + + for path, expected in test_values.items(): + self.assertEqual(expected, File._guess_compression(path)) + + def test_file_guess_time_range(self): + test_values = { + 'archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz': + (datetime.datetime(2014, 1, 1), datetime.datetime(2014, 2, 1)), + 'recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos': + (datetime.datetime(2019, 7, 3, 2, 5, 0), datetime.datetime(2019, 7, 3, 3, 5, 0)), + 'archive/relay-descriptors/certs.tar.xz': + (None, None), + 'archive/relay-descriptors/microdescs/microdescs-2014-12.tar.xz': + (datetime.datetime(2014, 12, 1), datetime.datetime(2015, 1, 1)), + 'recent/relay-descriptors/extra-infos/2019-07-03-23-05-00-extra-infos': + (datetime.datetime(2019, 7, 3, 23, 5, 0), datetime.datetime(2019, 7, 4, 0, 5, 0)) + } + + for path, (expected_start, expected_end) in test_values.items(): + f = File(path, 7515396, '2014-02-07 03:59') + self.assertEqual(expected_start, f.start) + self.assertEqual(expected_end, f.end) + + # tests for the CollecTor class + @patch(URL_OPEN) - def test_download_plaintext(self, urlopen_mock): - urlopen_mock.return_value = io.BytesIO(MINIMAL_INDEX_JSON) + def test_index_plaintext(self, urlopen_mock): + urlopen_mock.return_value = io.BytesIO(EXAMPLE_INDEX_JSON) collector = CollecTor() - self.assertEqual(MINIMAL_INDEX, collector.index(Compression.PLAINTEXT)) + self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.PLAINTEXT)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json', timeout = None) @patch(URL_OPEN) - def test_download_gzip(self, urlopen_mock): + def test_index_gzip(self, urlopen_mock): if not Compression.GZIP.available: self.skipTest('(gzip compression unavailable)') return import zlib - urlopen_mock.return_value = io.BytesIO(zlib.compress(MINIMAL_INDEX_JSON)) + urlopen_mock.return_value = io.BytesIO(zlib.compress(EXAMPLE_INDEX_JSON)) collector = CollecTor() - self.assertEqual(MINIMAL_INDEX, collector.index(Compression.GZIP)) + self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.GZIP)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None) @patch(URL_OPEN) - def test_download_bz2(self, urlopen_mock): + def test_index_bz2(self, urlopen_mock): if not Compression.BZ2.available: self.skipTest('(bz2 compression unavailable)') return import bz2 - urlopen_mock.return_value = io.BytesIO(bz2.compress(MINIMAL_INDEX_JSON)) + urlopen_mock.return_value = io.BytesIO(bz2.compress(EXAMPLE_INDEX_JSON)) collector = CollecTor() - self.assertEqual(MINIMAL_INDEX, collector.index(Compression.BZ2)) + self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.BZ2)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None) @patch(URL_OPEN) - def test_download_lzma(self, urlopen_mock): + def test_index_lzma(self, urlopen_mock): if not Compression.LZMA.available: self.skipTest('(lzma compression unavailable)') return import lzma - urlopen_mock.return_value = io.BytesIO(lzma.compress(MINIMAL_INDEX_JSON)) + urlopen_mock.return_value = io.BytesIO(lzma.compress(EXAMPLE_INDEX_JSON)) collector = CollecTor() - self.assertEqual(MINIMAL_INDEX, collector.index(Compression.LZMA)) + self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.LZMA)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.xz', timeout = None) @patch(URL_OPEN) - def test_download_retries(self, urlopen_mock): + def test_index_retries(self, urlopen_mock): urlopen_mock.side_effect = IOError('boom') collector = CollecTor(retries = 0) @@ -95,11 +132,6 @@ class TestCollector(unittest.TestCase): self.assertRaisesRegexp(IOError, 'boom', collector.index) self.assertEqual(5, urlopen_mock.call_count) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(MINIMAL_INDEX_JSON))) - def test_index(self): - collector = CollecTor() - self.assertEqual(MINIMAL_INDEX, collector.index(Compression.PLAINTEXT)) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json'))) def test_index_malformed_json(self): collector = CollecTor() @@ -118,104 +150,47 @@ class TestCollector(unittest.TestCase): collector = CollecTor() self.assertRaisesRegexp(IOError, 'Failed to decompress as %s' % compression, collector.index, compression) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) - def test_real_index(self): - collector = CollecTor() - self.assertEqual(EXAMPLE_INDEX, collector.index(compression = Compression.PLAINTEXT)) - @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) - def test_contents(self): + def test_files(self): collector = CollecTor() files = collector.files() - self.assertEqual(85, len(files)) - test_path = 'archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz' - extrainfo_file = list(filter(lambda x: x.path == test_path, files))[0] - self.assertEqual(test_path, extrainfo_file.path) + extrainfo_file = list(filter(lambda x: x.path.endswith('extra-infos-2007-09.tar.xz'), files))[0] + self.assertEqual('archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz', extrainfo_file.path) self.assertEqual(Compression.LZMA, extrainfo_file.compression) - self.assertEqual(True, extrainfo_file.tar) self.assertEqual(6459884, extrainfo_file.size) self.assertEqual(datetime.datetime(2016, 6, 23, 9, 54), extrainfo_file.last_modified) - def test_file_compression_attributes(self): - f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') - self.assertEqual(Compression.LZMA, f.compression) - self.assertEqual(True, f.tar) - - f = File('archive/webstats/webstats-2015-03.tar', 20480, '2018-03-19 16:07') - self.assertEqual(Compression.PLAINTEXT, f.compression) - self.assertEqual(True, f.tar) - - f = File('recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos', 1162899, '2019-07-03 02:05') - self.assertEqual(Compression.PLAINTEXT, f.compression) - self.assertEqual(False, f.tar) - - def test_file_date_attributes(self): - f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') - self.assertEqual(datetime.datetime(2014, 1, 1), f.start) - self.assertEqual(datetime.datetime(2014, 2, 1), f.end) - - f = File('recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos', 1162899, '2019-07-03 02:05') - self.assertEqual(datetime.datetime(2019, 7, 3, 2, 5, 0), f.start) - self.assertEqual(datetime.datetime(2019, 7, 3, 3, 5, 0), f.end) - - f = File('archive/relay-descriptors/certs.tar.xz', 144696, '2019-07-03 03:29') - self.assertEqual(None, f.start) - self.assertEqual(None, f.end) - - # check date boundaries - - f = File('archive/relay-descriptors/microdescs/microdescs-2014-12.tar.xz', 7515396, '2014-02-07 03:59') - self.assertEqual(datetime.datetime(2015, 1, 1), f.end) - - f = File('recent/relay-descriptors/extra-infos/2019-07-03-23-05-00-extra-infos', 1162899, '2019-07-03 02:05') - self.assertEqual(datetime.datetime(2019, 7, 4, 0, 5, 0), f.end) - @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) - def test_file_query_by_type(self): + def test_files_by_descriptor_type(self): collector = CollecTor() - expected = [ + self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', - ] - - self.assertEqual(expected, list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor')))) + ], [f.path for f in collector.files(descriptor_type = 'server-descriptor')]) @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) - def test_file_query_by_date(self): + def test_file_by_date(self): collector = CollecTor() self.assertEqual([ 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', - ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1))))) + ], [f.path for f in collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1))]) self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', - ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', end = datetime.datetime(2007, 1, 1))))) + ], [f.path for f in collector.files(descriptor_type = 'server-descriptor', end = datetime.datetime(2007, 1, 1))]) self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', - ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1))))) - - def test_guess_descriptor_types(self): - f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21') - self.assertEqual(('bridge-extra-info 1.3',), f._guess_descriptor_types()) - - f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') - self.assertEqual(('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'), f._guess_descriptor_types()) - - f = File('archive/webstats/webstats-2015-03.tar', 20480, '2018-03-19 16:07') - self.assertEqual((), f._guess_descriptor_types()) - - f = File('archive/no_such_file.tar', 20480, '2018-03-19 16:07') - self.assertEqual((), f._guess_descriptor_types()) + ], [f.path for f in collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1))])

1 0

[stem/master] Filter files by descriptor type or time
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 756aba1eb6c73c79bc7c09b151c93c3c3a3c3c80 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jul 22 15:42:32 2019 -0700 Filter files by descriptor type or time Methods I'll be offering to users include filtering by time range or descriptor type so including these as arguments in our files() method. --- stem/descriptor/collector.py | 38 +++++++++++++++++++++++++++++--------- test/unit/descriptor/collector.py | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index c1458784..426e48ae 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -74,6 +74,9 @@ REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old YEAR_DATE = re.compile('-(\\d{4})-(\\d{2})\\.') SEC_DATE = re.compile('(\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2})') +# distant future date so we can sort files without a timestamp at the end + +FUTURE = datetime.datetime(9999, 1, 1) # mapping of path prefixes to their descriptor type (sampled 7/11/19) @@ -272,7 +275,7 @@ class CollecTor(object): self.timeout = timeout self._cached_index = None - self._cached_files = None # {path => file} mappings in the index + self._cached_files = None self._cached_index_at = 0 if compression == 'best': @@ -306,11 +309,15 @@ class CollecTor(object): return self._cached_index - def files(self): + def files(self, descriptor_type = None, start = None, end = None): """ - Provides files CollecTor presently has. + Provides files CollecTor presently has, sorted oldest to newest. + + :param str descriptor_type: descriptor type or prefix to retrieve + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with - :returns: **hash** mapping paths to :class:`~stem.descriptor.collector.File` + :returns: **list** of :class:`~stem.descriptor.collector.File` :raises: If unable to retrieve the index this provide... @@ -324,7 +331,20 @@ class CollecTor(object): if not self._cached_files or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: self._cached_files = CollecTor._files(self.index(), []) - return self._cached_files + matches = [] + + for entry in self._cached_files: + if start and (entry.start is None or entry.start < start): + continue + elif end and (entry.end is None or entry.end > end): + continue + + if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in entry.guess_descriptor_types()]): + matches.append(entry) + + matches.sort(key = lambda x: x.start if x.start else FUTURE) + + return matches @staticmethod def _files(val, path): @@ -334,19 +354,19 @@ class CollecTor(object): :param dict val: index hash :param list path: path we've transversed into - :returns: **dict** mapping paths to files + :returns: **list** of :class:`~stem.descriptor.collector.File` """ - files = {} + files = [] if isinstance(val, dict): for k, v in val.items(): if k == 'files': for attr in v: file_path = '/'.join(path + [attr.get('path')]) - files[file_path] = File(file_path, attr.get('size'), attr.get('last_modified')) + files.append(File(file_path, attr.get('size'), attr.get('last_modified'))) elif k == 'directories': for attr in v: - files.update(CollecTor._files(attr, path + [attr.get('path')])) + files.extend(CollecTor._files(attr, path + [attr.get('path')])) return files diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 297050ea..9d442deb 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -131,7 +131,7 @@ class TestCollector(unittest.TestCase): self.assertEqual(85, len(files)) test_path = 'archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz' - extrainfo_file = files[test_path] + extrainfo_file = filter(lambda x: x.path == test_path, files)[0] self.assertEqual(test_path, extrainfo_file.path) self.assertEqual(Compression.LZMA, extrainfo_file.compression) self.assertEqual(True, extrainfo_file.tar) @@ -172,6 +172,41 @@ class TestCollector(unittest.TestCase): f = File('recent/relay-descriptors/extra-infos/2019-07-03-23-05-00-extra-infos', 1162899, '2019-07-03 02:05') self.assertEqual(datetime.datetime(2019, 7, 4, 0, 5, 0), f.end) + @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + def test_file_query_by_type(self): + collector = CollecTor(compression = Compression.PLAINTEXT) + + expected = [ + 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', + 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', + 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', + 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', + 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', + 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', + ] + + self.assertEqual(expected, map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor'))) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + def test_file_query_by_date(self): + collector = CollecTor(compression = Compression.PLAINTEXT) + + self.assertEqual([ + 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', + 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', + 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', + ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1)))) + + self.assertEqual([ + 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', + 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', + 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', + ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', end = datetime.datetime(2007, 1, 1)))) + + self.assertEqual([ + 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', + ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1)))) + def test_guess_descriptor_types(self): f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21') self.assertEqual(('bridge-extra-info 1.3',), f.guess_descriptor_types())

1 0

[stem/master] Default compression attribute to plaintext
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 0519cf57fcfe54de71956d7127bd8551ede2dfac Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 28 14:27:31 2019 -0700 Default compression attribute to plaintext Unrecognized file extensions should be treated as being uncompressed. For instance, if CollecTor includes a json file we want to download it as-is (rather than choke on an unexpected None value). --- stem/descriptor/collector.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 69425dd5..d1c90e0e 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -314,12 +314,11 @@ class File(object): Determine file comprssion from CollecTor's filename. """ - if '.' not in path or path.endswith('.tar'): - return Compression.PLAINTEXT - else: - for compression in (Compression.LZMA, Compression.BZ2, Compression.GZIP): - if path.endswith(compression.extension): - return compression + for compression in (Compression.LZMA, Compression.BZ2, Compression.GZIP): + if path.endswith(compression.extension): + return compression + + return Compression.PLAINTEXT @staticmethod def _guess_time_range(path):

1 0