tor-commits August 2019

tor-commits@lists.torproject.org

19 participants
2737 discussions

[stem/master] Python3 unit test fixes
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 2f0dc11f7e9bb4ee11e3c543ed993f4e7361987d Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 26 14:50:45 2019 -0700 Python3 unit test fixes Handful of tweaks to fix python3 compatability for our unit tests. Non-test changes are just non-behavioral (just renaming and such). --- stem/descriptor/collector.py | 32 +++++++++++++++++--------------- test/unit/descriptor/collector.py | 12 ++++++------ 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 1f7ddb96..69425dd5 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -432,21 +432,21 @@ class CollecTor(object): matches = [] - for entry in self._cached_files: - if start and (entry.start is None or entry.start < start): + for f in self._cached_files: + if start and (f.start is None or f.start < start): continue - elif end and (entry.end is None or entry.end > end): + elif end and (f.end is None or f.end > end): continue - if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in entry._guess_descriptor_types()]): - matches.append(entry) + if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in f._guess_descriptor_types()]): + matches.append(f) return matches @staticmethod def _files(val, path): """ - Provies a mapping of paths to files within the index. + Recursively provies files within the index. :param dict val: index hash :param list path: path we've transversed into @@ -454,16 +454,18 @@ class CollecTor(object): :returns: **list** of :class:`~stem.descriptor.collector.File` """ + if not isinstance(val, dict): + return [] # leaf node without any files + files = [] - if isinstance(val, dict): - for k, v in val.items(): - if k == 'files': - for attr in v: - file_path = '/'.join(path + [attr.get('path')]) - files.append(File(file_path, attr.get('size'), attr.get('last_modified'))) - elif k == 'directories': - for attr in v: - files.extend(CollecTor._files(attr, path + [attr.get('path')])) + for k, v in val.items(): + if k == 'files': + for attr in v: + file_path = '/'.join(path + [attr.get('path')]) + files.append(File(file_path, attr.get('size'), attr.get('last_modified'))) + elif k == 'directories': + for attr in v: + files.extend(CollecTor._files(attr, path + [attr.get('path')])) return files diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 914b52b9..ad0087dd 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -79,7 +79,7 @@ class TestCollector(unittest.TestCase): collector = CollecTor() self.assertEqual(MINIMAL_INDEX, collector.index(Compression.LZMA)) - urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.lzma', timeout = None) + urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.xz', timeout = None) @patch(URL_OPEN) def test_download_retries(self, urlopen_mock): @@ -131,7 +131,7 @@ class TestCollector(unittest.TestCase): self.assertEqual(85, len(files)) test_path = 'archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz' - extrainfo_file = filter(lambda x: x.path == test_path, files)[0] + extrainfo_file = list(filter(lambda x: x.path == test_path, files))[0] self.assertEqual(test_path, extrainfo_file.path) self.assertEqual(Compression.LZMA, extrainfo_file.compression) self.assertEqual(True, extrainfo_file.tar) @@ -185,7 +185,7 @@ class TestCollector(unittest.TestCase): 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', ] - self.assertEqual(expected, map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor'))) + self.assertEqual(expected, list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor')))) @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) def test_file_query_by_date(self): @@ -195,17 +195,17 @@ class TestCollector(unittest.TestCase): 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', - ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1)))) + ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1))))) self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', - ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', end = datetime.datetime(2007, 1, 1)))) + ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', end = datetime.datetime(2007, 1, 1))))) self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', - ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1)))) + ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1))))) def test_guess_descriptor_types(self): f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21')

1 0

[stem/master] Drop unused error_attr attribute
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit d2c6adf33a3d087a19d469ead0719cb8c3389f27 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 26 10:27:50 2019 -0700 Drop unused error_attr attribute Removing an unused attribute that is no longer set nor read. This was added in commit aebf3e4 to retain original stacktraces, then effectively reverted in commit 8c61fc8 without removing this attribute. --- stem/descriptor/remote.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index f715d743..251fc26f 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -477,8 +477,7 @@ class Query(object): self.fall_back_to_authority = fall_back_to_authority self.content = None - self.error = None # TODO: maybe remove in favor of error_attr in stem 2.x - self._error_attr = None + self.error = None self.is_done = False self.download_url = None

1 0

[stem/master] Note compression of files
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 9d9d635669126bf0f183cfe4d961dd9933508865 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 12 11:14:19 2019 -0700 Note compression of files Attribute on the File class we'll want for decompression. --- stem/descriptor/collector.py | 12 ++++++++++++ test/unit/descriptor/collector.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index d94d5871..a47612cb 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -161,16 +161,28 @@ class File(object): File within CollecTor. :var str path: file path within collector + :var stem.descriptor.Compression compression: file compression, **None** if + this cannot be determined + :var bool tar: **True** if a tarball, **False** otherwise :var int size: size of the file :var datetime last_modified: when the file was last modified """ def __init__(self, path, size, last_modified): self.path = path + self.compression = None + self.tar = path.endswith('.tar') or '.tar.' in path self.size = size self.last_modified = datetime.datetime.strptime(last_modified, '%Y-%m-%d %H:%M') self._guessed_type = None + if '.' not in self.path or self.path.endswith('.tar'): + self.compression = Compression.PLAINTEXT + else: + for compression in (Compression.LZMA, Compression.BZ2, Compression.GZIP): + if self.path.endswith(compression.extension): + self.compression = compression + def guess_descriptor_types(self): """ Descriptor @type this file is expected to have based on its path. If unable diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 609a4929..3c4d39a0 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -133,9 +133,24 @@ class TestCollector(unittest.TestCase): extrainfo_file = files[test_path] self.assertEqual(test_path, extrainfo_file.path) + self.assertEqual(Compression.LZMA, extrainfo_file.compression) + self.assertEqual(True, extrainfo_file.tar) self.assertEqual(6459884, extrainfo_file.size) self.assertEqual(datetime.datetime(2016, 6, 23, 9, 54), extrainfo_file.last_modified) + def test_file_compression_attributes(self): + f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') + self.assertEqual(Compression.LZMA, f.compression) + self.assertEqual(True, f.tar) + + f = File('archive/webstats/webstats-2015-03.tar', 20480, '2018-03-19 16:07') + self.assertEqual(Compression.PLAINTEXT, f.compression) + self.assertEqual(True, f.tar) + + f = File('recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos', 1162899, '2019-07-03 02:05') + self.assertEqual(Compression.PLAINTEXT, f.compression) + self.assertEqual(False, f.tar) + def test_guess_descriptor_types(self): f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21') self.assertEqual(('bridge-extra-info 1.3',), f.guess_descriptor_types())

1 0

[stem/master] Move compression argument into index method
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 601266ef7576de121d12c6b275df9b2103e336d7 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 23 14:27:09 2019 -0700 Move compression argument into index method Originally I thought that CollecTor offered each file with all compression types (like DirPorts) but on reflection that's wrong. The index is the only file that's compressed in multiple ways so moving the 'compression' argument from CollecTor's constructor to the index() method. Overall this makes things quite a bit cleaner. --- stem/descriptor/__init__.py | 9 ++++--- stem/descriptor/collector.py | 54 +++++++++++++++------------------------ test/unit/descriptor/collector.py | 48 +++++++++++++++++----------------- 3 files changed, 50 insertions(+), 61 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 2c69aef0..2044c7bf 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -225,14 +225,17 @@ class _Compression(object): """ if not self.available: - if self.name == 'zstd': + if self._name == 'zstd': raise ImportError('Decompressing zstd data requires https://pypi.org/project/zstandard/') - elif self.name == 'lzma': + elif self._name == 'lzma': raise ImportError('Decompressing lzma data requires https://docs.python.org/3/library/lzma.html') else: raise ImportError("'%s' decompression module is unavailable" % self._module_name) - return self._decompression_func(self._module, content) + try: + return self._decompression_func(self._module, content) + except Exception as exc: + raise IOError('Failed to decompress as %s: %s' % (self, exc)) def __str__(self): return self._name diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index bb9f1925..754a96b9 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -64,10 +64,6 @@ try: except ImportError: import urllib2 as urllib -import stem.prereq -import stem.util.enum -import stem.util.str_tools - COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old @@ -113,12 +109,11 @@ COLLECTOR_DESC_TYPES = { } -def _download(url, compression, timeout, retries): +def _download(url, timeout, retries): """ Download from the given url. :param str url: uncompressed url to download from - :param descriptor.Compression compression: decompression type :param int timeout: timeout when connection becomes idle, no timeout applied if **None** :param int retires: maximum attempts to impose @@ -135,13 +130,9 @@ def _download(url, compression, timeout, retries): """ start_time = time.time() - extension = compression.extension if compression not in (None, Compression.PLAINTEXT) else '' - - if not url.endswith(extension): - url += extension try: - response = urllib.urlopen(url, timeout = timeout).read() + return urllib.urlopen(url, timeout = timeout).read() except: exc = sys.exc_info()[1] @@ -150,19 +141,11 @@ def _download(url, compression, timeout, retries): if retries > 0 and (timeout is None or timeout > 0): log.debug("Failed to download from CollecTor at '%s' (%i retries remaining): %s" % (url, retries, exc)) - return _download(url, compression, timeout, retries - 1) + return _download(url, timeout, retries - 1) else: log.debug("Failed to download from CollecTor at '%s': %s" % (url, exc)) raise - if compression not in (None, Compression.PLAINTEXT): - try: - response = compression.decompress(response) - except Exception as exc: - raise IOError('Unable to decompress %s response from %s: %s' % (compression, url, exc)) - - return stem.util.str_tools._to_unicode(response) - class File(object): """ @@ -262,15 +245,12 @@ class CollecTor(object): provided in `an index <https://collector.torproject.org/index/index.json>`_ that's fetched as required. - :var descriptor.Compression compression: compression type to - download from, if undefiled we'll use the best decompression available :var int retries: number of times to attempt the request if downloading it fails :var float timeout: duration before we'll time out our request """ - def __init__(self, compression = 'best', retries = 2, timeout = None): - self.compression = Compression.PLAINTEXT + def __init__(self, retries = 2, timeout = None): self.retries = retries self.timeout = timeout @@ -278,18 +258,13 @@ class CollecTor(object): self._cached_files = None self._cached_index_at = 0 - if compression == 'best': - for option in (Compression.LZMA, Compression.BZ2, Compression.GZIP): - if option.available: - self.compression = option - break - elif compression is not None: - self.compression = compression - - def index(self): + def index(self, compression = 'best'): """ Provides the archives available in CollecTor. + :param descriptor.Compression compression: compression type to + download from, if undefiled we'll use the best decompression available + :returns: :class:`~stem.descriptor.collector.Index` with the archive contents @@ -303,7 +278,18 @@ class CollecTor(object): """ if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: - response = _download(COLLECTOR_URL + 'index/index.json', self.compression, self.timeout, self.retries) + if compression == 'best': + for option in (Compression.LZMA, Compression.BZ2, Compression.GZIP, Compression.PLAINTEXT): + if option.available: + compression = option + break + elif compression is None: + compression = Compression.PLAINTEXT + + extension = compression.extension if compression != Compression.PLAINTEXT else '' + url = COLLECTOR_URL + 'index/index.json' + extension + response = compression.decompress(_download(url, self.timeout, self.retries)) + self._cached_index = json.loads(response) self._cached_index_at = time.time() diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 9d442deb..86641f32 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -38,8 +38,8 @@ class TestCollector(unittest.TestCase): def test_download_plaintext(self, urlopen_mock): urlopen_mock.return_value = io.BytesIO(MINIMAL_INDEX_JSON) - collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(MINIMAL_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(MINIMAL_INDEX, collector.index(Compression.PLAINTEXT)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json', timeout = None) @patch(URL_OPEN) @@ -51,8 +51,8 @@ class TestCollector(unittest.TestCase): import zlib urlopen_mock.return_value = io.BytesIO(zlib.compress(MINIMAL_INDEX_JSON)) - collector = CollecTor(compression = Compression.GZIP) - self.assertEqual(MINIMAL_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(MINIMAL_INDEX, collector.index(Compression.GZIP)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None) @patch(URL_OPEN) @@ -64,8 +64,8 @@ class TestCollector(unittest.TestCase): import bz2 urlopen_mock.return_value = io.BytesIO(bz2.compress(MINIMAL_INDEX_JSON)) - collector = CollecTor(compression = Compression.BZ2) - self.assertEqual(MINIMAL_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(MINIMAL_INDEX, collector.index(Compression.BZ2)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None) @patch(URL_OPEN) @@ -77,8 +77,8 @@ class TestCollector(unittest.TestCase): import lzma urlopen_mock.return_value = io.BytesIO(lzma.compress(MINIMAL_INDEX_JSON)) - collector = CollecTor(compression = Compression.LZMA) - self.assertEqual(MINIMAL_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(MINIMAL_INDEX, collector.index(Compression.LZMA)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.lzma', timeout = None) @patch(URL_OPEN) @@ -97,35 +97,35 @@ class TestCollector(unittest.TestCase): @patch(URL_OPEN, Mock(return_value = io.BytesIO(MINIMAL_INDEX_JSON))) def test_index(self): - collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(MINIMAL_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(MINIMAL_INDEX, collector.index(Compression.PLAINTEXT)) @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json'))) def test_index_malformed_json(self): - collector = CollecTor(compression = Compression.PLAINTEXT) + collector = CollecTor() if stem.prereq.is_python_3(): - self.assertRaisesRegexp(ValueError, 'Expecting value: line 1 column 1', collector.index) + self.assertRaisesRegexp(ValueError, 'Expecting value: line 1 column 1', collector.index, Compression.PLAINTEXT) else: - self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index) + self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index, Compression.PLAINTEXT) def test_index_malformed_compression(self): for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA): if not compression.available: - next + continue with patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not compressed'))): - collector = CollecTor(compression = compression) - self.assertRaisesRegexp(IOError, 'Unable to decompress %s response' % compression, collector.index) + collector = CollecTor() + self.assertRaisesRegexp(IOError, 'Failed to decompress as %s' % compression, collector.index, compression) @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) def test_real_index(self): - collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(EXAMPLE_INDEX, collector.index()) + collector = CollecTor() + self.assertEqual(EXAMPLE_INDEX, collector.index(compression = Compression.PLAINTEXT)) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) def test_contents(self): - collector = CollecTor(compression = Compression.PLAINTEXT) + collector = CollecTor() files = collector.files() self.assertEqual(85, len(files)) @@ -172,9 +172,9 @@ class TestCollector(unittest.TestCase): f = File('recent/relay-descriptors/extra-infos/2019-07-03-23-05-00-extra-infos', 1162899, '2019-07-03 02:05') self.assertEqual(datetime.datetime(2019, 7, 4, 0, 5, 0), f.end) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) def test_file_query_by_type(self): - collector = CollecTor(compression = Compression.PLAINTEXT) + collector = CollecTor() expected = [ 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', @@ -187,9 +187,9 @@ class TestCollector(unittest.TestCase): self.assertEqual(expected, map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor'))) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) def test_file_query_by_date(self): - collector = CollecTor(compression = Compression.PLAINTEXT) + collector = CollecTor() self.assertEqual([ 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors',

1 0

[stem/master] Helper to guess descriptor type
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit bee688e60c19c5ce8879ffbcc1e0c6b6c2e8597c Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jul 11 14:35:40 2019 -0700 Helper to guess descriptor type Ideally our index would include descriptor types, but without that next best thing is to guess based on file paths. --- stem/descriptor/collector.py | 55 +++++++++++++++++++++++++++++++++++++++ test/unit/descriptor/collector.py | 15 ++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 1742c596..d94d5871 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -70,6 +70,40 @@ import stem.util.str_tools COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old +# mapping of path prefixes to their descriptor type (sampled 7/11/19) + +COLLECTOR_DESC_TYPES = { + 'archive/bridge-descriptors/server-descriptors/': 'bridge-server-descriptor 1.2', + 'archive/bridge-descriptors/extra-infos/': 'bridge-extra-info 1.3', + 'archive/bridge-descriptors/statuses/': 'bridge-network-status 1.1', + 'archive/bridge-pool-assignments/': 'bridge-pool-assignment 1.0', + 'archive/exit-lists/': 'tordnsel 1.0', + 'archive/relay-descriptors/bandwidths/': 'bandwidth-file 1.0', + 'archive/relay-descriptors/certs': 'dir-key-certificate-3 1.0', + 'archive/relay-descriptors/consensuses/': 'network-status-consensus-3 1.0', + 'archive/relay-descriptors/extra-infos/': 'extra-info 1.0', + 'archive/relay-descriptors/microdescs/': ('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'), + 'archive/relay-descriptors/server-descriptors/': 'server-descriptor 1.0', + 'archive/relay-descriptors/statuses/': 'network-status-2 1.0', + 'archive/relay-descriptors/tor/': 'directory 1.0', + 'archive/relay-descriptors/votes/': 'network-status-vote-3 1.0', + 'archive/torperf/': 'torperf 1.0', + 'archive/webstats/': (), + 'recent/bridge-descriptors/extra-infos/': 'bridge-extra-info 1.3', + 'recent/bridge-descriptors/server-descriptors/': 'bridge-server-descriptor 1.2', + 'recent/bridge-descriptors/statuses/': 'bridge-network-status 1.2', + 'recent/exit-lists/': 'tordnsel 1.0', + 'recent/relay-descriptors/bandwidths/': 'bandwidth-file 1.0', + 'recent/relay-descriptors/consensuses/': 'network-status-consensus-3 1.0', + 'recent/relay-descriptors/extra-infos/': 'extra-info 1.0', + 'recent/relay-descriptors/microdescs/consensus-microdesc/': 'network-status-microdesc-consensus-3 1.0', + 'recent/relay-descriptors/microdescs/micro/': 'microdescriptor 1.0', + 'recent/relay-descriptors/server-descriptors/': 'server-descriptor 1.0', + 'recent/relay-descriptors/votes/': 'network-status-vote-3 1.0', + 'recent/torperf/': 'torperf 1.1', + 'recent/webstats/': (), +} + def _download(url, compression, timeout, retries): """ @@ -135,6 +169,27 @@ class File(object): self.path = path self.size = size self.last_modified = datetime.datetime.strptime(last_modified, '%Y-%m-%d %H:%M') + self._guessed_type = None + + def guess_descriptor_types(self): + """ + Descriptor @type this file is expected to have based on its path. If unable + to determine any this tuple is empty. + + :returns: **tuple** with the descriptor types this file is expected to have + """ + + if self._guessed_type is None: + guessed_type = () + + for path_prefix, types in COLLECTOR_DESC_TYPES.items(): + if self.path.startswith(path_prefix): + guessed_type = (types,) if isinstance(types, str) else types + break + + self._guessed_type = guessed_type + + return self._guessed_type class CollecTor(object): diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 3403ee50..609a4929 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -9,7 +9,7 @@ import unittest import stem.prereq from stem.descriptor import Compression -from stem.descriptor.collector import CollecTor +from stem.descriptor.collector import CollecTor, File from test.unit.descriptor import get_resource from test.unit.descriptor.data.collector_index import EXAMPLE_INDEX @@ -135,3 +135,16 @@ class TestCollector(unittest.TestCase): self.assertEqual(test_path, extrainfo_file.path) self.assertEqual(6459884, extrainfo_file.size) self.assertEqual(datetime.datetime(2016, 6, 23, 9, 54), extrainfo_file.last_modified) + + def test_guess_descriptor_types(self): + f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21') + self.assertEqual(('bridge-extra-info 1.3',), f.guess_descriptor_types()) + + f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') + self.assertEqual(('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'), f.guess_descriptor_types()) + + f = File('archive/webstats/webstats-2015-03.tar', 20480, '2018-03-19 16:07') + self.assertEqual((), f.guess_descriptor_types()) + + f = File('archive/no_such_file.tar', 20480, '2018-03-19 16:07') + self.assertEqual((), f.guess_descriptor_types())

1 0

[stem/master] Initial download function
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 4196e37f0f43d033b23f740c1e3e3d9612ba4655 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jul 29 19:49:37 2019 -0700 Initial download function Few rough edges, but a copy-paste of the initial pydoc demo now works. --- stem/descriptor/collector.py | 69 +++++++++++++++++++++++++++++++++++--- stem/descriptor/remote.py | 6 +--- test/integ/descriptor/collector.py | 4 +-- 3 files changed, 67 insertions(+), 12 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index f76fa225..a78d60c4 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -16,7 +16,7 @@ With this you can either download and read directly from CollecTor... import datetime import stem.descriptor.collector - yesterday = datetime.date.today() - datetime.timedelta(1) + yesterday = datetime.datetime.today() - datetime.timedelta(1) # provide yesterday's exits @@ -33,7 +33,7 @@ With this you can either download and read directly from CollecTor... import stem.descriptor import stem.descriptor.collector - yesterday = datetime.date.today() - datetime.timedelta(1) + yesterday = datetime.datetime.today() - datetime.timedelta(1) path = os.path.expanduser('~/descriptor_cache/server_desc_today') with open(path, 'wb') as cache_file: @@ -53,6 +53,7 @@ import datetime import json import os import re +import shutil import sys import tempfile import time @@ -70,6 +71,7 @@ except ImportError: COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old +SINGLETON_COLLECTOR = None YEAR_DATE = re.compile('-(\\d{4})-(\\d{2})\\.') SEC_DATE = re.compile('(\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2})') @@ -113,6 +115,52 @@ COLLECTOR_DESC_TYPES = { } +def get_instance(): + """ + Provides the singleton :class:`~stem.descriptor.collector.CollecTor` + used for this module's shorthand functions. + + :returns: singleton :class:`~stem.descriptor.collector.CollecTor` instance + """ + + global SINGLETON_COLLECTOR + + if SINGLETON_COLLECTOR is None: + SINGLETON_COLLECTOR = CollecTor() + + return SINGLETON_COLLECTOR + + +def get_server_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3): + """ + Provides server descriptors for the given time range, sorted oldest to + newest. + + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with + :param str cache_to: directory to cache archives into, if an archive is + available here it is not downloaded + :param int timeout: timeout for downloading each individual archive when the + connection becomes idle, no timeout applied if **None** + :param int retires: maximum attempts to impose on a per-archive basis + + :returns: **iterator** of + :class:`~stem.descriptor.server_descriptor.ServerDescriptor` for the given + time range + + :raises: + * **socket.timeout** if our request timed out + * **urllib2.URLError** for most request failures + + Note that the urllib2 module may fail with other exception types, in + which case we'll pass it along. + """ + + for f in get_instance().files('server-descriptor', start, end): + for desc in f.read(cache_to, timeout = timeout, retries = retries): + yield desc + + def _download(url, timeout, retries): """ Download from the given url. @@ -229,13 +277,24 @@ class File(object): if self._downloaded_to and os.path.exists(self._downloaded_to): directory = os.path.dirname(self._downloaded_to) else: - with tempfile.TemporaryDirectory() as tmp_directory: - return self.read(tmp_directory, descriptor_type, timeout, retries) + # TODO: The following can be replaced with simpler usage of + # tempfile.TemporaryDirectory when we drop python 2.x support. + + tmp_directory = tempfile.mkdtemp() + + for desc in self.read(tmp_directory, descriptor_type, timeout, retries): + yield desc + + shutil.rmtree(tmp_directory) + + return # TODO: the following will not work if the tar contains multiple types or a type we do not support path = self.download(directory, True, timeout, retries) - return parse_file(path, descriptor_type) + + for desc in parse_file(path, descriptor_type): + yield desc def download(self, directory, decompress = True, timeout = None, retries = 3): """ diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 251fc26f..af24f624 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -164,11 +164,7 @@ DIR_PORT_BLACKLIST = ('tor26', 'Serge') def get_instance(): """ Provides the singleton :class:`~stem.descriptor.remote.DescriptorDownloader` - used for the following functions... - - * :func:`stem.descriptor.remote.get_server_descriptors` - * :func:`stem.descriptor.remote.get_extrainfo_descriptors` - * :func:`stem.descriptor.remote.get_consensus` + used for this module's shorthand functions. .. versionadded:: 1.5.0 diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py index dbb09d5a..6a0ec5ac 100644 --- a/test/integ/descriptor/collector.py +++ b/test/integ/descriptor/collector.py @@ -35,8 +35,8 @@ class TestCollector(unittest.TestCase): if compression and not compression.available: self.skipTest('(%s unavailable)' % compression) - collector = CollecTor(compression = compression) - index = collector.index() + collector = CollecTor() + index = collector.index(compression = compression) self.assertEqual('https://collector.torproject.org', index['path']) self.assertEqual(['archive', 'contrib', 'recent'], [entry['path'] for entry in index['directories']])

1 0

[stem/master] Adjust CollecTor File class
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 4c744badc658d3d93d113972bfbf7cb463298ee4 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jul 29 17:35:32 2019 -0700 Adjust CollecTor File class Handful of very tiny adjustments. Dropping the unused tar attribute, making descriptor guessing a static function (like the others), fixing some minor edge cases, etc. --- stem/descriptor/collector.py | 51 ++++++------ test/unit/descriptor/collector.py | 161 ++++++++++++++++---------------------- 2 files changed, 92 insertions(+), 120 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index d1c90e0e..f76fa225 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -157,7 +157,6 @@ class File(object): :var str path: file path within collector :var stem.descriptor.Compression compression: file compression, **None** if this cannot be determined - :var bool tar: **True** if a tarball, **False** otherwise :var int size: size of the file :var datetime start: beginning of the time range descriptors are for, @@ -170,13 +169,12 @@ class File(object): def __init__(self, path, size, last_modified): self.path = path self.compression = File._guess_compression(path) - self.tar = path.endswith('.tar') or '.tar.' in path self.size = size self.start, self.end = File._guess_time_range(path) self.last_modified = datetime.datetime.strptime(last_modified, '%Y-%m-%d %H:%M') - self._guessed_type = None + self._guessed_type = File._guess_descriptor_types(path) self._downloaded_to = None # location we last downloaded to def read(self, directory = None, descriptor_type = None, timeout = None, retries = 3): @@ -220,21 +218,21 @@ class File(object): """ if descriptor_type is None: - descriptor_types = self._guess_descriptor_types() - - if not descriptor_types: + if not self._guessed_type: raise ValueError("Unable to determine this file's descriptor type") - elif len(descriptor_types) > 1: - raise ValueError("Unable to determine disambiguate file's descriptor type from %s" % ', '.join(descriptor_types)) + elif len(self._guessed_type) > 1: + raise ValueError("Unable to determine disambiguate file's descriptor type from %s" % ', '.join(self._guessed_type)) - descriptor_type = descriptor_types[0] + descriptor_type = self._guessed_type[0] if directory is None: if self._downloaded_to and os.path.exists(self._downloaded_to): directory = os.path.dirname(self._downloaded_to) else: with tempfile.TemporaryDirectory() as tmp_directory: - return self.read(tmp_directory, timeout, retries) + return self.read(tmp_directory, descriptor_type, timeout, retries) + + # TODO: the following will not work if the tar contains multiple types or a type we do not support path = self.download(directory, True, timeout, retries) return parse_file(path, descriptor_type) @@ -267,7 +265,7 @@ class File(object): filename = self.path.split('/')[-1] - if decompress: + if self.compression != Compression.PLAINTEXT and decompress: filename = filename.rsplit('.', 1)[0] path = os.path.join(directory, filename) @@ -277,36 +275,35 @@ class File(object): elif os.path.exists(path): return path # file already exists - with open(path, 'wb') as output_file: - response = _download(COLLECTOR_URL + self.path, timeout, retries) + response = _download(COLLECTOR_URL + self.path, timeout, retries) - if decompress: - response = self.compression.decompress(response) + if decompress: + response = self.compression.decompress(response) + with open(path, 'wb') as output_file: output_file.write(response) self._downloaded_to = path return path - def _guess_descriptor_types(self): + @staticmethod + def _guess_descriptor_types(path): """ Descriptor @type this file is expected to have based on its path. If unable to determine any this tuple is empty. - :returns: **tuple** with the descriptor types this file is expected to have - """ + Hopefully this will be replaced with an explicit value in the future: - if self._guessed_type is None: - guessed_type = () + https://trac.torproject.org/projects/tor/ticket/31204 - for path_prefix, types in COLLECTOR_DESC_TYPES.items(): - if self.path.startswith(path_prefix): - guessed_type = (types,) if isinstance(types, str) else types - break + :returns: **tuple** with the descriptor types this file is expected to have + """ - self._guessed_type = guessed_type + for path_prefix, types in COLLECTOR_DESC_TYPES.items(): + if path.startswith(path_prefix): + return (types,) if isinstance(types, str) else types - return self._guessed_type + return () @staticmethod def _guess_compression(path): @@ -437,7 +434,7 @@ class CollecTor(object): elif end and (f.end is None or f.end > end): continue - if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in f._guess_descriptor_types()]): + if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in f._guessed_type]): matches.append(f) return matches diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index ad0087dd..77c1c460 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -21,68 +21,105 @@ except ImportError: URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' -MINIMAL_INDEX = { - 'index_created': '2017-12-25 21:06', - 'build_revision': '56a303e', - 'path': 'https://collector.torproject.org' -} - -MINIMAL_INDEX_JSON = b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org"}' with open(get_resource('collector_index.json'), 'rb') as index_file: - EXAMPLE_INDEX_CONTENT = index_file.read() + EXAMPLE_INDEX_JSON = index_file.read() class TestCollector(unittest.TestCase): + # tests for the File class + + def test_file_guess_descriptor_types(self): + test_values = { + 'archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz': ('bridge-extra-info 1.3',), + 'archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz': ('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'), + 'archive/webstats/webstats-2015-03.tar': (), + 'archive/no_such_file.tar': (), + } + + for path, expected in test_values.items(): + self.assertEqual(expected, File._guess_descriptor_types(path)) + + def test_file_guess_compression(self): + test_values = { + 'archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz': Compression.LZMA, + 'archive/webstats/webstats-2015-03.tar': Compression.PLAINTEXT, + 'recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos': Compression.PLAINTEXT, + } + + for path, expected in test_values.items(): + self.assertEqual(expected, File._guess_compression(path)) + + def test_file_guess_time_range(self): + test_values = { + 'archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz': + (datetime.datetime(2014, 1, 1), datetime.datetime(2014, 2, 1)), + 'recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos': + (datetime.datetime(2019, 7, 3, 2, 5, 0), datetime.datetime(2019, 7, 3, 3, 5, 0)), + 'archive/relay-descriptors/certs.tar.xz': + (None, None), + 'archive/relay-descriptors/microdescs/microdescs-2014-12.tar.xz': + (datetime.datetime(2014, 12, 1), datetime.datetime(2015, 1, 1)), + 'recent/relay-descriptors/extra-infos/2019-07-03-23-05-00-extra-infos': + (datetime.datetime(2019, 7, 3, 23, 5, 0), datetime.datetime(2019, 7, 4, 0, 5, 0)) + } + + for path, (expected_start, expected_end) in test_values.items(): + f = File(path, 7515396, '2014-02-07 03:59') + self.assertEqual(expected_start, f.start) + self.assertEqual(expected_end, f.end) + + # tests for the CollecTor class + @patch(URL_OPEN) - def test_download_plaintext(self, urlopen_mock): - urlopen_mock.return_value = io.BytesIO(MINIMAL_INDEX_JSON) + def test_index_plaintext(self, urlopen_mock): + urlopen_mock.return_value = io.BytesIO(EXAMPLE_INDEX_JSON) collector = CollecTor() - self.assertEqual(MINIMAL_INDEX, collector.index(Compression.PLAINTEXT)) + self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.PLAINTEXT)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json', timeout = None) @patch(URL_OPEN) - def test_download_gzip(self, urlopen_mock): + def test_index_gzip(self, urlopen_mock): if not Compression.GZIP.available: self.skipTest('(gzip compression unavailable)') return import zlib - urlopen_mock.return_value = io.BytesIO(zlib.compress(MINIMAL_INDEX_JSON)) + urlopen_mock.return_value = io.BytesIO(zlib.compress(EXAMPLE_INDEX_JSON)) collector = CollecTor() - self.assertEqual(MINIMAL_INDEX, collector.index(Compression.GZIP)) + self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.GZIP)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None) @patch(URL_OPEN) - def test_download_bz2(self, urlopen_mock): + def test_index_bz2(self, urlopen_mock): if not Compression.BZ2.available: self.skipTest('(bz2 compression unavailable)') return import bz2 - urlopen_mock.return_value = io.BytesIO(bz2.compress(MINIMAL_INDEX_JSON)) + urlopen_mock.return_value = io.BytesIO(bz2.compress(EXAMPLE_INDEX_JSON)) collector = CollecTor() - self.assertEqual(MINIMAL_INDEX, collector.index(Compression.BZ2)) + self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.BZ2)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None) @patch(URL_OPEN) - def test_download_lzma(self, urlopen_mock): + def test_index_lzma(self, urlopen_mock): if not Compression.LZMA.available: self.skipTest('(lzma compression unavailable)') return import lzma - urlopen_mock.return_value = io.BytesIO(lzma.compress(MINIMAL_INDEX_JSON)) + urlopen_mock.return_value = io.BytesIO(lzma.compress(EXAMPLE_INDEX_JSON)) collector = CollecTor() - self.assertEqual(MINIMAL_INDEX, collector.index(Compression.LZMA)) + self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.LZMA)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.xz', timeout = None) @patch(URL_OPEN) - def test_download_retries(self, urlopen_mock): + def test_index_retries(self, urlopen_mock): urlopen_mock.side_effect = IOError('boom') collector = CollecTor(retries = 0) @@ -95,11 +132,6 @@ class TestCollector(unittest.TestCase): self.assertRaisesRegexp(IOError, 'boom', collector.index) self.assertEqual(5, urlopen_mock.call_count) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(MINIMAL_INDEX_JSON))) - def test_index(self): - collector = CollecTor() - self.assertEqual(MINIMAL_INDEX, collector.index(Compression.PLAINTEXT)) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json'))) def test_index_malformed_json(self): collector = CollecTor() @@ -118,104 +150,47 @@ class TestCollector(unittest.TestCase): collector = CollecTor() self.assertRaisesRegexp(IOError, 'Failed to decompress as %s' % compression, collector.index, compression) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) - def test_real_index(self): - collector = CollecTor() - self.assertEqual(EXAMPLE_INDEX, collector.index(compression = Compression.PLAINTEXT)) - @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) - def test_contents(self): + def test_files(self): collector = CollecTor() files = collector.files() - self.assertEqual(85, len(files)) - test_path = 'archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz' - extrainfo_file = list(filter(lambda x: x.path == test_path, files))[0] - self.assertEqual(test_path, extrainfo_file.path) + extrainfo_file = list(filter(lambda x: x.path.endswith('extra-infos-2007-09.tar.xz'), files))[0] + self.assertEqual('archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz', extrainfo_file.path) self.assertEqual(Compression.LZMA, extrainfo_file.compression) - self.assertEqual(True, extrainfo_file.tar) self.assertEqual(6459884, extrainfo_file.size) self.assertEqual(datetime.datetime(2016, 6, 23, 9, 54), extrainfo_file.last_modified) - def test_file_compression_attributes(self): - f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') - self.assertEqual(Compression.LZMA, f.compression) - self.assertEqual(True, f.tar) - - f = File('archive/webstats/webstats-2015-03.tar', 20480, '2018-03-19 16:07') - self.assertEqual(Compression.PLAINTEXT, f.compression) - self.assertEqual(True, f.tar) - - f = File('recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos', 1162899, '2019-07-03 02:05') - self.assertEqual(Compression.PLAINTEXT, f.compression) - self.assertEqual(False, f.tar) - - def test_file_date_attributes(self): - f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') - self.assertEqual(datetime.datetime(2014, 1, 1), f.start) - self.assertEqual(datetime.datetime(2014, 2, 1), f.end) - - f = File('recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos', 1162899, '2019-07-03 02:05') - self.assertEqual(datetime.datetime(2019, 7, 3, 2, 5, 0), f.start) - self.assertEqual(datetime.datetime(2019, 7, 3, 3, 5, 0), f.end) - - f = File('archive/relay-descriptors/certs.tar.xz', 144696, '2019-07-03 03:29') - self.assertEqual(None, f.start) - self.assertEqual(None, f.end) - - # check date boundaries - - f = File('archive/relay-descriptors/microdescs/microdescs-2014-12.tar.xz', 7515396, '2014-02-07 03:59') - self.assertEqual(datetime.datetime(2015, 1, 1), f.end) - - f = File('recent/relay-descriptors/extra-infos/2019-07-03-23-05-00-extra-infos', 1162899, '2019-07-03 02:05') - self.assertEqual(datetime.datetime(2019, 7, 4, 0, 5, 0), f.end) - @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) - def test_file_query_by_type(self): + def test_files_by_descriptor_type(self): collector = CollecTor() - expected = [ + self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', - ] - - self.assertEqual(expected, list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor')))) + ], [f.path for f in collector.files(descriptor_type = 'server-descriptor')]) @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX)) - def test_file_query_by_date(self): + def test_file_by_date(self): collector = CollecTor() self.assertEqual([ 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', - ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1))))) + ], [f.path for f in collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1))]) self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', - ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', end = datetime.datetime(2007, 1, 1))))) + ], [f.path for f in collector.files(descriptor_type = 'server-descriptor', end = datetime.datetime(2007, 1, 1))]) self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', - ], list(map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1))))) - - def test_guess_descriptor_types(self): - f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21') - self.assertEqual(('bridge-extra-info 1.3',), f._guess_descriptor_types()) - - f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') - self.assertEqual(('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'), f._guess_descriptor_types()) - - f = File('archive/webstats/webstats-2015-03.tar', 20480, '2018-03-19 16:07') - self.assertEqual((), f._guess_descriptor_types()) - - f = File('archive/no_such_file.tar', 20480, '2018-03-19 16:07') - self.assertEqual((), f._guess_descriptor_types()) + ], [f.path for f in collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1))])

1 0

[stem/master] Filter files by descriptor type or time
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 756aba1eb6c73c79bc7c09b151c93c3c3a3c3c80 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jul 22 15:42:32 2019 -0700 Filter files by descriptor type or time Methods I'll be offering to users include filtering by time range or descriptor type so including these as arguments in our files() method. --- stem/descriptor/collector.py | 38 +++++++++++++++++++++++++++++--------- test/unit/descriptor/collector.py | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index c1458784..426e48ae 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -74,6 +74,9 @@ REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old YEAR_DATE = re.compile('-(\\d{4})-(\\d{2})\\.') SEC_DATE = re.compile('(\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2})') +# distant future date so we can sort files without a timestamp at the end + +FUTURE = datetime.datetime(9999, 1, 1) # mapping of path prefixes to their descriptor type (sampled 7/11/19) @@ -272,7 +275,7 @@ class CollecTor(object): self.timeout = timeout self._cached_index = None - self._cached_files = None # {path => file} mappings in the index + self._cached_files = None self._cached_index_at = 0 if compression == 'best': @@ -306,11 +309,15 @@ class CollecTor(object): return self._cached_index - def files(self): + def files(self, descriptor_type = None, start = None, end = None): """ - Provides files CollecTor presently has. + Provides files CollecTor presently has, sorted oldest to newest. + + :param str descriptor_type: descriptor type or prefix to retrieve + :param datetime.datetime start: time range to begin with + :param datetime.datetime end: time range to end with - :returns: **hash** mapping paths to :class:`~stem.descriptor.collector.File` + :returns: **list** of :class:`~stem.descriptor.collector.File` :raises: If unable to retrieve the index this provide... @@ -324,7 +331,20 @@ class CollecTor(object): if not self._cached_files or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: self._cached_files = CollecTor._files(self.index(), []) - return self._cached_files + matches = [] + + for entry in self._cached_files: + if start and (entry.start is None or entry.start < start): + continue + elif end and (entry.end is None or entry.end > end): + continue + + if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in entry.guess_descriptor_types()]): + matches.append(entry) + + matches.sort(key = lambda x: x.start if x.start else FUTURE) + + return matches @staticmethod def _files(val, path): @@ -334,19 +354,19 @@ class CollecTor(object): :param dict val: index hash :param list path: path we've transversed into - :returns: **dict** mapping paths to files + :returns: **list** of :class:`~stem.descriptor.collector.File` """ - files = {} + files = [] if isinstance(val, dict): for k, v in val.items(): if k == 'files': for attr in v: file_path = '/'.join(path + [attr.get('path')]) - files[file_path] = File(file_path, attr.get('size'), attr.get('last_modified')) + files.append(File(file_path, attr.get('size'), attr.get('last_modified'))) elif k == 'directories': for attr in v: - files.update(CollecTor._files(attr, path + [attr.get('path')])) + files.extend(CollecTor._files(attr, path + [attr.get('path')])) return files diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 297050ea..9d442deb 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -131,7 +131,7 @@ class TestCollector(unittest.TestCase): self.assertEqual(85, len(files)) test_path = 'archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz' - extrainfo_file = files[test_path] + extrainfo_file = filter(lambda x: x.path == test_path, files)[0] self.assertEqual(test_path, extrainfo_file.path) self.assertEqual(Compression.LZMA, extrainfo_file.compression) self.assertEqual(True, extrainfo_file.tar) @@ -172,6 +172,41 @@ class TestCollector(unittest.TestCase): f = File('recent/relay-descriptors/extra-infos/2019-07-03-23-05-00-extra-infos', 1162899, '2019-07-03 02:05') self.assertEqual(datetime.datetime(2019, 7, 4, 0, 5, 0), f.end) + @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + def test_file_query_by_type(self): + collector = CollecTor(compression = Compression.PLAINTEXT) + + expected = [ + 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', + 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', + 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', + 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', + 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', + 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', + ] + + self.assertEqual(expected, map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor'))) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + def test_file_query_by_date(self): + collector = CollecTor(compression = Compression.PLAINTEXT) + + self.assertEqual([ + 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', + 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', + 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', + ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1)))) + + self.assertEqual([ + 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', + 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', + 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', + ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', end = datetime.datetime(2007, 1, 1)))) + + self.assertEqual([ + 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', + ], map(lambda x: x.path, collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1)))) + def test_guess_descriptor_types(self): f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21') self.assertEqual(('bridge-extra-info 1.3',), f.guess_descriptor_types())

1 0

[stem/master] Default compression attribute to plaintext
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 0519cf57fcfe54de71956d7127bd8551ede2dfac Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 28 14:27:31 2019 -0700 Default compression attribute to plaintext Unrecognized file extensions should be treated as being uncompressed. For instance, if CollecTor includes a json file we want to download it as-is (rather than choke on an unexpected None value). --- stem/descriptor/collector.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 69425dd5..d1c90e0e 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -314,12 +314,11 @@ class File(object): Determine file comprssion from CollecTor's filename. """ - if '.' not in path or path.endswith('.tar'): - return Compression.PLAINTEXT - else: - for compression in (Compression.LZMA, Compression.BZ2, Compression.GZIP): - if path.endswith(compression.extension): - return compression + for compression in (Compression.LZMA, Compression.BZ2, Compression.GZIP): + if path.endswith(compression.extension): + return compression + + return Compression.PLAINTEXT @staticmethod def _guess_time_range(path):

1 0

[stem/master] Download helper utility
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 6d4cbd2180d11682d7d65b6926c562155907d049 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 1 19:58:07 2019 -0700 Download helper utility Stem only raises documented exceptions, but urllib makes this difficult in that it raises a wide variety of exceptions. Wrapping it within a DownloadFailed exception that retains its wrapped exception. --- stem/descriptor/collector.py | 76 +++------------------------- stem/util/connection.py | 112 ++++++++++++++++++++++++++++++++++++++++++ test/integ/util/connection.py | 33 +++++++++++-- test/unit/util/connection.py | 43 ++++++++++++++++ 4 files changed, 192 insertions(+), 72 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index a78d60c4..d9f159e1 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -54,20 +54,13 @@ import json import os import re import shutil -import sys import tempfile import time +import stem.util.connection import stem.util.str_tools from stem.descriptor import Compression, parse_file -from stem.util import log - -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old @@ -148,12 +141,7 @@ def get_server_descriptors(start = None, end = None, cache_to = None, timeout = :class:`~stem.descriptor.server_descriptor.ServerDescriptor` for the given time range - :raises: - * **socket.timeout** if our request timed out - * **urllib2.URLError** for most request failures - - Note that the urllib2 module may fail with other exception types, in - which case we'll pass it along. + :raises: :class:`~stem.util.connection.DownloadFailed` if the download fails """ for f in get_instance().files('server-descriptor', start, end): @@ -161,43 +149,6 @@ def get_server_descriptors(start = None, end = None, cache_to = None, timeout = yield desc -def _download(url, timeout, retries): - """ - Download from the given url. - - :param str url: uncompressed url to download from - :param int timeout: timeout when connection becomes idle, no timeout applied - if **None** - :param int retires: maximum attempts to impose - - :returns: content of the given url - - :raises: - * **socket.timeout** if our request timed out - * **urllib2.URLError** for most request failures - - Note that the urllib2 module may fail with other exception types, in - which case we'll pass it along. - """ - - start_time = time.time() - - try: - return urllib.urlopen(url, timeout = timeout).read() - except: - exc = sys.exc_info()[1] - - if timeout is not None: - timeout -= time.time() - start_time - - if retries > 0 and (timeout is None or timeout > 0): - log.debug("Failed to download from CollecTor at '%s' (%i retries remaining): %s" % (url, retries, exc)) - return _download(url, timeout, retries - 1) - else: - log.debug("Failed to download from CollecTor at '%s': %s" % (url, exc)) - raise - - class File(object): """ File within CollecTor. @@ -258,11 +209,7 @@ class File(object): :raises: * **ValueError** if unable to determine the descirptor type * **TypeError** if we cannot parse this descriptor type - * **socket.timeout** if our request timed out - * **urllib2.URLError** for most request failures - - Note that the urllib2 module may fail with other exception types, in - which case we'll pass it along. + * :class:`~stem.util.connection.DownloadFailed` if the download fails """ if descriptor_type is None: @@ -309,12 +256,7 @@ class File(object): :returns: **str** with the path we downloaded to - :raises: - * **socket.timeout** if our request timed out - * **urllib2.URLError** for most request failures - - Note that the urllib2 module may fail with other exception types, in - which case we'll pass it along. + :raises: :class:`~stem.util.connection.DownloadFailed` if the download fails """ # TODO: If checksums get added to the index we should replace @@ -334,7 +276,7 @@ class File(object): elif os.path.exists(path): return path # file already exists - response = _download(COLLECTOR_URL + self.path, timeout, retries) + response = stem.util.connection.download(COLLECTOR_URL + self.path, timeout, retries) if decompress: response = self.compression.decompress(response) @@ -441,8 +383,7 @@ class CollecTor(object): * **ValueError** if json is malformed * **IOError** if unable to decompress - * **socket.timeout** if our request timed out - * **urllib2.URLError** for most request failures + * :class:`~stem.util.connection.DownloadFailed` if the download fails """ if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: @@ -456,7 +397,7 @@ class CollecTor(object): extension = compression.extension if compression != Compression.PLAINTEXT else '' url = COLLECTOR_URL + 'index/index.json' + extension - response = compression.decompress(_download(url, self.timeout, self.retries)) + response = compression.decompress(stem.util.connection.download(url, self.timeout, self.retries)) self._cached_index = json.loads(stem.util.str_tools._to_unicode(response)) self._cached_index_at = time.time() @@ -478,8 +419,7 @@ class CollecTor(object): * **ValueError** if json is malformed * **IOError** if unable to decompress - * **socket.timeout** if our request timed out - * **urllib2.URLError** for most request failures + * :class:`~stem.util.connection.DownloadFailed` if the download fails """ if not self._cached_files or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: diff --git a/stem/util/connection.py b/stem/util/connection.py index c23d74e7..7be7fe09 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -8,6 +8,10 @@ Connection and networking based utility functions. :: + DownloadFailed - Inability to download a resource. + +- DownloadTimeout - Download timeout reached. + + download - download from a given url get_connections - quieries the connections belonging to a given process system_resolvers - provides connection resolution methods that are likely to be available port_usage - brief description of the common usage for a port @@ -58,6 +62,10 @@ import collections import os import platform import re +import socket +import sys +import time +import traceback import stem.util import stem.util.proc @@ -65,6 +73,12 @@ import stem.util.system from stem.util import conf, enum, log, str_tools +try: + # account for urllib's change between python 2.x and 3.x + import urllib.request as urllib +except ImportError: + import urllib2 as urllib + # Connection resolution is risky to log about since it's highly likely to # contain sensitive information. That said, it's also difficult to get right in # a platform independent fashion. To opt into the logging requried to @@ -162,6 +176,104 @@ class Connection(collections.namedtuple('Connection', ['local_address', 'local_p """ +class DownloadFailed(IOError): + """ + Inability to download a resource. Python's urllib module raises + a wide variety of undocumented exceptions (urllib2.URLError, + socket.timeout, and others). + + This wraps lower level failures in a common exception type that + retains their exception and `stacktrace + <https://docs.python.org/3/library/traceback.html>`_. + + .. versionadded:: 1.8.0 + + :var str url: url we failed to download from + :var Exception error: original urllib exception + :var traceback stacktrace: original stacktrace + :var str stacktrace_str: string representation of the stacktrace + """ + + def __init__(self, url, error, stacktrace, message = None): + if message is None: + # The string representation of exceptions can reside in several places. + # urllib.URLError use a 'reason' attribute that in turn may referrence + # low level structures such as socket.gaierror. Whereas most exceptions + # use a 'message' attribute. + + reason = str(error) + + all_str_repr = ( + getattr(getattr(error, 'reason', None), 'strerror', None), + getattr(error, 'reason', None), + getattr(error, 'message', None), + ) + + for str_repr in all_str_repr: + if str_repr and isinstance(str_repr, str): + reason = str_repr + break + + message = 'Failed to download from %s (%s): %s' % (url, type(error).__name__, reason) + + super(DownloadFailed, self).__init__(message) + + self.url = url + self.error = error + self.stacktrace = stacktrace + self.stacktrace_str = ''.join(traceback.format_tb(stacktrace)) + + +class DownloadTimeout(DownloadFailed): + """ + Timeout reached while downloading this resource. + + .. versionadded:: 1.8.0 + """ + + def __init__(self, url, error, stacktrace, timeout): + super(DownloadTimeout, self).__init__('Failed to download from %s: %0.1f second timeout reached' % (url, timeout)) + + +def download(url, timeout = None, retries = None): + """ + Download from the given url. + + .. versionadded:: 1.8.0 + + :param str url: uncompressed url to download from + :param int timeout: timeout when connection becomes idle, no timeout applied + if **None** + :param int retires: maximum attempts to impose + + :returns: **bytes** content of the given url + + :raises: :class:`~stem.util.connection.DownloadFailed` if the download fails + """ + + if retries is None: + retries = 0 + + start_time = time.time() + + try: + return urllib.urlopen(url, timeout = timeout).read() + except socket.timeout as exc: + raise DownloadTimeout(url, exc, sys.exc_info()[2], timeout) + except: + exc, stacktrace = sys.exc_info()[1:3] + + if timeout is not None: + timeout -= time.time() - start_time + + if retries > 0 and (timeout is None or timeout > 0): + log.debug('Failed to download from %s (%i retries remaining): %s' % (url, retries, exc)) + return download(url, timeout, retries - 1) + else: + log.debug('Failed to download from %s: %s' % (url, exc)) + raise DownloadFailed(url, exc, stacktrace) + + def get_connections(resolver = None, process_pid = None, process_name = None): """ Retrieves a list of the current connections for a given process. This diff --git a/test/integ/util/connection.py b/test/integ/util/connection.py index 12ce8ac4..4617fe56 100644 --- a/test/integ/util/connection.py +++ b/test/integ/util/connection.py @@ -5,11 +5,18 @@ that we're running. import unittest +import stem.util.connection import stem.util.system import test.require import test.runner -from stem.util.connection import RESOLVER_COMMAND, Resolver, get_connections, system_resolvers +from stem.util.connection import Resolver + +try: + # account for urllib's change between python 2.x and 3.x + import urllib.request as urllib +except ImportError: + import urllib2 as urllib class TestConnection(unittest.TestCase): @@ -20,22 +27,40 @@ class TestConnection(unittest.TestCase): if test.runner.Torrc.PORT not in runner.get_options(): self.skipTest('(no control port)') return - elif resolver not in system_resolvers(): + elif resolver not in stem.util.connection.system_resolvers(): self.skipTest('(resolver unavailable on this platform)') return with runner.get_tor_socket(): - connections = get_connections(resolver, process_pid = runner.get_pid()) + connections = stem.util.connection.get_connections(resolver, process_pid = runner.get_pid()) for conn in connections: if conn.local_address == '127.0.0.1' and conn.local_port == test.runner.CONTROL_PORT: return - resolver_command = RESOLVER_COMMAND[resolver].format(pid = runner.get_pid()) + resolver_command = stem.util.connection.RESOLVER_COMMAND[resolver].format(pid = runner.get_pid()) resolver_output = stem.util.system.call(resolver_command) self.fail('Unable to find our controller connection with %s (%s). Connections found were...\n\n%s\n\nCommand output was...\n\n%s' % (resolver, resolver_command, '\n'.join(map(str, connections)), resolver_output)) + @test.require.only_run_once + @test.require.online + def test_download(self): + response = stem.util.connection.download('https://collector.torproject.org/index/index.json') + self.assertTrue(b'"path":"https://collector.torproject.org"' in response) + + @test.require.only_run_once + @test.require.online + def test_download_failure(self): + try: + stem.util.connection.download('https://no.such.testing.url') + self.fail('expected a stem.util.connection.DownloadFailed to be raised') + except stem.util.connection.DownloadFailed as exc: + self.assertEqual('Failed to download from https://no.such.testing.url (URLError): Name or service not known', str(exc)) + self.assertEqual('https://no.such.testing.url', exc.url) + self.assertEqual('Name or service not known', exc.error.reason.strerror) + self.assertEqual(urllib.URLError, type(exc.error)) + def test_connections_by_proc(self): self.check_resolver(Resolver.PROC) diff --git a/test/unit/util/connection.py b/test/unit/util/connection.py index a2162029..57718446 100644 --- a/test/unit/util/connection.py +++ b/test/unit/util/connection.py @@ -2,6 +2,7 @@ Unit tests for the stem.util.connection functions. """ +import io import platform import unittest @@ -10,11 +11,20 @@ import stem.util.connection from stem.util.connection import Resolver, Connection try: + # account for urllib's change between python 2.x and 3.x + import urllib.request as urllib +except ImportError: + import urllib2 as urllib + +try: # added in python 3.3 from unittest.mock import Mock, patch except ImportError: from mock import Mock, patch +URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' +URL = 'https://example.unit.test.url' + NETSTAT_OUTPUT = """\ Active Internet connections (w/o servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name @@ -166,6 +176,39 @@ _tor tor 15843 20* internet stream tcp 0x0 192.168.1.100:36174 --> class TestConnection(unittest.TestCase): + @patch(URL_OPEN) + def test_download(self, urlopen_mock): + urlopen_mock.return_value = io.BytesIO(b'hello') + + self.assertEqual(b'hello', stem.util.connection.download(URL)) + urlopen_mock.assert_called_with(URL, timeout = None) + + @patch(URL_OPEN) + def test_download_failure(self, urlopen_mock): + urlopen_mock.side_effect = urllib.URLError('boom') + + try: + stem.util.connection.download(URL) + self.fail('expected a stem.util.connection.DownloadFailed to be raised') + except stem.util.connection.DownloadFailed as exc: + self.assertEqual('Failed to download from https://example.unit.test.url (URLError): boom', str(exc)) + self.assertEqual(URL, exc.url) + self.assertEqual('boom', exc.error.reason) + self.assertEqual(urllib.URLError, type(exc.error)) + self.assertTrue('return urllib.urlopen(url, timeout = timeout).read()' in exc.stacktrace_str) + + @patch(URL_OPEN) + def test_download_retries(self, urlopen_mock): + urlopen_mock.side_effect = urllib.URLError('boom') + + self.assertRaisesRegexp(IOError, 'boom', stem.util.connection.download, URL) + self.assertEqual(1, urlopen_mock.call_count) + + urlopen_mock.reset_mock() + + self.assertRaisesRegexp(IOError, 'boom', stem.util.connection.download, URL, retries = 4) + self.assertEqual(5, urlopen_mock.call_count) + @patch('os.access') @patch('stem.util.system.is_available') @patch('stem.util.proc.is_available')

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits August 2019