commit a60efda6275fbee76349896ba6f1ed63415c4b94 Author: Damian Johnson atagar@torproject.org Date: Fri Jul 5 14:12:52 2019 -0700
Drop url helper function
Originally I was unsure if I'd want this. Turns out nope - it helped with url testability, but we can exercise the same with better mocks. --- stem/descriptor/collector.py | 31 +++--------- test/unit/descriptor/collector.py | 102 +++++++++++++++++++++++++++++++------- 2 files changed, 89 insertions(+), 44 deletions(-)
diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 746e4b1d..d723e97a 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -70,34 +70,11 @@ COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old
-def url(resource, compression = Compression.PLAINTEXT): - """ - Provides CollecTor url for the given resource. - - :param str resource: resource type of the url - :param descriptor.Compression compression: compression type to - download from - - :returns: **str** with the CollecTor url - """ - - # TODO: Unsure how to most elegantly map resources to urls. No doubt - # this'll change as we add more types. - - if resource == 'index': - path = ('index', 'index.json') - else: - raise ValueError("'%s' isn't a recognized resource type" % resource) - - extension = compression.extension if compression not in (None, Compression.PLAINTEXT) else '' - return COLLECTOR_URL + '/'.join(path) + extension - - def _download(url, compression, timeout, retries): """ Download from the given url.
- :param str url: url to download from + :param str url: uncompressed url to download from :param descriptor.Compression compression: decompression type :param int timeout: timeout when connection becomes idle, no timeout applied if **None** @@ -115,6 +92,10 @@ def _download(url, compression, timeout, retries): """
start_time = time.time() + extension = compression.extension if compression not in (None, Compression.PLAINTEXT) else '' + + if not url.endswith(extension): + url += extension
try: response = urllib.urlopen(url, timeout = timeout).read() @@ -185,7 +166,7 @@ class CollecTor(object): """
if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: - response = _download(url('index', self.compression), self.compression, self.timeout, self.retries) + response = _download(COLLECTOR_URL + 'index/index.json', self.compression, self.timeout, self.retries) self._cached_index = json.loads(response) self._cached_index_at = time.time()
diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index c46fb60c..cfb57c3d 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -8,7 +8,7 @@ import unittest import stem.prereq
from stem.descriptor import Compression -from stem.descriptor.collector import CollecTor, url +from stem.descriptor.collector import CollecTor
try: # added in python 3.3 @@ -18,34 +18,95 @@ except ImportError:
URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen'
+MINIMAL_INDEX = { + 'index_created': '2017-12-25 21:06', + 'build_revision': '56a303e', + 'path': 'https://collector.torproject.org' +} + +MINIMAL_INDEX_JSON = b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org%22%7D' +
class TestCollector(unittest.TestCase): - def test_url(self): - self.assertEqual('https://collector.torproject.org/index/index.json', url('index')) - self.assertEqual('https://collector.torproject.org/index/index.json', url('index', compression = None)) - self.assertEqual('https://collector.torproject.org/index/index.json', url('index', compression = Compression.PLAINTEXT)) - self.assertEqual('https://collector.torproject.org/index/index.json.gz', url('index', compression = Compression.GZIP)) - self.assertEqual('https://collector.torproject.org/index/index.json.bz2', url('index', compression = Compression.BZ2)) - self.assertEqual('https://collector.torproject.org/index/index.json.xz', url('index', compression = Compression.LZMA)) + @patch(URL_OPEN) + def test_download_plaintext(self, urlopen_mock): + urlopen_mock.return_value = io.BytesIO(MINIMAL_INDEX_JSON) + + collector = CollecTor(compression = Compression.PLAINTEXT) + self.assertEqual(MINIMAL_INDEX, collector.index()) + urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json', timeout = None)
@patch(URL_OPEN) - def test_retries(self, urlopen_mock): - collector = CollecTor(retries = 4) + def test_download_gzip(self, urlopen_mock): + if not Compression.GZIP.available: + self.skipTest('(gzip compression unavailable)') + return + + import zlib + urlopen_mock.return_value = io.BytesIO(zlib.compress(MINIMAL_INDEX_JSON)) + + collector = CollecTor(compression = Compression.GZIP) + self.assertEqual(MINIMAL_INDEX, collector.index()) + urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None) + + @patch(URL_OPEN) + def test_download_bz2(self, urlopen_mock): + if not Compression.BZ2.available: + self.skipTest('(bz2 compression unavailable)') + return + + import bz2 + urlopen_mock.return_value = io.BytesIO(bz2.compress(MINIMAL_INDEX_JSON)) + + collector = CollecTor(compression = Compression.BZ2) + self.assertEqual(MINIMAL_INDEX, collector.index()) + urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None) + + @patch(URL_OPEN) + def test_download_lzma(self, urlopen_mock): + if not Compression.LZMA.available: + self.skipTest('(lzma compression unavailable)') + return + + import lzma + urlopen_mock.return_value = io.BytesIO(lzma.compress(MINIMAL_INDEX_JSON)) + + collector = CollecTor(compression = Compression.LZMA) + self.assertEqual(MINIMAL_INDEX, collector.index()) + urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.lzma', timeout = None) + + @patch(URL_OPEN) + def test_download_zstd(self, urlopen_mock): + if not Compression.ZSTD.available: + self.skipTest('(zstd compression unavailable)') + return + + import zstd + compressor = zstd.ZstdCompressor() + urlopen_mock.return_value = io.BytesIO(compressor.compress(MINIMAL_INDEX_JSON)) + + collector = CollecTor(compression = Compression.ZSTD) + self.assertEqual(MINIMAL_INDEX, collector.index()) + urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.zst', timeout = None) + + @patch(URL_OPEN) + def test_download_retries(self, urlopen_mock): urlopen_mock.side_effect = IOError('boom')
+ collector = CollecTor(retries = 0) + self.assertRaisesRegexp(IOError, 'boom', collector.index) + self.assertEqual(1, urlopen_mock.call_count) + + urlopen_mock.reset_mock() + + collector = CollecTor(retries = 4) self.assertRaisesRegexp(IOError, 'boom', collector.index) self.assertEqual(5, urlopen_mock.call_count)
- @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org%22%7D'))) + @patch(URL_OPEN, Mock(return_value = io.BytesIO(MINIMAL_INDEX_JSON))) def test_index(self): - expected = { - 'index_created': '2017-12-25 21:06', - 'build_revision': '56a303e', - 'path': 'https://collector.torproject.org' - } - collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(expected, collector.index()) + self.assertEqual(MINIMAL_INDEX, collector.index())
@patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json'))) def test_index_malformed_json(self): @@ -57,7 +118,10 @@ class TestCollector(unittest.TestCase): self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index)
def test_index_malformed_compression(self): - for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA): + for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA, Compression.ZSTD): + if not compression.available: + next + with patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not compressed'))): collector = CollecTor(compression = compression) self.assertRaisesRegexp(IOError, 'Unable to decompress %s response' % compression, collector.index)