[tor-commits] [stem/master] Drop url helper function
atagar at torproject.org
atagar at torproject.org
Sat Aug 17 20:44:27 UTC 2019
commit a60efda6275fbee76349896ba6f1ed63415c4b94
Author: Damian Johnson <atagar at torproject.org>
Date: Fri Jul 5 14:12:52 2019 -0700
Drop url helper function
Originally I was unsure if I'd want this. Turns out nope - it helped with url
testability, but we can exercise the same with better mocks.
---
stem/descriptor/collector.py | 31 +++---------
test/unit/descriptor/collector.py | 102 +++++++++++++++++++++++++++++++-------
2 files changed, 89 insertions(+), 44 deletions(-)
diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index 746e4b1d..d723e97a 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -70,34 +70,11 @@ COLLECTOR_URL = 'https://collector.torproject.org/'
REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old
-def url(resource, compression = Compression.PLAINTEXT):
- """
- Provides CollecTor url for the given resource.
-
- :param str resource: resource type of the url
- :param descriptor.Compression compression: compression type to
- download from
-
- :returns: **str** with the CollecTor url
- """
-
- # TODO: Unsure how to most elegantly map resources to urls. No doubt
- # this'll change as we add more types.
-
- if resource == 'index':
- path = ('index', 'index.json')
- else:
- raise ValueError("'%s' isn't a recognized resource type" % resource)
-
- extension = compression.extension if compression not in (None, Compression.PLAINTEXT) else ''
- return COLLECTOR_URL + '/'.join(path) + extension
-
-
def _download(url, compression, timeout, retries):
"""
Download from the given url.
- :param str url: url to download from
+ :param str url: uncompressed url to download from
:param descriptor.Compression compression: decompression type
:param int timeout: timeout when connection becomes idle, no timeout applied
if **None**
@@ -115,6 +92,10 @@ def _download(url, compression, timeout, retries):
"""
start_time = time.time()
+ extension = compression.extension if compression not in (None, Compression.PLAINTEXT) else ''
+
+ if not url.endswith(extension):
+ url += extension
try:
response = urllib.urlopen(url, timeout = timeout).read()
@@ -185,7 +166,7 @@ class CollecTor(object):
"""
if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE:
- response = _download(url('index', self.compression), self.compression, self.timeout, self.retries)
+ response = _download(COLLECTOR_URL + 'index/index.json', self.compression, self.timeout, self.retries)
self._cached_index = json.loads(response)
self._cached_index_at = time.time()
diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py
index c46fb60c..cfb57c3d 100644
--- a/test/unit/descriptor/collector.py
+++ b/test/unit/descriptor/collector.py
@@ -8,7 +8,7 @@ import unittest
import stem.prereq
from stem.descriptor import Compression
-from stem.descriptor.collector import CollecTor, url
+from stem.descriptor.collector import CollecTor
try:
# added in python 3.3
@@ -18,34 +18,95 @@ except ImportError:
URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen'
+MINIMAL_INDEX = {
+ 'index_created': '2017-12-25 21:06',
+ 'build_revision': '56a303e',
+ 'path': 'https://collector.torproject.org'
+}
+
+MINIMAL_INDEX_JSON = b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org"}'
+
class TestCollector(unittest.TestCase):
- def test_url(self):
- self.assertEqual('https://collector.torproject.org/index/index.json', url('index'))
- self.assertEqual('https://collector.torproject.org/index/index.json', url('index', compression = None))
- self.assertEqual('https://collector.torproject.org/index/index.json', url('index', compression = Compression.PLAINTEXT))
- self.assertEqual('https://collector.torproject.org/index/index.json.gz', url('index', compression = Compression.GZIP))
- self.assertEqual('https://collector.torproject.org/index/index.json.bz2', url('index', compression = Compression.BZ2))
- self.assertEqual('https://collector.torproject.org/index/index.json.xz', url('index', compression = Compression.LZMA))
+ @patch(URL_OPEN)
+ def test_download_plaintext(self, urlopen_mock):
+ urlopen_mock.return_value = io.BytesIO(MINIMAL_INDEX_JSON)
+
+ collector = CollecTor(compression = Compression.PLAINTEXT)
+ self.assertEqual(MINIMAL_INDEX, collector.index())
+ urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json', timeout = None)
@patch(URL_OPEN)
- def test_retries(self, urlopen_mock):
- collector = CollecTor(retries = 4)
+ def test_download_gzip(self, urlopen_mock):
+ if not Compression.GZIP.available:
+ self.skipTest('(gzip compression unavailable)')
+ return
+
+ import zlib
+ urlopen_mock.return_value = io.BytesIO(zlib.compress(MINIMAL_INDEX_JSON))
+
+ collector = CollecTor(compression = Compression.GZIP)
+ self.assertEqual(MINIMAL_INDEX, collector.index())
+ urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None)
+
+ @patch(URL_OPEN)
+ def test_download_bz2(self, urlopen_mock):
+ if not Compression.BZ2.available:
+ self.skipTest('(bz2 compression unavailable)')
+ return
+
+ import bz2
+ urlopen_mock.return_value = io.BytesIO(bz2.compress(MINIMAL_INDEX_JSON))
+
+ collector = CollecTor(compression = Compression.BZ2)
+ self.assertEqual(MINIMAL_INDEX, collector.index())
+ urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None)
+
+ @patch(URL_OPEN)
+ def test_download_lzma(self, urlopen_mock):
+ if not Compression.LZMA.available:
+ self.skipTest('(lzma compression unavailable)')
+ return
+
+ import lzma
+ urlopen_mock.return_value = io.BytesIO(lzma.compress(MINIMAL_INDEX_JSON))
+
+ collector = CollecTor(compression = Compression.LZMA)
+ self.assertEqual(MINIMAL_INDEX, collector.index())
+ urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.lzma', timeout = None)
+
+ @patch(URL_OPEN)
+ def test_download_zstd(self, urlopen_mock):
+ if not Compression.ZSTD.available:
+ self.skipTest('(zstd compression unavailable)')
+ return
+
+ import zstd
+ compressor = zstd.ZstdCompressor()
+ urlopen_mock.return_value = io.BytesIO(compressor.compress(MINIMAL_INDEX_JSON))
+
+ collector = CollecTor(compression = Compression.ZSTD)
+ self.assertEqual(MINIMAL_INDEX, collector.index())
+ urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.zst', timeout = None)
+
+ @patch(URL_OPEN)
+ def test_download_retries(self, urlopen_mock):
urlopen_mock.side_effect = IOError('boom')
+ collector = CollecTor(retries = 0)
+ self.assertRaisesRegexp(IOError, 'boom', collector.index)
+ self.assertEqual(1, urlopen_mock.call_count)
+
+ urlopen_mock.reset_mock()
+
+ collector = CollecTor(retries = 4)
self.assertRaisesRegexp(IOError, 'boom', collector.index)
self.assertEqual(5, urlopen_mock.call_count)
- @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org"}')))
+ @patch(URL_OPEN, Mock(return_value = io.BytesIO(MINIMAL_INDEX_JSON)))
def test_index(self):
- expected = {
- 'index_created': '2017-12-25 21:06',
- 'build_revision': '56a303e',
- 'path': 'https://collector.torproject.org'
- }
-
collector = CollecTor(compression = Compression.PLAINTEXT)
- self.assertEqual(expected, collector.index())
+ self.assertEqual(MINIMAL_INDEX, collector.index())
@patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json')))
def test_index_malformed_json(self):
@@ -57,7 +118,10 @@ class TestCollector(unittest.TestCase):
self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index)
def test_index_malformed_compression(self):
- for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA):
+ for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA, Compression.ZSTD):
+ if not compression.available:
+ next
+
with patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not compressed'))):
collector = CollecTor(compression = compression)
self.assertRaisesRegexp(IOError, 'Unable to decompress %s response' % compression, collector.index)
More information about the tor-commits
mailing list