August 2019 - tor-commits - lists.torproject.org

[stem/master] Test index request failures
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit d0ec0cdc476151a4e698867a47f1c3ffc4662fdb Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 25 17:32:22 2017 -0800 Test index request failures --- stem/descriptor/collector.py | 14 +++++++++++--- test/unit/descriptor/collector.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 5d65fd65..2360d827 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -100,7 +100,11 @@ class Compression(object): :returns: **bytes** with the decompressed content - :raises: **ImportError** if this method if decompression is unavalable + :raises: + If unable to decompress this provide... + + * **IOError** if content isn't compressed with this + * **ImportError** if this method if decompression is unavalable """ if not self.available: @@ -190,7 +194,8 @@ class CollecTor(object): :raises: If unable to retrieve the index this provide... - * **ValueError** if the index is malformed + * **ValueError** if json is malformed + * **IOError** if unable to decompress * **socket.timeout** if our request timed out * **urllib2.URLError** for most request failures """ @@ -201,7 +206,10 @@ class CollecTor(object): response = urllib.urlopen(url('index', self.compression), timeout = self.timeout).read() if self.compression: - response = self.compression.decompress(response) + try: + response = self.compression.decompress(response) + except Exception as exc: + raise IOError('Unable to decompress response as %s: %s' % (self.compression, exc)) self._cached_index = json.loads(stem.util.str_tools._to_unicode(response)) self._cached_index_at = time.time() diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 01f8f6ee..a0f8cb2e 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -36,3 +36,18 @@ class TestCollector(unittest.TestCase): collector = CollecTor(compression = None) self.assertEqual(expected, collector.index()) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json'))) + def test_index_malformed_json(self): + collector = CollecTor(compression = None) + + if stem.prereq.is_python_3(): + self.assertRaisesRegexp(ValueError, 'Expecting value: line 1 column 1', collector.index) + else: + self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index) + + def test_index_malformed_compression(self): + for compression in (GZIP, BZ2, LZMA): + with patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not compressed'))): + collector = CollecTor(compression = compression) + self.assertRaisesRegexp(IOError, 'Unable to decompress response as %s' % compression, collector.index)

1 0

[stem/master] Bump version collector module is getting added
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 234b4916e9ef4cc640fbaba0b49b80558f76aacc Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jun 15 16:02:29 2019 -0700 Bump version collector module is getting added Ooph! Just realized I began a collector branch way back in December 2017. Rebased atop our present master. Stem 1.7 was already released, 1.8 is now the next upcoming version. --- stem/descriptor/collector.py | 8 ++++---- test/integ/descriptor/collector.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 2360d827..21a774e9 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -1,4 +1,4 @@ -# Copyright 2017, Damian Johnson and The Tor Project +# Copyright 2017-2019, Damian Johnson and The Tor Project # See LICENSE for licensing information """ @@ -23,7 +23,7 @@ With this you can either download and read directly from CollecTor... for desc in collector.get_server_descriptors(start = yesterday): if desc.exit_policy.is_exiting_allowed(): - print ' %s (%s)' % (desc.nickname, desc.fingerprint) + print(' %s (%s)' % (desc.nickname, desc.fingerprint)) ... or download the descriptors to disk and read them later. @@ -45,9 +45,9 @@ With this you can either download and read directly from CollecTor... for desc in reader: if desc.exit_policy.is_exiting_allowed(): - print ' %s (%s)' % (desc.nickname, desc.fingerprint) + print(' %s (%s)' % (desc.nickname, desc.fingerprint)) -.. versionadded:: 1.7.0 +.. versionadded:: 1.8.0 """ import io diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py index 3233f2e5..1af329a5 100644 --- a/test/integ/descriptor/collector.py +++ b/test/integ/descriptor/collector.py @@ -38,4 +38,4 @@ class TestCollector(unittest.TestCase): index = collector.index() self.assertEqual('https://collector.torproject.org', index['path']) - self.assertEqual(['archive', 'recent'], [entry['path'] for entry in index['directories']]) + self.assertEqual(['archive', 'contrib', 'recent'], [entry['path'] for entry in index['directories']])

1 0

[stem/master] Use Compression class in stem.descriptor.remote
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 0c84d3cf4dcedd3629e6bb51c607240b1b2257b9 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 19 15:55:39 2019 -0700 Use Compression class in stem.descriptor.remote Now that our new Compression class is in place, deduplicating the remote module's decompression code with it. --- stem/descriptor/__init__.py | 9 ++++-- stem/descriptor/remote.py | 67 ++++++++++++++++++++++++------------------ test/unit/descriptor/remote.py | 11 +++---- 3 files changed, 52 insertions(+), 35 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 0b3fda91..2c69aef0 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -225,7 +225,12 @@ class _Compression(object): """ if not self.available: - raise ImportError("'%s' decompression module is unavailable" % self._module_name) + if self.name == 'zstd': + raise ImportError('Decompressing zstd data requires https://pypi.org/project/zstandard/') + elif self.name == 'lzma': + raise ImportError('Decompressing lzma data requires https://docs.python.org/3/library/lzma.html') + else: + raise ImportError("'%s' decompression module is unavailable" % self._module_name) return self._decompression_func(self._module, content) @@ -247,7 +252,7 @@ Compression = stem.util.enum.Enum( ('GZIP', _Compression('gzip', 'zlib', 'gzip', '.gz', lambda module, content: module.decompress(content, module.MAX_WBITS | 32))), ('BZ2', _Compression('bzip2', 'bz2', 'bzip2', '.bz2', lambda module, content: module.decompress(content))), ('LZMA', _Compression('lzma', 'lzma', 'x-tor-lzma', '.xz', lambda module, content: module.decompress(content))), - ('ZSTD', _Compression('zstd', 'zstd', 'zstd', '.zst', _zstd_decompress)), + ('ZSTD', _Compression('zstd', 'zstd', 'x-zstd', '.zst', _zstd_decompress)), ) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 15f46070..f715d743 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -100,7 +100,6 @@ import random import sys import threading import time -import zlib import stem import stem.client @@ -119,6 +118,8 @@ try: except ImportError: import urllib2 as urllib +# TODO: remove in stem 2.x, replaced with stem.descriptor.Compression + Compression = stem.util.enum.Enum( ('PLAINTEXT', 'identity'), ('GZIP', 'gzip'), # can also be 'deflate' @@ -126,6 +127,13 @@ Compression = stem.util.enum.Enum( ('LZMA', 'x-tor-lzma'), ) +COMPRESSION_MIGRATION = { + 'identity': stem.descriptor.Compression.PLAINTEXT, + 'gzip': stem.descriptor.Compression.GZIP, + 'x-zstd': stem.descriptor.Compression.ZSTD, + 'x-tor-lzma': stem.descriptor.Compression.LZMA, +} + # Tor has a limited number of descriptors we can fetch explicitly by their # fingerprint or hashes due to a limit on the url length by squid proxies. @@ -364,6 +372,11 @@ class Query(object): .. versionchanged:: 1.8.0 Defaulting to gzip compression rather than plaintext downloads. + .. versionchanged:: 1.8.0 + Using :class:`~stem.descriptor.__init__.Compression` for our compression + argument, usage of strings or this module's Compression enum is deprecated + and will be removed in stem 2.x. + :var str resource: resource being fetched, such as '/tor/server/all' :var str descriptor_type: type of descriptors being fetched (for options see :func:`~stem.descriptor.__init__.parse_file`), this is guessed from the @@ -371,7 +384,7 @@ class Query(object): :var list endpoints: :class:`~stem.DirPort` or :class:`~stem.ORPort` of the authority or mirror we're querying, this uses authorities if undefined - :var list compression: list of :data:`stem.descriptor.remote.Compression` + :var list compression: list of :data:`stem.descriptor.Compression` we're willing to accept, when none are mutually supported downloads fall back to Compression.PLAINTEXT :var int retries: number of times to attempt the request if downloading it @@ -429,6 +442,19 @@ class Query(object): if not compression: compression = [Compression.PLAINTEXT] + # TODO: Normalize from our old compression enum to + # stem.descriptor.Compression. This will get removed in Stem 2.x. + + new_compression = [] + + for legacy_compression in compression: + if isinstance(legacy_compression, stem.descriptor._Compression): + new_compression.append(legacy_compression) + elif legacy_compression in COMPRESSION_MIGRATION: + new_compression.append(COMPRESSION_MIGRATION[legacy_compression]) + else: + raise ValueError("'%s' (%s) is not a recognized type of compression" % (legacy_compression, type(legacy_compression).__name__)) + if descriptor_type: self.descriptor_type = descriptor_type else: @@ -446,7 +472,7 @@ class Query(object): raise ValueError("Endpoints must be an stem.ORPort, stem.DirPort, or two value tuple. '%s' is a %s." % (endpoint, type(endpoint).__name__)) self.resource = resource - self.compression = compression + self.compression = new_compression self.retries = retries self.fall_back_to_authority = fall_back_to_authority @@ -1009,7 +1035,7 @@ def _download_from_orport(endpoint, compression, resource): with relay.create_circuit() as circ: request = '\r\n'.join(( 'GET %s HTTP/1.0' % resource, - 'Accept-Encoding: %s' % ', '.join(compression), + 'Accept-Encoding: %s' % ', '.join(map(lambda c: c.encoding, compression)), 'User-Agent: %s' % stem.USER_AGENT, )) + '\r\n\r\n' @@ -1051,7 +1077,7 @@ def _download_from_dirport(url, compression, timeout): urllib.Request( url, headers = { - 'Accept-Encoding': ', '.join(compression), + 'Accept-Encoding': ', '.join(map(lambda c: c.encoding, compression)), 'User-Agent': stem.USER_AGENT, } ), @@ -1080,29 +1106,14 @@ def _decompress(data, encoding): * **ImportError** if missing the decompression module """ - if encoding == Compression.PLAINTEXT: - return data - elif encoding in (Compression.GZIP, 'deflate'): - return zlib.decompress(data, zlib.MAX_WBITS | 32) - elif encoding == Compression.ZSTD: - if not stem.prereq.is_zstd_available(): - raise ImportError('Decompressing zstd data requires https://pypi.org/project/zstandard/') - - import zstd - output_buffer = io.BytesIO() - - with zstd.ZstdDecompressor().write_to(output_buffer) as decompressor: - decompressor.write(data) - - return output_buffer.getvalue() - elif encoding == Compression.LZMA: - if not stem.prereq.is_lzma_available(): - raise ImportError('Decompressing lzma data requires https://docs.python.org/3/library/lzma.html') - - import lzma - return lzma.decompress(data) - else: - raise ValueError("'%s' isn't a recognized type of encoding" % encoding) + if encoding == 'deflate': + return stem.descriptor.Compression.GZIP.decompress(data) + + for compression in stem.descriptor.Compression: + if encoding == compression.encoding: + return compression.decompress(data) + + raise ValueError("'%s' isn't a recognized type of encoding" % encoding) def _guess_descriptor_type(resource): diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 8c5e835b..6dbaf43e 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -8,6 +8,7 @@ import time import unittest import stem +import stem.descriptor import stem.descriptor.remote import stem.prereq import stem.util.str_tools @@ -181,26 +182,26 @@ class TestDescriptorDownloader(unittest.TestCase): def test_gzip_url_override(self): query = stem.descriptor.remote.Query(TEST_RESOURCE + '.z', compression = Compression.PLAINTEXT, start = False) - self.assertEqual([Compression.GZIP], query.compression) + self.assertEqual([stem.descriptor.Compression.GZIP], query.compression) self.assertEqual(TEST_RESOURCE, query.resource) def test_zstd_support_check(self): with patch('stem.prereq.is_zstd_available', Mock(return_value = True)): query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.ZSTD, start = False) - self.assertEqual([Compression.ZSTD], query.compression) + self.assertEqual([stem.descriptor.Compression.ZSTD], query.compression) with patch('stem.prereq.is_zstd_available', Mock(return_value = False)): query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.ZSTD, start = False) - self.assertEqual([Compression.PLAINTEXT], query.compression) + self.assertEqual([stem.descriptor.Compression.PLAINTEXT], query.compression) def test_lzma_support_check(self): with patch('stem.prereq.is_lzma_available', Mock(return_value = True)): query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.LZMA, start = False) - self.assertEqual([Compression.LZMA], query.compression) + self.assertEqual([stem.descriptor.Compression.LZMA], query.compression) with patch('stem.prereq.is_lzma_available', Mock(return_value = False)): query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.LZMA, start = False) - self.assertEqual([Compression.PLAINTEXT], query.compression) + self.assertEqual([stem.descriptor.Compression.PLAINTEXT], query.compression) @patch(URL_OPEN, _dirport_mock(read_resource('compressed_identity'), encoding = 'identity')) def test_compression_plaintext(self):

1 0

[stem/master] Have decompress provide bytes
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit cedbfc735df8282ca309c7a54693ba842f671dfb Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 25 15:03:59 2017 -0800 Have decompress provide bytes Converting to unicode here doesn't buy us anything. --- stem/descriptor/collector.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index c6d70544..5d65fd65 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -98,7 +98,7 @@ class Compression(object): :param bytes content: content to be decompressed - :returns: **unicode** with the decompressed content + :returns: **bytes** with the decompressed content :raises: **ImportError** if this method if decompression is unavalable """ @@ -108,19 +108,17 @@ class Compression(object): if self._module_name == 'gzip': if stem.prereq.is_python_3(): - decompressed = self._module.decompress(content) + return self._module.decompress(content) else: # prior to python 3.2 gzip only had GzipFile - decompressed = self._module.GzipFile(fileobj = io.BytesIO(content)).read() + return self._module.GzipFile(fileobj = io.BytesIO(content)).read() elif self._module_name == 'bz2': - decompressed = self._module.decompress(content) + return self._module.decompress(content) elif self._module_name == 'lzma': - decompressed = self._module.decompress(content) + return self._module.decompress(content) else: raise ImportError('BUG: No implementation for %s decompression' % self) - return stem.util.str_tools._to_unicode(decompressed) - def __str__(self): return self._module_name @@ -200,14 +198,12 @@ class CollecTor(object): if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: # TODO: add retry support - response_bytes = urllib.urlopen(url('index', self.compression), timeout = self.timeout).read() + response = urllib.urlopen(url('index', self.compression), timeout = self.timeout).read() if self.compression: - response = self.compression.decompress(response_bytes) - else: - response = stem.util.str_tools._to_unicode(response_bytes) + response = self.compression.decompress(response) - self._cached_index = json.loads(response) + self._cached_index = json.loads(stem.util.str_tools._to_unicode(response)) self._cached_index_at = time.time() return self._cached_index

1 0

[stem/master] Compression class
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 9e9458501fa4b2f8819f93ff853c658b9e63366c Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jun 18 18:37:05 2019 -0700 Compression class Both our collector and remote modules need to decompress descriptors, so adding a little helper class to assist with availability checks and decompression. --- stem/descriptor/__init__.py | 79 +++++++++++++++++++++++++++++++ stem/descriptor/collector.py | 77 +++--------------------------- test/integ/descriptor/collector.py | 9 ++-- test/settings.cfg | 3 +- test/unit/descriptor/collector.py | 11 +++-- test/unit/descriptor/compression.py | 39 +++++++++++++++ test/unit/descriptor/data/compressed_bz2 | Bin 0 -> 1691 bytes 7 files changed, 138 insertions(+), 80 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 4d13ec60..0b3fda91 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -11,6 +11,8 @@ Package for parsing and processing descriptor data. parse_file - Parses the descriptors in a file. create_signing_key - Cretes a signing key that can be used for creating descriptors. + Compression - method of descriptor decompression + Descriptor - Common parent for all descriptor file types. | |- content - creates the text of a new descriptor | |- create - creates a new descriptor @@ -172,6 +174,83 @@ DocumentHandler = stem.util.enum.UppercaseEnum( ) +class _Compression(object): + """ + Compression method supported by CollecTor. + + :var bool available: **True** if this method of decryption is available, + **False** otherwise + :var str encoding: `http 'Accept-Encoding' parameter <https://en.wikipedia.org/wiki/HTTP_compression#Content-Encoding_tokens>`_ + :var str extension: file extension of this compression + + .. versionadded:: 1.8.0 + """ + + def __init__(self, name, module, encoding, extension, decompression_func): + if module is None: + self._module = None + self.available = True + else: + # Compression modules are optional. Usually gzip and bz2 are available, + # but they might be missing if compiling python yourself. As for lzma it + # was added in python 3.3. + + try: + self._module = __import__(module) + self.available = True + except ImportError: + self._module = None + self.available = False + + self.extension = extension + self.encoding = encoding + + self._name = name + self._module_name = module + self._decompression_func = decompression_func + + def decompress(self, content): + """ + Decompresses the given content via this method. + + :param bytes content: content to be decompressed + + :returns: **bytes** with the decompressed content + + :raises: + If unable to decompress this provide... + + * **IOError** if content isn't compressed with this + * **ImportError** if this method if decompression is unavalable + """ + + if not self.available: + raise ImportError("'%s' decompression module is unavailable" % self._module_name) + + return self._decompression_func(self._module, content) + + def __str__(self): + return self._name + + +def _zstd_decompress(module, content): + output_buffer = io.BytesIO() + + with module.ZstdDecompressor().write_to(output_buffer) as decompressor: + decompressor.write(content) + + return output_buffer.getvalue() + + +Compression = stem.util.enum.Enum( + ('PLAINTEXT', _Compression('plaintext', None, 'identity', '.txt', lambda module, content: content)), + ('GZIP', _Compression('gzip', 'zlib', 'gzip', '.gz', lambda module, content: module.decompress(content, module.MAX_WBITS | 32))), + ('BZ2', _Compression('bzip2', 'bz2', 'bzip2', '.bz2', lambda module, content: module.decompress(content))), + ('LZMA', _Compression('lzma', 'lzma', 'x-tor-lzma', '.xz', lambda module, content: module.decompress(content))), + ('ZSTD', _Compression('zstd', 'zstd', 'zstd', '.zst', _zstd_decompress)), +) + + class TypeAnnotation(collections.namedtuple('TypeAnnotation', ['name', 'major_version', 'minor_version'])): """ `Tor metrics type annotation diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 21a774e9..b3f99241 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -50,10 +50,11 @@ With this you can either download and read directly from CollecTor... .. versionadded:: 1.8.0 """ -import io import json import time +from stem.descriptor import Compression + try: # account for urllib's change between python 2.x and 3.x import urllib.request as urllib @@ -68,82 +69,18 @@ COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old -class Compression(object): - """ - Compression method supported by CollecTor. - - :var bool available: **True** if this method of decryption is available, - **False** otherwise - :var str extension: file extension of this compression - """ - - def __init__(self, module, extension): - # Compression modules are optional. Usually gzip and bz2 are available, but - # they might be missing if compiling python yourself. As for lzma it was - # added in python 3.3. - - try: - self._module = __import__(module) - self.available = True - except ImportError: - self._module = None - self.available = False - - self.extension = extension - self._module_name = module - - def decompress(self, content): - """ - Decompresses the given content via this method. - - :param bytes content: content to be decompressed - - :returns: **bytes** with the decompressed content - - :raises: - If unable to decompress this provide... - - * **IOError** if content isn't compressed with this - * **ImportError** if this method if decompression is unavalable - """ - - if not self.available: - raise ImportError("'%s' decompression module is unavailable" % self) - - if self._module_name == 'gzip': - if stem.prereq.is_python_3(): - return self._module.decompress(content) - else: - # prior to python 3.2 gzip only had GzipFile - return self._module.GzipFile(fileobj = io.BytesIO(content)).read() - elif self._module_name == 'bz2': - return self._module.decompress(content) - elif self._module_name == 'lzma': - return self._module.decompress(content) - else: - raise ImportError('BUG: No implementation for %s decompression' % self) - - def __str__(self): - return self._module_name - - -GZIP = Compression('gzip', '.gz') -BZ2 = Compression('bz2', '.bz2') -LZMA = Compression('lzma', '.xz') - - def url(resource, compression = None): """ Provides CollecTor url for the given resource. :param str resource: resource type of the url - :param descriptor.collector.Compression compression: compression type to + :param descriptor.Compression compression: compression type to download from :returns: **str** with the CollecTor url """ - # TODO: Not yet sure how to most elegantly map resources to urls. No doubt + # TODO: Unsure how to most elegantly map resources to urls. No doubt # this'll change as we add more types. if resource == 'index': @@ -152,7 +89,7 @@ def url(resource, compression = None): raise ValueError("'%s' isn't a recognized resource type" % resource) suffix = compression.extension if compression else '' - return ''.join((COLLECTOR_URL, '/'.join(path), suffix)) + return COLLECTOR_URL + '/'.join(path) + suffix class CollecTor(object): @@ -161,7 +98,7 @@ class CollecTor(object): provided in `an index <https://collector.torproject.org/index/index.json>`_ that's fetched as required. - :var descriptor.collector.Compression compression: compression type to + :var descriptor.Compression compression: compression type to download from, if undefiled we'll use the best decompression available :var int retries: number of times to attempt the request if downloading it fails @@ -172,7 +109,7 @@ class CollecTor(object): if compression == 'best': self.compression = None - for option in (LZMA, BZ2, GZIP): + for option in (Compression.LZMA, Compression.BZ2, Compression.GZIP): if option.available: self.compression = option break diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py index 1af329a5..dbb09d5a 100644 --- a/test/integ/descriptor/collector.py +++ b/test/integ/descriptor/collector.py @@ -6,7 +6,8 @@ import unittest import test.require -from stem.descriptor.collector import GZIP, BZ2, LZMA, CollecTor +from stem.descriptor import Compression +from stem.descriptor.collector import CollecTor class TestCollector(unittest.TestCase): @@ -18,17 +19,17 @@ class TestCollector(unittest.TestCase): @test.require.only_run_once @test.require.online def test_index_gzip(self): - self._test_index(GZIP) + self._test_index(Compression.GZIP) @test.require.only_run_once @test.require.online def test_index_bz2(self): - self._test_index(BZ2) + self._test_index(Compression.BZ2) @test.require.only_run_once @test.require.online def test_index_lzma(self): - self._test_index(LZMA) + self._test_index(Compression.LZMA) def _test_index(self, compression): if compression and not compression.available: diff --git a/test/settings.cfg b/test/settings.cfg index 6f71a329..1bdb1a0a 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -239,10 +239,11 @@ test.unit_tests |test.unit.util.tor_tools.TestTorTools |test.unit.util.__init__.TestBaseUtil |test.unit.installation.TestInstallation -|test.unit.descriptor.collector.TestCollector |test.unit.descriptor.descriptor.TestDescriptor +|test.unit.descriptor.compression.TestCompression |test.unit.descriptor.export.TestExport |test.unit.descriptor.reader.TestDescriptorReader +|test.unit.descriptor.collector.TestCollector |test.unit.descriptor.remote.TestDescriptorDownloader |test.unit.descriptor.server_descriptor.TestServerDescriptor |test.unit.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index a0f8cb2e..b2b464ce 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -7,7 +7,8 @@ import unittest import stem.prereq -from stem.descriptor.collector import GZIP, BZ2, LZMA, CollecTor, url +from stem.descriptor import Compression +from stem.descriptor.collector import CollecTor, url try: # added in python 3.3 @@ -22,9 +23,9 @@ class TestCollector(unittest.TestCase): def test_url(self): self.assertEqual('https://collector.torproject.org/index/index.json', url('index')) self.assertEqual('https://collector.torproject.org/index/index.json', url('index', compression = None)) - self.assertEqual('https://collector.torproject.org/index/index.json.gz', url('index', compression = GZIP)) - self.assertEqual('https://collector.torproject.org/index/index.json.bz2', url('index', compression = BZ2)) - self.assertEqual('https://collector.torproject.org/index/index.json.xz', url('index', compression = LZMA)) + self.assertEqual('https://collector.torproject.org/index/index.json.gz', url('index', compression = Compression.GZIP)) + self.assertEqual('https://collector.torproject.org/index/index.json.bz2', url('index', compression = Compression.BZ2)) + self.assertEqual('https://collector.torproject.org/index/index.json.xz', url('index', compression = Compression.LZMA)) @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org"}'))) def test_index(self): @@ -47,7 +48,7 @@ class TestCollector(unittest.TestCase): self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index) def test_index_malformed_compression(self): - for compression in (GZIP, BZ2, LZMA): + for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA): with patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not compressed'))): collector = CollecTor(compression = compression) self.assertRaisesRegexp(IOError, 'Unable to decompress response as %s' % compression, collector.index) diff --git a/test/unit/descriptor/compression.py b/test/unit/descriptor/compression.py new file mode 100644 index 00000000..3945bc9c --- /dev/null +++ b/test/unit/descriptor/compression.py @@ -0,0 +1,39 @@ +""" +Unit tests for stem.descriptor.Compression. +""" + +import unittest + +from stem.descriptor import Compression + +from test.unit.descriptor import get_resource + + +class TestCompression(unittest.TestCase): + def test_decompress_plaintext(self): + self._check_file(Compression.PLAINTEXT, 'compressed_identity') + + def test_decompress_gzip(self): + self._check_file(Compression.GZIP, 'compressed_gzip') + + def test_decompress_bz2(self): + self._check_file(Compression.BZ2, 'compressed_bz2') + + def test_decompress_lzma(self): + self._check_file(Compression.LZMA, 'compressed_lzma') + + def test_decompress_zstd(self): + self._check_file(Compression.ZSTD, 'compressed_zstd') + + def _check_file(self, compression, filename): + """ + Decompress one of our 'compressed_*' server descriptors. + """ + + if not compression.available: + self.skipTest('(%s unavailable)' % compression) + return + + with open(get_resource(filename), 'rb') as compressed_file: + content = compression.decompress(compressed_file.read()) + self.assertTrue(content.startswith(b'router moria1 128.31.0.34 9101 0 9131')) diff --git a/test/unit/descriptor/data/compressed_bz2 b/test/unit/descriptor/data/compressed_bz2 new file mode 100644 index 00000000..4d645a71 Binary files /dev/null and b/test/unit/descriptor/data/compressed_bz2 differ

1 0

[stem/master] Test failures presented invalid '--test' arguments
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 79cf4d383ef4e60fb386a4c148ee263d169e6d56 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jun 15 16:38:45 2019 -0700 Test failures presented invalid '--test' arguments Our integ tests are rightfully failing right now, but the message presented at the end is incorrect... You can re-run just these tests with: ./run_tests.py --all --target ONLINE --test test.integ.manual ./run_tests.py --all --target ONLINE --test test.integ.descriptor.collector The trouble is that to reduce verbosity I dropped the need for module prefixes (so you can run '--test descriptor.collector' instead), but... 1. The above message didn't account for that. 2. We were bugged in that we no longer worked with full module names, so running the command we advised didn't work. Both accepting full module names and now presenting short test module names. --- run_tests.py | 15 ++------------- test/arguments.py | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/run_tests.py b/run_tests.py index 77db73b5..a3a90d04 100755 --- a/run_tests.py +++ b/run_tests.py @@ -146,18 +146,7 @@ def _get_tests(modules, module_prefixes): if not module_prefixes: yield import_name else: - # Example import_name: test.unit.util.conf.TestConf - # - # Our '--test' argument doesn't include the prefix, so excluding it from - # the names we look for. - - if import_name.startswith('test.unit.'): - cropped_name = import_name[10:] - elif import_name.startswith('test.integ.'): - cropped_name = import_name[11:] - else: - cropped_name = import_name - + cropped_name = test.arguments.crop_module_name(import_name) cropped_name = cropped_name.rsplit('.', 1)[0] # exclude the class name for prefix in module_prefixes: @@ -370,7 +359,7 @@ def main(): println('\nYou can re-run just these tests with:\n', ERROR, STDERR) for module in error_modules: - println(' %s --test %s' % (' '.join(sys.argv), module), ERROR, STDERR) + println(' %s --test %s' % (' '.join(sys.argv), test.arguments.crop_module_name(module)), ERROR, STDERR) else: if skipped_tests > 0: println('%i TESTS WERE SKIPPED' % skipped_tests, STATUS) diff --git a/test/arguments.py b/test/arguments.py index 0fa492d8..a871b4e5 100644 --- a/test/arguments.py +++ b/test/arguments.py @@ -104,7 +104,7 @@ def parse(argv): args['attribute_targets'] = attribute_targets elif opt == '--test': - args['specific_test'].append(arg) + args['specific_test'].append(crop_module_name(arg)) elif opt in ('-l', '--log'): arg = arg.upper() @@ -149,3 +149,22 @@ def get_help(): help_msg += '\n' return help_msg + + +def crop_module_name(name): + """ + Test modules have a 'test.unit.' or 'test.integ.' prefix which can + be omitted from our '--test' argument. Cropping this so we can do + normalized comparisons. + + :param str name: module name to crop + + :returns: **str** with the cropped module name + """ + + if name.startswith('test.unit.'): + return name[10:] + elif name.startswith('test.integ.'): + return name[11:] + else: + return name

1 0

[translation/support-portal] Update translations for support-portal
by translation＠torproject.org 17 Aug '19

17 Aug '19

commit 1292f38687a8937557c56c7c4ef9e03cbcc306a9 Author: Translation commit bot <translation(a)torproject.org> Date: Sat Aug 17 13:15:45 2019 +0000 Update translations for support-portal --- contents+fa.po | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contents+fa.po b/contents+fa.po index b93cd6894..e205f299f 100644 --- a/contents+fa.po +++ b/contents+fa.po @@ -6,6 +6,7 @@ # Emma Peel, 2019 # Vox, 2019 # Reza Ghasemi, 2019 +# MYZJ, 2019 # msgid "" msgstr "" @@ -13,7 +14,7 @@ msgstr "" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2019-08-07 17:41+CET\n" "PO-Revision-Date: 2018-10-02 22:41+0000\n" -"Last-Translator: Reza Ghasemi, 2019\n" +"Last-Translator: MYZJ, 2019\n" "Language-Team: Persian (https://www.transifex.com/otf/teams/1519/fa/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -5351,7 +5352,7 @@ msgstr "لوگوی تور " #: templates/navbar.html:6 msgid "DONATE NOW" -msgstr "" +msgstr "اکنون اهداء کنید" #: templates/navbar.html:41 msgid "Download Tor Browser"

1 0

[translation/tbmanual-contentspot] Update translations for tbmanual-contentspot
by translation＠torproject.org 17 Aug '19

17 Aug '19

commit 9d9f28aa890cf25b6532b20d80300ed024c482bc Author: Translation commit bot <translation(a)torproject.org> Date: Sat Aug 17 12:59:39 2019 +0000 Update translations for tbmanual-contentspot --- contents+fa.po | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/contents+fa.po b/contents+fa.po index 0b462f336..2c9f8ec0d 100644 --- a/contents+fa.po +++ b/contents+fa.po @@ -1498,6 +1498,7 @@ msgid "" "By default, Tor Browser protects your security by encrypting your browsing " "data." msgstr "" +"به صورت پیش فرض، مرورگر تور با رمزنگاری داده ها، از امنیت شما محافظت می کند." #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body) @@ -1524,7 +1525,7 @@ msgstr "" #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body) msgid "##### ACCESSING THE SECURITY SETTINGS" -msgstr "" +msgstr "##### دسترسی به تنظیمات امنیتی" #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body) @@ -1546,6 +1547,8 @@ msgid "" "<img class=\"col-lg-6\" src=\"../../static/images/security-settings-" "anim.gif\">" msgstr "" +"<img class=\"col-lg-6\" src=\"../../static/images/security-settings-" +"anim.gif\">" #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body) @@ -1587,7 +1590,7 @@ msgstr "" #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body) msgid "This may cause some sites to lose functionality." -msgstr "" +msgstr "این کار ممکن است موجب شود تا عملکرد برخی از سایت ها دچار اختلال گردد." #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body) @@ -1607,7 +1610,7 @@ msgstr "" #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body) msgid "These changes affect images, media, and scripts." -msgstr "" +msgstr "این تغییرات بر روی تصاویر، رسانه ها و اسکریپت ها اثر می گذارد." #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body) @@ -1647,7 +1650,7 @@ msgstr "" #: https//tb-manual.torproject.org/updating/ #: (content/updating/contents+en.lrtopic.body) msgid "##### UPDATING TOR BROWSER AUTOMATICALLY" -msgstr "" +msgstr "##### به روزرسانی خودکار مرورگر تور" #: https//tb-manual.torproject.org/updating/ #: (content/updating/contents+en.lrtopic.body) @@ -1723,7 +1726,7 @@ msgstr "" #: https//tb-manual.torproject.org/plugins/ #: (content/plugins/contents+en.lrtopic.title) msgid "PLUGINS, ADD-ONS AND JAVASCRIPT" -msgstr "" +msgstr "پلاگین ها، افزونه ها و جاوا اسکریپت" #: https//tb-manual.torproject.org/plugins/ #: (content/plugins/contents+en.lrtopic.description) @@ -1733,7 +1736,7 @@ msgstr "مرورگر تور چگونه افزودنی, پلاگین ها و Java #: https//tb-manual.torproject.org/plugins/ #: (content/plugins/contents+en.lrtopic.body) msgid "##### FLASH PLAYER" -msgstr "" +msgstr "##### فِلَش پلیر" #: https//tb-manual.torproject.org/plugins/ #: (content/plugins/contents+en.lrtopic.body) @@ -1900,7 +1903,7 @@ msgstr "" #: https//tb-manual.torproject.org/troubleshooting/ #: (content/troubleshooting/contents+en.lrtopic.body) msgid "##### IS YOUR CONNECTION CENSORED?" -msgstr "" +msgstr "##### آیا ارتباط اینترنتی شما سانسور شده است؟" #: https//tb-manual.torproject.org/troubleshooting/ #: (content/troubleshooting/contents+en.lrtopic.body) @@ -2016,6 +2019,9 @@ msgid "" "* BitTorrent in specific is <mark><a href=\"https://blog.torproject.org" "/bittorrent-over-tor-isnt-good-idea\">not anonymous over Tor</a></mark>." msgstr "" +"بیت تورنت، به طور خاص <mark><a href=\"https://blog.torproject.org" +"/bittorrent-over-tor-isnt-good-idea\">نمی تواند بر روی تور، ناشناس " +"باشد</a></mark>." #: https//tb-manual.torproject.org/uninstalling/ #: (content/uninstalling/contents+en.lrtopic.description) @@ -2257,7 +2263,7 @@ msgstr "کمک مالی" #: templates/navbar.html:6 msgid "DONATE NOW" -msgstr "" +msgstr "اکنون اهداء کنید" #: templates/navbar.html:41 msgid "Download Tor Browser" @@ -2273,15 +2279,15 @@ msgstr "موضوعات" #: templates/macros/topic.html:19 msgid "Contributors to this page:" -msgstr "" +msgstr "شرکت کنندگان در این صفحه" #: templates/macros/topic.html:21 msgid "Edit this page" -msgstr "" +msgstr "ویرایش این صفحه" #: templates/macros/topic.html:22 msgid "Suggest Feedback" -msgstr "" +msgstr "پیشنهاد بازخورد" #: templates/macros/topic.html:23 msgid "Permalink"

1 0

[translation/tbmanual-contentspot] Update translations for tbmanual-contentspot
by translation＠torproject.org 17 Aug '19

17 Aug '19

commit 4e2f5c4a45732f1bb511acc09d2b1a2fba73c534 Author: Translation commit bot <translation(a)torproject.org> Date: Sat Aug 17 11:59:47 2019 +0000 Update translations for tbmanual-contentspot --- contents+fa.po | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/contents+fa.po b/contents+fa.po index ea3e1f160..0b462f336 100644 --- a/contents+fa.po +++ b/contents+fa.po @@ -486,6 +486,11 @@ msgid "" "ask your network administrator for guidance. If your connection does not use" " a proxy, click “Continue”." msgstr "" +"صفحه بعد، از شما درباره استفاده از پروکسی بر روی اتصال اینترنتی تان می پرسد." +" در بیشتر موارد، این مورد نیازی نیست. اگر از پروکسی استفاده می کنید، معمولا " +"مانند همان تنظیماتی که برای سایر مرورگرهای سیستمتان بکار رفته است، تنظیم می " +"شود. در صورت امکان، از مدیر شبکه خودتان در مورد تنظیمات آن راهنمایی بگیرید. " +"اگر اتصال اینترنت شما از پروکسی استفاده نمی کند، بر روی \"ادامه\" کلیک کنید." #: https//tb-manual.torproject.org/running-tor-browser/ #: (content/running-tor-browser/contents+en.lrtopic.body) @@ -508,6 +513,8 @@ msgid "" "Most Pluggable Transports, such as obfs3 and obfs4, rely on the use of " "“bridge” relays." msgstr "" +"بیشتر Pluggable Transport ها مانند obfs3 و obfs4، با استفاده از ایستگاه های " +"پلِ قابل اعتماد، کار می کنند." #: https//tb-manual.torproject.org/bridges/ #: (content/bridges/contents+en.lrtopic.body) @@ -519,6 +526,13 @@ msgid "" "combination with pluggable transports helps to disguise the fact that you " "are using Tor." msgstr "" +"بیشتر <a href=\"/transports\">Pluggable Transport ها</a> مانند obfs3 و " +"obfs4، با استفاده از ایستگاه های پلِ قابل اعتماد، کار می کنند. مانند ایستگاه" +" های معمولی تور، پل ها نیز توسط افراد داوطلب اجرا می شوند؛ اما برخلاف " +"ایستگاه های معمولی، ایستگاه های پل، به صورت عمومی منتشر نمی شوند. بنابراین " +"دشمنان شبکه تور نمی توانند آن ها را به آسانی شناسایی کنند. استفاده از پل ها " +"به همراه Pluggable Transport ها موجب می شود تا کسی متوجه استفاده شما از تور " +"نشود." #: https//tb-manual.torproject.org/bridges/ #: (content/bridges/contents+en.lrtopic.body) @@ -527,6 +541,9 @@ msgid "" "techniques that do not rely on bridges. You do not need to obtain bridge " "addresses in order to use these transports." msgstr "" +"سایر Pluggable Transport ها مانند meek، از فناوری های دیگری برای دور زدن " +"سانسور استفاده می کنند که وابسته به ایستگاه های پل نیستند، بنابراین شما برای" +" استفاده از این نوع روش ها، نیازی به داشتن پل ندارید." #: https//tb-manual.torproject.org/bridges/ #: (content/bridges/contents+en.lrtopic.body) @@ -578,6 +595,10 @@ msgid "" " of the URL bar, then select 'Tor Network Settings...' to access these " "options." msgstr "" +"اگر برای نخستین بار است که از مرورگر تور استفاده می کنید، بر روی " +"\"پیکربندی\" کلیک کنید تا پنجره تنظیمات شبکه تور برای شما باز شود. در غیر " +"این صورت، بر روی دکمه Tor - آیکون پیاز - در کنار نوار آدرس کلیک کنید. سپس " +"گزینه 'تنظیمات شبکه تور...' را برای دسترسی به گزینه ها انتخاب کنید." #: https//tb-manual.torproject.org/bridges/ #: (content/bridges/contents+en.lrtopic.body) @@ -613,7 +634,7 @@ msgstr "" #: https//tb-manual.torproject.org/transports/ #: (content/transports/contents+en.lrtopic.title) msgid "PLUGGABLE TRANSPORTS" -msgstr "" +msgstr "PLUGGABLE TRANSPORT ها" #: https//tb-manual.torproject.org/transports/ #: (content/transports/contents+en.lrtopic.description) @@ -804,7 +825,7 @@ msgstr "" #: https//tb-manual.torproject.org/circumvention/ #: (content/circumvention/contents+en.lrtopic.title) msgid "CIRCUMVENTION" -msgstr "" +msgstr "دور زدن سانسور" #: https//tb-manual.torproject.org/circumvention/ #: (content/circumvention/contents+en.lrtopic.description) @@ -1064,7 +1085,7 @@ msgstr "" #: https//tb-manual.torproject.org/managing-identities/ #: (content/managing-identities/contents+en.lrtopic.body) msgid "###### NEW TOR CIRCUIT FOR THIS SITE" -msgstr "" +msgstr "###### چرخه جدید تور برای این سایت" #: https//tb-manual.torproject.org/managing-identities/ #: (content/managing-identities/contents+en.lrtopic.body) @@ -1251,7 +1272,7 @@ msgstr "" #: https//tb-manual.torproject.org/secure-connections/ #: (content/secure-connections/contents+en.lrtopic.body) msgid "##### POTENTIALLY VISIBLE DATA" -msgstr "" +msgstr "##### قابلیت مشاهده داده ها" #: https//tb-manual.torproject.org/secure-connections/ #: (content/secure-connections/contents+en.lrtopic.body)

1 0

[translation/tails-persistence-setup] Update translations for tails-persistence-setup
by translation＠torproject.org 17 Aug '19

17 Aug '19

commit d7da1353d11926544a32b1feb4190623458971e2 Author: Translation commit bot <translation(a)torproject.org> Date: Sat Aug 17 11:57:26 2019 +0000 Update translations for tails-persistence-setup --- eo/eo.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eo/eo.po b/eo/eo.po index c3026b5ed..4736cfc0d 100644 --- a/eo/eo.po +++ b/eo/eo.po @@ -9,7 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: Tails developers <tails(a)boum.org>\n" "POT-Creation-Date: 2019-03-17 09:03+0100\n" "PO-Revision-Date: 2019-07-11 23:13+0000\n" -"Last-Translator: Tyquan Bush\n" +"Last-Translator: 6c9ad24fb5ff3a00859fb84514d79173\n" "Language-Team: Esperanto (http://www.transifex.com/otf/torproject/language/eo/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n"

1 0