May 2018 - tor-commits - lists.torproject.org

[stem/master] Unit test directory equality
by atagar＠torproject.org 08 May '18

08 May '18

commit d893ec53b03bda8fa973e0552c65999b5eb5167f Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 6 13:03:24 2018 -0700 Unit test directory equality Now that we've cleaned up our equality checks it's a natural spot to start testing. Found an interesting bug where 'False' and 'None' values resulted in directories being considered equal. --- stem/util/__init__.py | 8 ++++++++ test/settings.cfg | 1 + test/unit/directory/__init__.py | 1 + test/unit/directory/authority.py | 28 ++++++++++++++++++++++++++++ test/unit/directory/fallback.py | 24 ++++++++++++++++++++++++ 5 files changed, 62 insertions(+) diff --git a/stem/util/__init__.py b/stem/util/__init__.py index 5ccd921a..711f28a5 100644 --- a/stem/util/__init__.py +++ b/stem/util/__init__.py @@ -31,6 +31,12 @@ else: str_type = unicode int_type = long +# Python hashes booleans to zero or one. Usually this would be fine, but since +# we use hashes for equality checks we need them to be something less common. + +TRUE_HASH_VALUE = 4813749 +FALSE_HASH_VALUE = 5826450 + def datetime_to_unix(timestamp): """ @@ -72,6 +78,8 @@ def _hash_attr(obj, *attributes, **kwargs): elif isinstance(attr_value, (list, tuple)): for entry in attr_value: my_hash = (my_hash + hash(entry)) * 1024 + elif isinstance(attr_value, bool): + my_hash += TRUE_HASH_VALUE if attr_value else FALSE_HASH_VALUE else: my_hash += hash(attr_value) diff --git a/test/settings.cfg b/test/settings.cfg index 2206c7e7..312e3ce9 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -220,6 +220,7 @@ test.unit_tests |test.unit.endpoint.TestEndpoint |test.unit.version.TestVersion |test.unit.manual.TestManual +|test.unit.directory.authority.TestAuthority |test.unit.directory.fallback.TestFallback |test.unit.tutorial.TestTutorial |test.unit.tutorial_examples.TestTutorialExamples diff --git a/test/unit/directory/__init__.py b/test/unit/directory/__init__.py index 78e42d7c..7ca7ea92 100644 --- a/test/unit/directory/__init__.py +++ b/test/unit/directory/__init__.py @@ -3,5 +3,6 @@ Unit tests for stem.directory. """ __all__ = [ + 'authority', 'fallback', ] diff --git a/test/unit/directory/authority.py b/test/unit/directory/authority.py new file mode 100644 index 00000000..4a5f3a12 --- /dev/null +++ b/test/unit/directory/authority.py @@ -0,0 +1,28 @@ +""" +Unit tests for stem.directory.Authority. +""" + +import unittest + +import stem.directory + + +class TestAuthority(unittest.TestCase): + def test_equality(self): + authority_attr = { + 'address': '5.9.110.236', + 'or_port': 9001, + 'dir_port': 9030, + 'fingerprint': '0756B7CD4DFC8182BE23143FAC0642F515182CEB', + 'nickname': 'rueckgrat', + 'v3ident': '23D15D965BC35114467363C165C4F724B64B4F66', + 'is_bandwidth_authority': False, + } + + self.assertEqual(stem.directory.Authority(**authority_attr), stem.directory.Authority(**authority_attr)) + + for attr in authority_attr: + for value in (None, 'something else'): + second_authority = dict(authority_attr) + second_authority[attr] = value + self.assertNotEqual(stem.directory.Authority(**authority_attr), stem.directory.Authority(**second_authority)) diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py index c95d74b7..cd857b9f 100644 --- a/test/unit/directory/fallback.py +++ b/test/unit/directory/fallback.py @@ -65,6 +65,30 @@ FALLBACK_ENTRY = b"""\ class TestFallback(unittest.TestCase): + def test_equality(self): + fallback_attr = { + 'address': '5.9.110.236', + 'or_port': 9001, + 'dir_port': 9030, + 'fingerprint': '0756B7CD4DFC8182BE23143FAC0642F515182CEB', + 'nickname': 'rueckgrat', + 'has_extrainfo': True, + 'orport_v6': ('2a01:4f8:162:51e2::2', 9001), + 'header': OrderedDict(( + ('type', 'fallback'), + ('version', '2.0.0'), + ('timestamp', '20170526090242'), + )), + } + + self.assertEqual(stem.directory.Fallback(**fallback_attr), stem.directory.Fallback(**fallback_attr)) + + for attr in fallback_attr: + for value in (None, 'something else'): + second_fallback = dict(fallback_attr) + second_fallback[attr] = value + self.assertNotEqual(stem.directory.Fallback(**fallback_attr), stem.directory.Fallback(**second_fallback)) + def test_from_cache(self): # quick sanity test that we can load cached content fallback_directories = stem.directory.Fallback.from_cache()

1 0

[stem/master] Double checking fields in equality check
by atagar＠torproject.org 08 May '18

08 May '18

commit 33b20dcbd4b1f0ddc84085766dfbcfffc5e5e18a Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 6 12:05:51 2018 -0700 Double checking fields in equality check Our Directory class hashed its attributes, then the subclasses hashed some of them again. Not harmful, but also not necessary. --- stem/directory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stem/directory.py b/stem/directory.py index 2982f4d0..7b78696a 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -113,7 +113,7 @@ class Directory(object): """ Provides cached Tor directory information. This information is hardcoded into Tor and occasionally changes, so the information this provides might - not necessarily match your version of tor. + not necessarily match the latest version of tor. .. versionadded:: 1.5.0 @@ -156,7 +156,7 @@ class Directory(object): raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') def __hash__(self): - return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint') + return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname') def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Directory) else False @@ -299,7 +299,7 @@ class Authority(Directory): return section_lines def __hash__(self): - return _hash_attr(self, 'nickname', 'v3ident', 'is_bandwidth_authority', parent = Directory) + return _hash_attr(self, 'v3ident', 'is_bandwidth_authority', parent = Directory) def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Authority) else False @@ -574,7 +574,7 @@ class Fallback(Directory): conf.save(path) def __hash__(self): - return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'has_extrainfo', 'orport_v6', 'header', parent = Directory) + return _hash_attr(self, 'has_extrainfo', 'orport_v6', 'header', parent = Directory) def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Fallback) else False

1 0

[stem/master] Move helper functions to end of remote module
by atagar＠torproject.org 08 May '18

08 May '18

commit df2df79855ff92d3452fa5841f18397895424861 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 13:46:53 2018 -0700 Move helper functions to end of remote module Generally I've kept with the ordering of: functions, classes, helpers. This way the publicly accessible stuff is first. --- stem/descriptor/remote.py | 304 +++++++++++++++++++++++----------------------- 1 file changed, 152 insertions(+), 152 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 1b6a7ea6..bc23063a 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -202,158 +202,6 @@ def get_consensus(authority_v3ident = None, microdescriptor = False, **query_arg return get_instance().get_consensus(authority_v3ident, microdescriptor, **query_args) -def _download_from_orport(endpoint, compression, resource): - """ - Downloads descriptors from the given orport. Payload is just like an http - response (headers and all)... - - :: - - HTTP/1.0 200 OK - Date: Mon, 23 Apr 2018 18:43:47 GMT - Content-Type: text/plain - X-Your-Address-Is: 216.161.254.25 - Content-Encoding: identity - Expires: Wed, 25 Apr 2018 18:43:47 GMT - - router dannenberg 193.23.244.244 443 0 80 - identity-ed25519 - ... rest of the descriptor content... - - :param stem.ORPort endpoint: endpoint to download from - :param list compression: compression methods for the request - :param str resource: descriptor resource to download - - :returns: two value tuple of the form (data, reply_headers) - - :raises: - * :class:`stem.ProtocolError` if not a valid descriptor response - * :class:`stem.SocketError` if unable to establish a connection - """ - - link_protocols = endpoint.link_protocols if endpoint.link_protocols else [3] - - with stem.client.Relay.connect(endpoint.address, endpoint.port, link_protocols) as relay: - with relay.create_circuit() as circ: - request = '\r\n'.join(( - 'GET %s HTTP/1.0' % resource, - 'Accept-Encoding: %s' % ', '.join(compression), - 'User-Agent: Stem/%s' % stem.__version__, - )) + '\r\n\r\n' - - circ.send('RELAY_BEGIN_DIR', stream_id = 1) - response = b''.join([cell.data for cell in circ.send('RELAY_DATA', request, stream_id = 1)]) - first_line, data = response.split(b'\r\n', 1) - header_data, data = data.split(b'\r\n\r\n', 1) - - if first_line != b'HTTP/1.0 200 OK': - raise stem.ProtocolError("Response should begin with HTTP success, but was '%s'" % first_line) - - headers = {} - - for line in str_tools._to_unicode(header_data).splitlines(): - if ': ' not in line: - raise stem.ProtocolError("'%s' is not a HTTP header:\n\n%s" % line) - - key, value = line.split(': ', 1) - headers[key] = value - - return _decompress(data, headers.get('Content-Encoding')), headers - - -def _download_from_dirport(url, compression, timeout): - """ - Downloads descriptors from the given url. - - :param str url: dirport url from which to download from - :param list compression: compression methods for the request - :param float timeout: duration before we'll time out our request - - :returns: two value tuple of the form (data, reply_headers) - - :raises: - * **socket.timeout** if our request timed out - * **urllib2.URLError** for most request failures - """ - - response = urllib.urlopen( - urllib.Request( - url, - headers = { - 'Accept-Encoding': ', '.join(compression), - 'User-Agent': 'Stem/%s' % stem.__version__, - } - ), - timeout = timeout, - ) - - return _decompress(response.read(), response.headers.get('Content-Encoding')), response.headers - - -def _decompress(data, encoding): - """ - Decompresses descriptor data. - - Tor doesn't include compression headers. As such when using gzip we - need to include '32' for automatic header detection... - - https://stackoverflow.com/questions/3122145/zlib-error-error-3-while-decomp… - - ... and with zstd we need to use the streaming API. - - :param bytes data: data we received - :param str encoding: 'Content-Encoding' header of the response - - :raises: - * **ValueError** if encoding is unrecognized - * **ImportError** if missing the decompression module - """ - - if encoding == Compression.PLAINTEXT: - return data.strip() - elif encoding in (Compression.GZIP, 'deflate'): - return zlib.decompress(data, zlib.MAX_WBITS | 32).strip() - elif encoding == Compression.ZSTD: - if not stem.prereq.is_zstd_available(): - raise ImportError('Decompressing zstd data requires https://pypi.python.org/pypi/zstandard') - - import zstd - output_buffer = io.BytesIO() - - with zstd.ZstdDecompressor().write_to(output_buffer) as decompressor: - decompressor.write(data) - - return output_buffer.getvalue().strip() - elif encoding == Compression.LZMA: - if not stem.prereq.is_lzma_available(): - raise ImportError('Decompressing lzma data requires https://docs.python.org/3/library/lzma.html') - - import lzma - return lzma.decompress(data).strip() - else: - raise ValueError("'%s' isn't a recognized type of encoding" % encoding) - - -def _guess_descriptor_type(resource): - # Attempts to determine the descriptor type based on the resource url. This - # raises a ValueError if the resource isn't recognized. - - if resource.startswith('/tor/server/'): - return 'server-descriptor 1.0' - elif resource.startswith('/tor/extra/'): - return 'extra-info 1.0' - elif resource.startswith('/tor/micro/'): - return 'microdescriptor 1.0' - elif resource.startswith('/tor/status-vote/current/consensus-microdesc'): - return 'network-status-microdesc-consensus-3 1.0' - elif resource.startswith('/tor/status-vote/'): - return 'network-status-consensus-3 1.0' - elif resource.startswith('/tor/keys/'): - return 'dir-key-certificate-3 1.0' - else: - raise ValueError("Unable to determine the descriptor type for '%s'" % resource) - - class Query(object): """ Asynchronous request for descriptor content from a directory authority or @@ -962,6 +810,158 @@ class DescriptorDownloader(object): return Query(resource, **args) +def _download_from_orport(endpoint, compression, resource): + """ + Downloads descriptors from the given orport. Payload is just like an http + response (headers and all)... + + :: + + HTTP/1.0 200 OK + Date: Mon, 23 Apr 2018 18:43:47 GMT + Content-Type: text/plain + X-Your-Address-Is: 216.161.254.25 + Content-Encoding: identity + Expires: Wed, 25 Apr 2018 18:43:47 GMT + + router dannenberg 193.23.244.244 443 0 80 + identity-ed25519 + ... rest of the descriptor content... + + :param stem.ORPort endpoint: endpoint to download from + :param list compression: compression methods for the request + :param str resource: descriptor resource to download + + :returns: two value tuple of the form (data, reply_headers) + + :raises: + * :class:`stem.ProtocolError` if not a valid descriptor response + * :class:`stem.SocketError` if unable to establish a connection + """ + + link_protocols = endpoint.link_protocols if endpoint.link_protocols else [3] + + with stem.client.Relay.connect(endpoint.address, endpoint.port, link_protocols) as relay: + with relay.create_circuit() as circ: + request = '\r\n'.join(( + 'GET %s HTTP/1.0' % resource, + 'Accept-Encoding: %s' % ', '.join(compression), + 'User-Agent: Stem/%s' % stem.__version__, + )) + '\r\n\r\n' + + circ.send('RELAY_BEGIN_DIR', stream_id = 1) + response = b''.join([cell.data for cell in circ.send('RELAY_DATA', request, stream_id = 1)]) + first_line, data = response.split(b'\r\n', 1) + header_data, data = data.split(b'\r\n\r\n', 1) + + if first_line != b'HTTP/1.0 200 OK': + raise stem.ProtocolError("Response should begin with HTTP success, but was '%s'" % first_line) + + headers = {} + + for line in str_tools._to_unicode(header_data).splitlines(): + if ': ' not in line: + raise stem.ProtocolError("'%s' is not a HTTP header:\n\n%s" % line) + + key, value = line.split(': ', 1) + headers[key] = value + + return _decompress(data, headers.get('Content-Encoding')), headers + + +def _download_from_dirport(url, compression, timeout): + """ + Downloads descriptors from the given url. + + :param str url: dirport url from which to download from + :param list compression: compression methods for the request + :param float timeout: duration before we'll time out our request + + :returns: two value tuple of the form (data, reply_headers) + + :raises: + * **socket.timeout** if our request timed out + * **urllib2.URLError** for most request failures + """ + + response = urllib.urlopen( + urllib.Request( + url, + headers = { + 'Accept-Encoding': ', '.join(compression), + 'User-Agent': 'Stem/%s' % stem.__version__, + } + ), + timeout = timeout, + ) + + return _decompress(response.read(), response.headers.get('Content-Encoding')), response.headers + + +def _decompress(data, encoding): + """ + Decompresses descriptor data. + + Tor doesn't include compression headers. As such when using gzip we + need to include '32' for automatic header detection... + + https://stackoverflow.com/questions/3122145/zlib-error-error-3-while-decomp… + + ... and with zstd we need to use the streaming API. + + :param bytes data: data we received + :param str encoding: 'Content-Encoding' header of the response + + :raises: + * **ValueError** if encoding is unrecognized + * **ImportError** if missing the decompression module + """ + + if encoding == Compression.PLAINTEXT: + return data.strip() + elif encoding in (Compression.GZIP, 'deflate'): + return zlib.decompress(data, zlib.MAX_WBITS | 32).strip() + elif encoding == Compression.ZSTD: + if not stem.prereq.is_zstd_available(): + raise ImportError('Decompressing zstd data requires https://pypi.python.org/pypi/zstandard') + + import zstd + output_buffer = io.BytesIO() + + with zstd.ZstdDecompressor().write_to(output_buffer) as decompressor: + decompressor.write(data) + + return output_buffer.getvalue().strip() + elif encoding == Compression.LZMA: + if not stem.prereq.is_lzma_available(): + raise ImportError('Decompressing lzma data requires https://docs.python.org/3/library/lzma.html') + + import lzma + return lzma.decompress(data).strip() + else: + raise ValueError("'%s' isn't a recognized type of encoding" % encoding) + + +def _guess_descriptor_type(resource): + # Attempts to determine the descriptor type based on the resource url. This + # raises a ValueError if the resource isn't recognized. + + if resource.startswith('/tor/server/'): + return 'server-descriptor 1.0' + elif resource.startswith('/tor/extra/'): + return 'extra-info 1.0' + elif resource.startswith('/tor/micro/'): + return 'microdescriptor 1.0' + elif resource.startswith('/tor/status-vote/current/consensus-microdesc'): + return 'network-status-microdesc-consensus-3 1.0' + elif resource.startswith('/tor/status-vote/'): + return 'network-status-consensus-3 1.0' + elif resource.startswith('/tor/keys/'): + return 'dir-key-certificate-3 1.0' + else: + raise ValueError("Unable to determine the descriptor type for '%s'" % resource) + + def get_authorities(): """ Provides cached Tor directory authority information. The directory

1 0

[stem/master] Separate stem.directory integ tests
by atagar＠torproject.org 08 May '18

08 May '18

commit 61a1fe90797814b871c94929b533cf98e7715e30 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 7 07:40:42 2018 -0700 Separate stem.directory integ tests Moving the authority and fallback integ tests into their own modules. --- test/integ/descriptor/remote.py | 72 --------------------------------------- test/integ/directory/__init__.py | 8 +++++ test/integ/directory/authority.py | 38 +++++++++++++++++++++ test/integ/directory/fallback.py | 54 +++++++++++++++++++++++++++++ test/settings.cfg | 2 ++ 5 files changed, 102 insertions(+), 72 deletions(-) diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index b3553598..0305d069 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -245,75 +245,3 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertTrue(isinstance(single_query_results[0], stem.descriptor.networkstatus.KeyCertificate)) self.assertEqual(2, len(list(multiple_query))) - - @test.require.online - def test_authority_cache_is_up_to_date(self): - """ - Check if the cached authorities bundled with Stem are up to date or not. - """ - - cached_authorities = stem.descriptor.remote.get_authorities() - latest_authorities = stem.descriptor.remote.DirectoryAuthority.from_remote() - - for nickname in cached_authorities: - if nickname not in latest_authorities: - self.fail('%s is no longer a directory authority in tor' % nickname) - - for nickname in latest_authorities: - if nickname not in cached_authorities: - self.fail('%s is now a directory authority in tor' % nickname) - - # tor doesn't note if an autority is a bwauth or not, so we need to exclude - # that from our comparison - - for attr in ('address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'v3ident'): - for auth in cached_authorities.values(): - cached_value = getattr(auth, attr) - latest_value = getattr(latest_authorities[auth.nickname], attr) - - if cached_value != latest_value: - self.fail('The %s of the %s authority is %s in tor but %s in stem' % (attr, auth.nickname, latest_value, cached_value)) - - @test.require.online - def test_fallback_cache_is_up_to_date(self): - """ - Check if the cached fallback directories bundled with Stem are up to date - or not. - """ - - cached_fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache() - latest_fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() - - if cached_fallback_directories != latest_fallback_directories: - self.fail("Stem's cached fallback directories are out of date. Please run 'cache_fallback_directories.py'...\n\n%s" % stem.descriptor.remote._fallback_directory_differences(cached_fallback_directories, latest_fallback_directories)) - - @test.require.online - def test_fallback_directory_reachability(self): - """ - Fetch information from each fallback directory to confirm that it's - available. - """ - - # Don't run this test by default. Once upon a time it was fine, but tor has - # added so many fallbacks now that this takes a looong time. :( - - self.skipTest('(skipped by default)') - return - - unsuccessful = {} - downloader = stem.descriptor.remote.DescriptorDownloader() - moria1_v3ident = stem.descriptor.remote.get_authorities()['moria1'].v3ident - - for fallback_directory in stem.descriptor.remote.FallbackDirectory.from_cache().values(): - try: - downloader.get_key_certificates(authority_v3idents = moria1_v3ident, endpoints = [(fallback_directory.address, fallback_directory.dir_port)]).run() - except Exception as exc: - unsuccessful[fallback_directory] = exc - - if unsuccessful: - lines = ['We were unable to contact the following fallback directories...\n'] - - for fallback_directory, exc in unsuccessful.items(): - lines.append('* %s:%s (%s): %s' % (fallback_directory.address, fallback_directory.dir_port, fallback_directory.fingerprint, exc)) - - self.fail('\n'.join(lines)) diff --git a/test/integ/directory/__init__.py b/test/integ/directory/__init__.py new file mode 100644 index 00000000..f4e52c12 --- /dev/null +++ b/test/integ/directory/__init__.py @@ -0,0 +1,8 @@ +""" +Integration tests for stem.directory. +""" + +__all__ = [ + 'authority', + 'fallback', +] diff --git a/test/integ/directory/authority.py b/test/integ/directory/authority.py new file mode 100644 index 00000000..fc5eb13e --- /dev/null +++ b/test/integ/directory/authority.py @@ -0,0 +1,38 @@ +""" +Integration tests for stem.directory.Authority. +""" + +import unittest + +import stem.directory +import test.require + + +class TestAuthority(unittest.TestCase): + @test.require.online + def test_cache_is_up_to_date(self): + """ + Check if the cached authorities we bundle are up to date. + """ + + cached_authorities = stem.directory.Authority.from_cache() + latest_authorities = stem.directory.Authority.from_remote() + + for nickname in cached_authorities: + if nickname not in latest_authorities: + self.fail('%s is no longer a directory authority in tor' % nickname) + + for nickname in latest_authorities: + if nickname not in cached_authorities: + self.fail('%s is now a directory authority in tor' % nickname) + + # tor doesn't note if an autority is a bwauth or not, so we need to exclude + # that from our comparison + + for attr in ('address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'v3ident'): + for auth in cached_authorities.values(): + cached_value = getattr(auth, attr) + latest_value = getattr(latest_authorities[auth.nickname], attr) + + if cached_value != latest_value: + self.fail('The %s of the %s authority is %s in tor but %s in stem' % (attr, auth.nickname, latest_value, cached_value)) diff --git a/test/integ/directory/fallback.py b/test/integ/directory/fallback.py new file mode 100644 index 00000000..e12b2659 --- /dev/null +++ b/test/integ/directory/fallback.py @@ -0,0 +1,54 @@ +""" +Integration tests for stem.directory.Fallback. +""" + +import unittest + +import stem.descriptor.remote +import stem.directory +import test.require + + +class TestFallback(unittest.TestCase): + @test.require.online + def test_cache_is_up_to_date(self): + """ + Check if the cached fallbacks we bundle are up to date. + """ + + cached_fallback_directories = stem.directory.Fallback.from_cache() + latest_fallback_directories = stem.directory.Fallback.from_remote() + + if cached_fallback_directories != latest_fallback_directories: + self.fail("Stem's cached fallback directories are out of date. Please run 'cache_fallback_directories.py'...\n\n%s" % stem.directory._fallback_directory_differences(cached_fallback_directories, latest_fallback_directories)) + + @test.require.online + def test_fallback_directory_reachability(self): + """ + Fetch information from each fallback directory to confirm that it's + available. + """ + + # Don't run this test by default. Once upon a time it was fine, but tor has + # added so many fallbacks now that this takes a looong time. :( + + self.skipTest('(skipped by default)') + return + + unsuccessful = {} + downloader = stem.descriptor.remote.DescriptorDownloader() + moria1_v3ident = stem.directory.Authority.from_cache()['moria1'].v3ident + + for fallback_directory in stem.directory.Fallback.from_cache().values(): + try: + downloader.get_key_certificates(authority_v3idents = moria1_v3ident, endpoints = [(fallback_directory.address, fallback_directory.dir_port)]).run() + except Exception as exc: + unsuccessful[fallback_directory] = exc + + if unsuccessful: + lines = ['We were unable to contact the following fallback directories...\n'] + + for fallback_directory, exc in unsuccessful.items(): + lines.append('* %s:%s (%s): %s' % (fallback_directory.address, fallback_directory.dir_port, fallback_directory.fingerprint, exc)) + + self.fail('\n'.join(lines)) diff --git a/test/settings.cfg b/test/settings.cfg index 312e3ce9..bd99a9d2 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -256,6 +256,8 @@ test.integ_tests |test.integ.interpreter.TestInterpreter |test.integ.version.TestVersion |test.integ.manual.TestManual +|test.integ.directory.authority.TestAuthority +|test.integ.directory.fallback.TestFallback |test.integ.client.connection.TestConnection |test.integ.response.protocolinfo.TestProtocolInfo |test.integ.socket.control_socket.TestControlSocket

1 0

[stem/master] Move authority docs into module header
by atagar＠torproject.org 08 May '18

08 May '18

commit d99d6470ccc50eeefec056f2305cdacd05e07eef Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 14:44:57 2018 -0700 Move authority docs into module header This more expansive explanation of how tor works belongs in our module docs, not this particular class. --- stem/directory.py | 62 +++++++++++++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/stem/directory.py b/stem/directory.py index 4190e885..2982f4d0 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -2,16 +2,38 @@ # See LICENSE for licensing information """ -Directories with Tor descriptor information. +Directories with `tor descriptor information +<../tutorials/mirror_mirror_on_the_wall.html>`_. At a very high level tor works +as follows... + +1. Volunteer starts a new tor relay, during which it sends a `server + descriptor <descriptor/server_descriptor.html>`_ to each of the directory + authorities. + +2. Each hour the directory authorities make a `vote + <descriptor/networkstatus.html>`_ that says who they think the active + relays are in the network and some attributes about them. + +3. The directory authorities send each other their votes, and compile that + into the `consensus <descriptor/networkstatus.html>`_. This document is very + similar to the votes, the only difference being that the majority of the + authorities agree upon and sign this document. The idividual relay entries + in the vote or consensus is called `router status entries + <descriptor/router_status_entry.html>`_. + +4. Tor clients (people using the service) download the consensus from an + authority, fallback, or other mirror to determine who the active relays in + the network are. They then use this to construct circuits and use the + network. :: - Directory - Relay we can retrieve directory information from - | |- from_cache - Provides fallback directories cached with Stem. - | +- from_remote - Retrieves fallback directories remotely from tor's latest commit. + Directory - Relay we can retrieve descriptor information from + | |- from_cache - Provides cached information bundled with Stem. + | +- from_remote - Downloads the latest directory information from tor. | - |- Authority - Information about a tor directory authority - +- Fallback - Directory mirror tor uses when authories are unavailable + |- Authority - Tor directory authority + +- Fallback - Mirrors that can be used instead of the authorities .. versionadded:: 1.7.0 """ @@ -149,28 +171,6 @@ class Authority(Directory): <https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc>`_ that enumerates the other relays within the network. - At a very high level tor works as follows... - - 1. A volunteer starts up a new tor relay, during which it sends a `server - descriptor <server_descriptor.html>`_ to each of the directory - authorities. - - 2. Each hour the directory authorities make a `vote <networkstatus.html>`_ - that says who they think the active relays are in the network and some - attributes about them. - - 3. The directory authorities send each other their votes, and compile that - into the `consensus <networkstatus.html>`_. This document is very similar - to the votes, the only difference being that the majority of the - authorities agree upon and sign this document. The idividual relay entries - in the vote or consensus is called `router status entries - <router_status_entry.html>`_. - - 4. Tor clients (people using the service) download the consensus from one of - the authorities or a mirror to determine the active relays within the - network. They in turn use this to construct their circuits and use the - network. - .. versionchanged:: 1.3.0 Added the is_bandwidth_authority attribute. @@ -319,14 +319,12 @@ class Fallback(Directory): :: import time - from stem.descriptor.remote import DescriptorDownloader + from stem.descriptor.remote import get_consensus from stem.directory import Fallback - downloader = DescriptorDownloader() - for fallback in Fallback.from_cache().values(): start = time.time() - downloader.get_consensus(endpoints = [(fallback.address, fallback.dir_port)]).run() + get_consensus(endpoints = [(fallback.address, fallback.dir_port)]).run() print('Downloading the consensus took %0.2f from %s' % (time.time() - start, fallback.fingerprint)) ::

1 0

[stem/master] Remove usage of directory aliases
by atagar＠torproject.org 08 May '18

08 May '18

commit 0a583e678ac9b359685e24e3299337d2f8ced99f Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 7 10:07:00 2018 -0700 Remove usage of directory aliases We provide backward compatablitiy, but we ourselves should reference the new module. --- docs/_static/example/compare_flags.py | 15 ++++++++++----- docs/_static/example/votes_by_bandwidth_authorities.py | 7 ++++--- stem/descriptor/remote.py | 8 ++++---- test/integ/descriptor/remote.py | 9 +++++---- test/unit/descriptor/remote.py | 4 ---- test/unit/tutorial_examples.py | 12 ++++++------ 6 files changed, 29 insertions(+), 26 deletions(-) diff --git a/docs/_static/example/compare_flags.py b/docs/_static/example/compare_flags.py index df317a89..1efb1d2c 100644 --- a/docs/_static/example/compare_flags.py +++ b/docs/_static/example/compare_flags.py @@ -1,16 +1,21 @@ -from collections import OrderedDict -from stem.descriptor import DocumentHandler, remote +import collections + +import stem.descriptor +import stem.descriptor.remote +import stem.directory # Query all authority votes asynchronously. -downloader = remote.DescriptorDownloader(document_handler=DocumentHandler.DOCUMENT) +downloader = stem.descriptor.remote.DescriptorDownloader( + document_handler = stem.descriptor.DocumentHandler.DOCUMENT, +) # An ordered dictionary ensures queries are finished in the order they were # added. -queries = OrderedDict() +queries = collections.OrderedDict() -for name, authority in remote.get_authorities().items(): +for name, authority in stem.directory.Authority.from_cache().items(): if authority.v3ident is None: continue # authority doesn't vote if it lacks a v3ident diff --git a/docs/_static/example/votes_by_bandwidth_authorities.py b/docs/_static/example/votes_by_bandwidth_authorities.py index f6816b74..9504bd94 100644 --- a/docs/_static/example/votes_by_bandwidth_authorities.py +++ b/docs/_static/example/votes_by_bandwidth_authorities.py @@ -1,11 +1,12 @@ -from stem.descriptor import remote +import stem.descriptor.remote +import stem.directory # request votes from all the bandwidth authorities queries = {} -downloader = remote.DescriptorDownloader() +downloader = stem.descriptor.remote.DescriptorDownloader() -for authority in remote.get_authorities().values(): +for authority in stem.directory.Authority.from_cache().values(): if authority.is_bandwidth_authority: queries[authority.nickname] = downloader.query( '/tor/status-vote/current/authority', diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index bc23063a..a8be9280 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -496,7 +496,7 @@ class Query(object): """ if use_authority or not self.endpoints: - picked = random.choice([auth for auth in get_authorities().values() if auth.nickname not in ('tor26', 'Bifroest')]) + picked = random.choice([auth for auth in stem.directory.Authority.from_cache().values() if auth.nickname not in ('tor26', 'Bifroest')]) return stem.DirPort(picked.address, picked.dir_port) else: return random.choice(self.endpoints) @@ -547,7 +547,7 @@ class DescriptorDownloader(object): def __init__(self, use_mirrors = False, **default_args): self._default_args = default_args - directories = list(get_authorities().values()) + directories = list(stem.directory.Authority.from_cache().values()) self._endpoints = [(directory.address, directory.dir_port) for directory in directories] if use_mirrors: @@ -569,7 +569,7 @@ class DescriptorDownloader(object): :raises: **Exception** if unable to determine the directory mirrors """ - directories = get_authorities().values() + directories = stem.directory.Authority.from_cache().values() new_endpoints = set([(directory.address, directory.dir_port) for directory in directories]) consensus = list(self.get_consensus(document_handler = stem.descriptor.DocumentHandler.DOCUMENT).run())[0] @@ -735,7 +735,7 @@ class DescriptorDownloader(object): """ Provides the present vote for a given directory authority. - :param stem.descriptor.remote.DirectoryAuthority authority: authority for which to retrieve a vote for + :param stem.directory.Authority authority: authority for which to retrieve a vote for :param query_args: additional arguments for the :class:`~stem.descriptor.remote.Query` constructor diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index 0305d069..c4e4f699 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -11,6 +11,7 @@ import stem.descriptor.networkstatus import stem.descriptor.remote import stem.descriptor.router_status_entry import stem.descriptor.server_descriptor +import stem.directory import test.require @@ -18,7 +19,7 @@ class TestDescriptorDownloader(unittest.TestCase): @test.require.only_run_once @test.require.online def test_downloading_via_orport(self): - moria1 = stem.descriptor.remote.get_authorities()['moria1'] + moria1 = stem.directory.Authority.from_cache()['moria1'] desc = list(stem.descriptor.remote.their_server_descriptor( endpoints = [stem.ORPort(moria1.address, moria1.or_port)], @@ -31,7 +32,7 @@ class TestDescriptorDownloader(unittest.TestCase): @test.require.only_run_once @test.require.online def test_downloading_via_dirport(self): - moria1 = stem.descriptor.remote.get_authorities()['moria1'] + moria1 = stem.directory.Authority.from_cache()['moria1'] desc = list(stem.descriptor.remote.their_server_descriptor( endpoints = [stem.DirPort(moria1.address, moria1.dir_port)], @@ -73,7 +74,7 @@ class TestDescriptorDownloader(unittest.TestCase): if auth.nickname == 'dannenberg-legacy': continue # skip due to https://trac.torproject.org/projects/tor/ticket/17906 - stem_auth = stem.descriptor.remote.get_authorities().get(auth.nickname) + stem_auth = stem.directory.Authority.from_cache().get(auth.nickname) if not stem_auth: self.fail("%s isn't a recognized directory authority in stem" % auth.nickname) @@ -99,7 +100,7 @@ class TestDescriptorDownloader(unittest.TestCase): queries = [] - for nickname, authority in stem.descriptor.remote.get_authorities().items(): + for nickname, authority in stem.directory.Authority.from_cache().items(): queries.append((stem.descriptor.remote.Query( '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31', 'server-descriptor 1.0', diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 478a7143..d654f61a 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -369,7 +369,3 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual(1, len(list(query))) self.assertEqual(1, len(list(query))) self.assertEqual(1, len(list(query))) - - def test_using_authorities_in_hash(self): - # ensure our DirectoryAuthority instances can be used in hashes - {stem.descriptor.remote.get_authorities()['moria1']: 'hello'} diff --git a/test/unit/tutorial_examples.py b/test/unit/tutorial_examples.py index d94da5bf..5ca5993e 100644 --- a/test/unit/tutorial_examples.py +++ b/test/unit/tutorial_examples.py @@ -234,8 +234,8 @@ class TestTutorialExamples(unittest.TestCase): @patch('sys.stdout', new_callable = StringIO) @patch('stem.descriptor.remote.Query') - @patch('stem.descriptor.remote.get_authorities') - def test_compare_flags(self, get_authorities_mock, query_mock, stdout_mock): + @patch('stem.directory.Authority.from_cache') + def test_compare_flags(self, authorities_mock, query_mock, stdout_mock): if stem.prereq._is_python_26(): # example imports OrderedDict from collections which doesn't work under # python 2.6 @@ -243,7 +243,7 @@ class TestTutorialExamples(unittest.TestCase): self.skipTest("(example doesn't support python 2.6)") return - get_authorities_mock().items.return_value = [('moria1', DIRECTORY_AUTHORITIES['moria1']), ('maatuska', DIRECTORY_AUTHORITIES['maatuska'])] + authorities_mock().items.return_value = [('moria1', DIRECTORY_AUTHORITIES['moria1']), ('maatuska', DIRECTORY_AUTHORITIES['maatuska'])] fingerprint = [ ('92FCB6748A40E6088E22FBAB943AB2DD743EA818', 'kvy2dIpA5giOIvurlDqy3XQ+qBg='), @@ -281,9 +281,9 @@ class TestTutorialExamples(unittest.TestCase): self.assert_equal_unordered(COMPARE_FLAGS_OUTPUT, stdout_mock.getvalue()) @patch('sys.stdout', new_callable = StringIO) - @patch('stem.descriptor.remote.get_authorities') + @patch('stem.directory.Authority.from_cache') @patch('stem.descriptor.remote.DescriptorDownloader.query') - def test_votes_by_bandwidth_authorities(self, query_mock, get_authorities_mock, stdout_mock): + def test_votes_by_bandwidth_authorities(self, query_mock, authorities_mock, stdout_mock): directory_values = [ DIRECTORY_AUTHORITIES['gabelmoo'], DIRECTORY_AUTHORITIES['moria1'], @@ -291,7 +291,7 @@ class TestTutorialExamples(unittest.TestCase): ] directory_values[0].address = '131.188.40.189' - get_authorities_mock().values.return_value = directory_values + authorities_mock().values.return_value = directory_values entry_with_measurement = RouterStatusEntryV3.create({'w': 'Bandwidth=1 Measured=1'}) entry_without_measurement = RouterStatusEntryV3.create()

1 0

[stem/master] Initial Authority.from_remote() unit tests
by atagar＠torproject.org 08 May '18

08 May '18

commit 1d467aee328887fb4259ebc498a6b8ddebc8e81d Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 7 10:40:31 2018 -0700 Initial Authority.from_remote() unit tests Adopting a couple Fallback tests for its Authority counterpart. The persistance tests aren't relevant to it (authorities are managed by hand). As for the from_str() tests I need to give this more thought. Maybe refactor the directory module a bit first. --- test/unit/directory/authority.py | 52 ++++++++++++++++++ test/unit/directory/fallback.py | 110 +++++++++++++++++++-------------------- 2 files changed, 107 insertions(+), 55 deletions(-) diff --git a/test/unit/directory/authority.py b/test/unit/directory/authority.py index 4a5f3a12..dcee4916 100644 --- a/test/unit/directory/authority.py +++ b/test/unit/directory/authority.py @@ -2,9 +2,29 @@ Unit tests for stem.directory.Authority. """ +import io import unittest import stem.directory +import stem.prereq + +try: + # added in python 3.3 + from unittest.mock import patch, Mock +except ImportError: + from mock import patch, Mock + +URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' + +AUTHORITY_GITWEB_CONTENT = b"""\ +"moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", +"tor26 orport=443 " + "v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 " + "ipv6=[2001:858:2:2:aabb:0:563b:1526]:443 " + "86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D", +""" class TestAuthority(unittest.TestCase): @@ -26,3 +46,35 @@ class TestAuthority(unittest.TestCase): second_authority = dict(authority_attr) second_authority[attr] = value self.assertNotEqual(stem.directory.Authority(**authority_attr), stem.directory.Authority(**second_authority)) + + def test_from_cache(self): + authorities = stem.directory.Authority.from_cache() + self.assertTrue(len(authorities) > 4) + self.assertEqual('128.31.0.39', authorities['moria1'].address) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(AUTHORITY_GITWEB_CONTENT))) + def test_from_remote(self): + expected = { + 'moria1': stem.directory.Authority( + nickname = 'moria1', + address = '128.31.0.39', + or_port = 9101, + dir_port = 9131, + fingerprint = '9695DFC35FFEB861329B9F1AB04C46397020CE31', + v3ident = 'D586D18309DED4CD6D57C18FDB97EFA96D330566', + ), + 'tor26': stem.directory.Authority( + nickname = 'tor26', + address = '86.59.21.38', + or_port = 443, + dir_port = 80, + fingerprint = '847B1F850344D7876491A54892F904934E4EB85D', + v3ident = '14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', + ), + } + + self.assertEqual(expected, stem.directory.Authority.from_remote()) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) + def test_from_remote_empty(self): + self.assertRaisesRegexp(IOError, 'did not have any content', stem.directory.Authority.from_remote) diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py index cd857b9f..1cbff9a3 100644 --- a/test/unit/directory/fallback.py +++ b/test/unit/directory/fallback.py @@ -23,7 +23,7 @@ except ImportError: URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' -FALLBACK_DIR_CONTENT = b"""\ +FALLBACK_GITWEB_CONTENT = b"""\ /* type=fallback */ /* version=2.0.0 */ /* timestamp=20170526090242 */ @@ -63,6 +63,12 @@ FALLBACK_ENTRY = b"""\ /* extrainfo=1 */ """ +HEADER = OrderedDict(( + ('type', 'fallback'), + ('version', '2.0.0'), + ('timestamp', '20170526090242'), +)) + class TestFallback(unittest.TestCase): def test_equality(self): @@ -90,16 +96,12 @@ class TestFallback(unittest.TestCase): self.assertNotEqual(stem.directory.Fallback(**fallback_attr), stem.directory.Fallback(**second_fallback)) def test_from_cache(self): - # quick sanity test that we can load cached content - fallback_directories = stem.directory.Fallback.from_cache() - self.assertTrue(len(fallback_directories) > 10) - self.assertEqual('5.39.92.199', fallback_directories['0BEA4A88D069753218EAAAD6D22EA87B9A1319D6'].address) + fallbacks = stem.directory.Fallback.from_cache() + self.assertTrue(len(fallbacks) > 10) + self.assertEqual('5.39.92.199', fallbacks['0BEA4A88D069753218EAAAD6D22EA87B9A1319D6'].address) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(FALLBACK_DIR_CONTENT))) + @patch(URL_OPEN, Mock(return_value = io.BytesIO(FALLBACK_GITWEB_CONTENT))) def test_from_remote(self): - fallback_directories = stem.directory.Fallback.from_remote() - header = OrderedDict((('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'))) - expected = { '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.directory.Fallback( address = '5.9.110.236', @@ -109,7 +111,7 @@ class TestFallback(unittest.TestCase): nickname = 'rueckgrat', has_extrainfo = True, orport_v6 = ('2a01:4f8:162:51e2::2', 9001), - header = header, + header = HEADER, ), '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.directory.Fallback( address = '193.171.202.146', @@ -119,15 +121,51 @@ class TestFallback(unittest.TestCase): nickname = None, has_extrainfo = False, orport_v6 = None, - header = header, + header = HEADER, ), } - self.assertEqual(expected, fallback_directories) + self.assertEqual(expected, stem.directory.Fallback.from_remote()) - def test_persistence(self): - header = OrderedDict((('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'))) + @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) + def test_from_remote_empty(self): + self.assertRaisesRegexp(IOError, 'did not have any content', stem.directory.Fallback.from_remote) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'\n'.join(FALLBACK_GITWEB_CONTENT.splitlines()[1:])))) + def test_from_remote_no_header(self): + self.assertRaisesRegexp(IOError, 'does not have a type field indicating it is fallback directory metadata', stem.directory.Fallback.from_remote) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(FALLBACK_GITWEB_CONTENT.replace(b'version=2.0.0', b'version')))) + def test_from_remote_malformed_header(self): + self.assertRaisesRegexp(IOError, 'Malformed fallback directory header line: /\* version \*/', stem.directory.Fallback.from_remote) + + def test_from_str(self): + expected = stem.directory.Fallback( + address = '5.9.110.236', + or_port = 9001, + dir_port = 9030, + fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', + nickname = 'rueckgrat', + has_extrainfo = True, + orport_v6 = ('2a01:4f8:162:51e2::2', 9001), + ) + + self.assertEqual(expected, stem.directory.Fallback._from_str(FALLBACK_ENTRY)) + + def test_from_str_malformed(self): + test_values = { + FALLBACK_ENTRY.replace(b'id=0756B7CD4DFC8182BE23143FAC0642F515182CEB', b''): 'Malformed fallback address line:', + FALLBACK_ENTRY.replace(b'5.9.110.236', b'5.9.110'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv4 address: 5.9.110', + FALLBACK_ENTRY.replace(b':9030', b':7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid dir_port: 7814713228', + FALLBACK_ENTRY.replace(b'orport=9001', b'orport=7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid or_port: 7814713228', + FALLBACK_ENTRY.replace(b'ipv6=[2a01', b'ipv6=[:::'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv6 address: ::::4f8:162:51e2::2', + FALLBACK_ENTRY.replace(b'nickname=rueckgrat', b'nickname=invalid~nickname'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid nickname: invalid~nickname', + } + + for entry, expected in test_values.items(): + self.assertRaisesRegexp(ValueError, expected, stem.directory.Fallback._from_str, entry) + def test_persistence(self): expected = { '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.directory.Fallback( address = '5.9.110.236', @@ -137,7 +175,7 @@ class TestFallback(unittest.TestCase): nickname = 'rueckgrat', has_extrainfo = True, orport_v6 = ('2a01:4f8:162:51e2::2', 9001), - header = header, + header = HEADER, ), '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.directory.Fallback( address = '193.171.202.146', @@ -147,7 +185,7 @@ class TestFallback(unittest.TestCase): nickname = None, has_extrainfo = False, orport_v6 = None, - header = header, + header = HEADER, ), } @@ -171,48 +209,10 @@ class TestFallback(unittest.TestCase): } with tempfile.NamedTemporaryFile(prefix = 'fallbacks.') as tmp: - stem.directory.Fallback._write(expected, 'abc', 'def', header, tmp.name) + stem.directory.Fallback._write(expected, 'abc', 'def', HEADER, tmp.name) conf = stem.util.conf.Config() conf.load(tmp.name) self.assertEqual(excepted_config, dict(conf)) self.assertEqual(expected, stem.directory.Fallback.from_cache(tmp.name)) - - @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) - def test_from_remote_empty(self): - self.assertRaisesRegexp(IOError, 'did not have any content', stem.directory.Fallback.from_remote) - - @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'\n'.join(FALLBACK_DIR_CONTENT.splitlines()[1:])))) - def test_from_remote_no_header(self): - self.assertRaisesRegexp(IOError, 'does not have a type field indicating it is fallback directory metadata', stem.directory.Fallback.from_remote) - - @patch(URL_OPEN, Mock(return_value = io.BytesIO(FALLBACK_DIR_CONTENT.replace(b'version=2.0.0', b'version')))) - def test_from_remote_malformed_header(self): - self.assertRaisesRegexp(IOError, 'Malformed fallback directory header line: /\* version \*/', stem.directory.Fallback.from_remote) - - def test_from_str(self): - expected = stem.directory.Fallback( - address = '5.9.110.236', - or_port = 9001, - dir_port = 9030, - fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', - nickname = 'rueckgrat', - has_extrainfo = True, - orport_v6 = ('2a01:4f8:162:51e2::2', 9001), - ) - - self.assertEqual(expected, stem.directory.Fallback._from_str(FALLBACK_ENTRY)) - - def test_from_str_malformed(self): - test_values = { - FALLBACK_ENTRY.replace(b'id=0756B7CD4DFC8182BE23143FAC0642F515182CEB', b''): 'Malformed fallback address line:', - FALLBACK_ENTRY.replace(b'5.9.110.236', b'5.9.110'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv4 address: 5.9.110', - FALLBACK_ENTRY.replace(b':9030', b':7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid dir_port: 7814713228', - FALLBACK_ENTRY.replace(b'orport=9001', b'orport=7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid or_port: 7814713228', - FALLBACK_ENTRY.replace(b'ipv6=[2a01', b'ipv6=[:::'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv6 address: ::::4f8:162:51e2::2', - FALLBACK_ENTRY.replace(b'nickname=rueckgrat', b'nickname=invalid~nickname'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid nickname: invalid~nickname', - } - - for entry, expected in test_values.items(): - self.assertRaisesRegexp(ValueError, expected, stem.directory.Fallback._from_str, entry)

1 0

[stem/master] Add orport_v6 to the Authority class
by atagar＠torproject.org 08 May '18

08 May '18

commit e554de7373e017c788ba411dd850d2e7c518390a Author: Damian Johnson <atagar(a)torproject.org> Date: Tue May 8 12:01:34 2018 -0700 Add orport_v6 to the Authority class Moving the orport_v6 attribute from Fallback up to the parent Directory class, effectively adding it to directory authorities. This was added to tor's backed in information. --- docs/change_log.rst | 1 + stem/directory.py | 51 +++++++++++++++++++++------------------- test/unit/directory/authority.py | 2 ++ 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index 67359a95..3836b8ad 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -60,6 +60,7 @@ The following are only available within Stem's `git repository * Moved the Directory classes into their own `stem.directory <api/directory.html>`_ module * Added :func:`~stem.descriptor.remote.Directory.from_cache` and :func:`~stem.descriptor.remote.Directory.from_remote` to the :class:`~stem.descriptor.remote.DirectoryAuthority` subclass * `Fallback directory v2 support <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_, which adds *nickname* and *extrainfo* + * Added the *orport_v6* attribute to the :class:`~stem.directory.Authority` class * Added server descriptor's new is_hidden_service_dir attribute * Don't retry downloading descriptors when we've timed out * Don't download from tor26 and Bifroest, which are authorities that frequently timeout diff --git a/stem/directory.py b/stem/directory.py index acd24a80..b638a354 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -91,17 +91,16 @@ class Directory(object): names, whereas fallbacks vary more and don't necessarily have a nickname to key off of. - .. versionchanged:: 1.3.0 - Moved nickname from subclasses to this base class. - :var str address: IPv4 address of the directory :var int or_port: port on which the relay services relay traffic :var int dir_port: port on which directory information is available :var str fingerprint: relay fingerprint :var str nickname: relay nickname + :var str orport_v6: **(address, port)** tuple for the directory's IPv6 + ORPort, or **None** if it doesn't have one """ - def __init__(self, address, or_port, dir_port, fingerprint, nickname): + def __init__(self, address, or_port, dir_port, fingerprint, nickname, orport_v6): identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint if not connection.is_valid_ipv4_address(address): @@ -115,11 +114,20 @@ class Directory(object): elif nickname and not tor_tools.is_valid_nickname(nickname): raise ValueError('%s has an invalid nickname: %s' % (fingerprint, nickname)) + if orport_v6: + if not isinstance(orport_v6, tuple) or len(orport_v6) != 2: + raise ValueError('%s orport_v6 should be a two value tuple: %s' % (identifier, str(orport_v6))) + elif not connection.is_valid_ipv6_address(orport_v6[0]): + raise ValueError('%s has an invalid IPv6 address: %s' % (identifier, orport_v6[0])) + elif not connection.is_valid_port(orport_v6[1]): + raise ValueError('%s has an invalid IPv6 port: %s' % (identifier, orport_v6[1])) + self.address = address self.or_port = int(or_port) self.dir_port = int(dir_port) self.fingerprint = fingerprint self.nickname = nickname + self.orport_v6 = (orport_v6[0], int(orport_v6[1])) if orport_v6 else None @staticmethod def from_cache(): @@ -169,7 +177,7 @@ class Directory(object): raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') def __hash__(self): - return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname') + return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'orport_v6') def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Directory) else False @@ -187,6 +195,9 @@ class Authority(Directory): .. versionchanged:: 1.3.0 Added the is_bandwidth_authority attribute. + .. versionchanged:: 1.7.0 + Added the orport_v6 attribute. + .. deprecated:: 1.7.0 The is_bandwidth_authority attribute is deprecated and will be removed in the future. @@ -194,8 +205,8 @@ class Authority(Directory): :var str v3ident: identity key fingerprint used to sign votes and consensus """ - def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False): - super(Authority, self).__init__(address, or_port, dir_port, fingerprint, nickname) + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, orport_v6 = None, v3ident = None, is_bandwidth_authority = False): + super(Authority, self).__init__(address, or_port, dir_port, fingerprint, nickname, orport_v6) identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint if v3ident and not tor_tools.is_valid_fingerprint(v3ident): @@ -271,7 +282,7 @@ class Authority(Directory): nickname, or_port = matches.get(AUTHORITY_NAME) v3ident = matches.get(AUTHORITY_V3IDENT) - # orport_v6 = matches.get(AUTHORITY_IPV6) # TODO: add this to stem's data? + orport_v6 = matches.get(AUTHORITY_IPV6) address, dir_port, fingerprint = matches.get(AUTHORITY_ADDR) return Authority( @@ -280,6 +291,7 @@ class Authority(Directory): dir_port = dir_port, fingerprint = fingerprint.replace(' ', ''), nickname = nickname, + orport_v6 = orport_v6, v3ident = v3ident, ) @@ -339,31 +351,18 @@ class Fallback(Directory): .. versionadded:: 1.5.0 .. versionchanged:: 1.7.0 - Added the nickname, has_extrainfo, and header attributes which are part of + Added the has_extrainfo, and header attributes which are part of the `second version of the fallback directories <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_. :var bool has_extrainfo: **True** if the relay should be able to provide extrainfo descriptors, **False** otherwise. - :var str orport_v6: **(address, port)** tuple for the directory's IPv6 - ORPort, or **None** if it doesn't have one :var dict header: metadata about the fallback directory file this originated from """ def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, has_extrainfo = False, orport_v6 = None, header = None): - super(Fallback, self).__init__(address, or_port, dir_port, fingerprint, nickname) - identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint - - if orport_v6: - if not isinstance(orport_v6, tuple) or len(orport_v6) != 2: - raise ValueError('%s orport_v6 should be a two value tuple: %s' % (identifier, str(orport_v6))) - elif not connection.is_valid_ipv6_address(orport_v6[0]): - raise ValueError('%s has an invalid IPv6 address: %s' % (identifier, orport_v6[0])) - elif not connection.is_valid_port(orport_v6[1]): - raise ValueError('%s has an invalid IPv6 port: %s' % (identifier, orport_v6[1])) - + super(Fallback, self).__init__(address, or_port, dir_port, fingerprint, nickname, orport_v6) self.has_extrainfo = has_extrainfo - self.orport_v6 = (orport_v6[0], int(orport_v6[1])) if orport_v6 else None self.header = header if header else OrderedDict() @staticmethod @@ -556,7 +555,7 @@ class Fallback(Directory): conf.save(path) def __hash__(self): - return _hash_attr(self, 'has_extrainfo', 'orport_v6', 'header', parent = Directory) + return _hash_attr(self, 'has_extrainfo', 'header', parent = Directory) def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Fallback) else False @@ -626,6 +625,7 @@ DIRECTORY_AUTHORITIES = { or_port = 443, dir_port = 80, fingerprint = '847B1F850344D7876491A54892F904934E4EB85D', + orport_v6 = ('2001:858:2:2:aabb:0:563b:1526', 443), v3ident = '14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', ), 'dizum': Authority( @@ -642,6 +642,7 @@ DIRECTORY_AUTHORITIES = { or_port = 443, dir_port = 80, fingerprint = 'F2044413DAC2E02E3D6BCF4735A19BCA1DE97281', + orport_v6 = ('2001:638:a000:4140::ffff:189', 443), v3ident = 'ED03BB616EB2F60BEC80151114BB25CEF515B226', ), 'dannenberg': Authority( @@ -658,6 +659,7 @@ DIRECTORY_AUTHORITIES = { or_port = 80, dir_port = 443, fingerprint = 'BD6A829255CB08E66FBE7D3748363586E46B3810', + orport_v6 = ('2001:67c:289c::9', 80), v3ident = '49015F787433103580E3B66A1707A00E60F2D15B', ), 'Faravahar': Authority( @@ -682,6 +684,7 @@ DIRECTORY_AUTHORITIES = { or_port = 443, dir_port = 80, fingerprint = '24E2F139121D4394C54B5BCC368B3B411857C413', + orport_v6 = ('2620:13:4000:6000::1000:118', 443), v3ident = '27102BC123E7AF1D4741AE047E160C91ADC76B21', ), 'Bifroest': Authority( diff --git a/test/unit/directory/authority.py b/test/unit/directory/authority.py index 8fd70880..56bd6ec8 100644 --- a/test/unit/directory/authority.py +++ b/test/unit/directory/authority.py @@ -35,6 +35,7 @@ class TestAuthority(unittest.TestCase): 'dir_port': 9030, 'fingerprint': '0756B7CD4DFC8182BE23143FAC0642F515182CEB', 'nickname': 'rueckgrat', + 'orport_v6': ('2a01:4f8:162:51e2::2', 9001), 'v3ident': '23D15D965BC35114467363C165C4F724B64B4F66', } @@ -68,6 +69,7 @@ class TestAuthority(unittest.TestCase): or_port = 443, dir_port = 80, fingerprint = '847B1F850344D7876491A54892F904934E4EB85D', + orport_v6 = ('2001:858:2:2:aabb:0:563b:1526', 443), v3ident = '14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', ), }

1 0

[stem/master] Deprecate is_bandwidth_authority attribute
by atagar＠torproject.org 08 May '18

08 May '18

commit 6e40a303a96fccd2db3e0370efc3d7d4c54d82a4 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 7 10:57:39 2018 -0700 Deprecate is_bandwidth_authority attribute This is a moving target that isn't tracked in tor, so I don't think we'll track it in Stem either. Folks can download the votes to check who is/isn't a bandwidth auth. This is how it's defined (providing bandwidth measurements in votes), after all. --- .../example/votes_by_bandwidth_authorities.py | 14 ++++---- stem/directory.py | 38 +++++++++------------- test/integ/directory/authority.py | 22 +------------ test/unit/directory/authority.py | 1 - 4 files changed, 24 insertions(+), 51 deletions(-) diff --git a/docs/_static/example/votes_by_bandwidth_authorities.py b/docs/_static/example/votes_by_bandwidth_authorities.py index 9504bd94..840d50d8 100644 --- a/docs/_static/example/votes_by_bandwidth_authorities.py +++ b/docs/_static/example/votes_by_bandwidth_authorities.py @@ -7,11 +7,10 @@ queries = {} downloader = stem.descriptor.remote.DescriptorDownloader() for authority in stem.directory.Authority.from_cache().values(): - if authority.is_bandwidth_authority: - queries[authority.nickname] = downloader.query( - '/tor/status-vote/current/authority', - endpoints = [(authority.address, authority.dir_port)], - ) + queries[authority.nickname] = downloader.query( + '/tor/status-vote/current/authority', + endpoints = [(authority.address, authority.dir_port)], + ) for authority_name, query in queries.items(): try: @@ -25,6 +24,9 @@ for authority_name, query in queries.items(): else: unmeasured += 1 - print(' %i measured entries and %i unmeasured' % (measured, unmeasured)) + if measured == 0: + print(' %s is not a bandwidth authority' % authority_name) + else: + print(' %i measured entries and %i unmeasured' % (measured, unmeasured)) except Exception as exc: print(" failed to get the vote (%s)" % exc) diff --git a/stem/directory.py b/stem/directory.py index 7b78696a..c70cbecf 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -2,7 +2,7 @@ # See LICENSE for licensing information """ -Directories with `tor descriptor information +Directories that provide `relay descriptor information <../tutorials/mirror_mirror_on_the_wall.html>`_. At a very high level tor works as follows... @@ -60,7 +60,7 @@ except ImportError: GITWEB_AUTHORITY_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc' GITWEB_FALLBACK_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc' -CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_fallbacks.cfg') +FALLBACK_CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_fallbacks.cfg') AUTHORITY_NAME = re.compile('"(\S+) orport=(\d+) .*"') AUTHORITY_V3IDENT = re.compile('"v3ident=([\dA-F]{40}) "') @@ -78,14 +78,14 @@ FALLBACK_IPV6 = re.compile('" ipv6=\[([\da-f:]+)\]:(\d+)"') class Directory(object): """ - Relay we can contact for directory information. + Relay we can contact for descriptor information. Our :func:`~stem.directory.Directory.from_cache` and :func:`~stem.directory.Directory.from_remote` functions key off a different identifier based on our subclass... - * **Authority** keys off the nickname. - * **Fallback** keys off fingerprints. + * :class:`~stem.directory.Authority` keys off the nickname. + * :class:`~stem.directory.Fallback` keys off fingerprints. This is because authorities are highly static and canonically known by their names, whereas fallbacks vary more and don't necessarily have a nickname to @@ -112,8 +112,8 @@ class Directory(object): def from_cache(): """ Provides cached Tor directory information. This information is hardcoded - into Tor and occasionally changes, so the information this provides might - not necessarily match the latest version of tor. + into Tor and occasionally changes, so the information provided by this + method may not necessarily match the latest version of tor. .. versionadded:: 1.5.0 @@ -169,14 +169,16 @@ class Authority(Directory): """ Tor directory authority, a special type of relay `hardcoded into tor <https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc>`_ - that enumerates the other relays within the network. + to enumerate the relays in the network. .. versionchanged:: 1.3.0 Added the is_bandwidth_authority attribute. + .. deprecated:: 1.7.0 + The is_bandwidth_authority attribute is deprecated and will be removed in + the future. + :var str v3ident: identity key fingerprint used to sign votes and consensus - :var bool is_bandwidth_authority: **True** if this is a bandwidth authority, - **False** otherwise """ def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False): @@ -357,7 +359,7 @@ class Fallback(Directory): self.header = header if header else OrderedDict() @staticmethod - def from_cache(path = CACHE_PATH): + def from_cache(path = FALLBACK_CACHE_PATH): conf = stem.util.conf.Config() conf.load(path) headers = OrderedDict([(k.split('.', 1)[1], conf.get(k)) for k in conf.keys() if k.startswith('header.')]) @@ -375,7 +377,7 @@ class Fallback(Directory): attr[attr_name] = conf.get(key) if not attr[attr_name] and attr_name not in ('nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): - raise IOError("'%s' is missing from %s" % (key, CACHE_PATH)) + raise IOError("'%s' is missing from %s" % (key, FALLBACK_CACHE_PATH)) if not connection.is_valid_ipv4_address(attr['address']): raise IOError("'%s.address' was an invalid IPv4 address (%s)" % (fingerprint, attr['address'])) @@ -540,7 +542,7 @@ class Fallback(Directory): return section_lines @staticmethod - def _write(fallbacks, tor_commit, stem_commit, headers, path = CACHE_PATH): + def _write(fallbacks, tor_commit, stem_commit, headers, path = FALLBACK_CACHE_PATH): """ Persists fallback directories to a location in a way that can be read by from_cache(). @@ -635,7 +637,6 @@ DIRECTORY_AUTHORITIES = { address = '128.31.0.39', or_port = 9101, dir_port = 9131, - is_bandwidth_authority = True, fingerprint = '9695DFC35FFEB861329B9F1AB04C46397020CE31', v3ident = 'D586D18309DED4CD6D57C18FDB97EFA96D330566', ), @@ -644,7 +645,6 @@ DIRECTORY_AUTHORITIES = { address = '86.59.21.38', or_port = 443, dir_port = 80, - is_bandwidth_authority = False, fingerprint = '847B1F850344D7876491A54892F904934E4EB85D', v3ident = '14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', ), @@ -653,7 +653,6 @@ DIRECTORY_AUTHORITIES = { address = '194.109.206.212', or_port = 443, dir_port = 80, - is_bandwidth_authority = False, fingerprint = '7EA6EAD6FD83083C538F44038BBFA077587DD755', v3ident = 'E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58', ), @@ -662,7 +661,6 @@ DIRECTORY_AUTHORITIES = { address = '131.188.40.189', or_port = 443, dir_port = 80, - is_bandwidth_authority = True, fingerprint = 'F2044413DAC2E02E3D6BCF4735A19BCA1DE97281', v3ident = 'ED03BB616EB2F60BEC80151114BB25CEF515B226', ), @@ -671,7 +669,6 @@ DIRECTORY_AUTHORITIES = { address = '193.23.244.244', or_port = 443, dir_port = 80, - is_bandwidth_authority = False, fingerprint = '7BE683E65D48141321C5ED92F075C55364AC7123', v3ident = '0232AF901C31A04EE9848595AF9BB7620D4C5B2E', ), @@ -680,7 +677,6 @@ DIRECTORY_AUTHORITIES = { address = '171.25.193.9', or_port = 80, dir_port = 443, - is_bandwidth_authority = True, fingerprint = 'BD6A829255CB08E66FBE7D3748363586E46B3810', v3ident = '49015F787433103580E3B66A1707A00E60F2D15B', ), @@ -689,7 +685,6 @@ DIRECTORY_AUTHORITIES = { address = '154.35.175.225', or_port = 443, dir_port = 80, - is_bandwidth_authority = True, fingerprint = 'CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC', v3ident = 'EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97', ), @@ -698,7 +693,6 @@ DIRECTORY_AUTHORITIES = { address = '199.58.81.140', or_port = 443, dir_port = 80, - is_bandwidth_authority = False, fingerprint = '74A910646BCEEFBCD2E874FC1DC997430F968145', v3ident = '23D15D965BC35114467363C165C4F724B64B4F66', ), @@ -707,7 +701,6 @@ DIRECTORY_AUTHORITIES = { address = '204.13.164.118', or_port = 443, dir_port = 80, - is_bandwidth_authority = True, fingerprint = '24E2F139121D4394C54B5BCC368B3B411857C413', v3ident = '27102BC123E7AF1D4741AE047E160C91ADC76B21', ), @@ -716,7 +709,6 @@ DIRECTORY_AUTHORITIES = { address = '37.218.247.217', or_port = 443, dir_port = 80, - is_bandwidth_authority = False, fingerprint = '1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1', v3ident = None, # does not vote in the consensus ), diff --git a/test/integ/directory/authority.py b/test/integ/directory/authority.py index fc5eb13e..c188fae3 100644 --- a/test/integ/directory/authority.py +++ b/test/integ/directory/authority.py @@ -15,24 +15,4 @@ class TestAuthority(unittest.TestCase): Check if the cached authorities we bundle are up to date. """ - cached_authorities = stem.directory.Authority.from_cache() - latest_authorities = stem.directory.Authority.from_remote() - - for nickname in cached_authorities: - if nickname not in latest_authorities: - self.fail('%s is no longer a directory authority in tor' % nickname) - - for nickname in latest_authorities: - if nickname not in cached_authorities: - self.fail('%s is now a directory authority in tor' % nickname) - - # tor doesn't note if an autority is a bwauth or not, so we need to exclude - # that from our comparison - - for attr in ('address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'v3ident'): - for auth in cached_authorities.values(): - cached_value = getattr(auth, attr) - latest_value = getattr(latest_authorities[auth.nickname], attr) - - if cached_value != latest_value: - self.fail('The %s of the %s authority is %s in tor but %s in stem' % (attr, auth.nickname, latest_value, cached_value)) + self.assertEqual(stem.directory.Authority.from_cache(), stem.directory.Authority.from_remote()) diff --git a/test/unit/directory/authority.py b/test/unit/directory/authority.py index dcee4916..431cbf52 100644 --- a/test/unit/directory/authority.py +++ b/test/unit/directory/authority.py @@ -36,7 +36,6 @@ class TestAuthority(unittest.TestCase): 'fingerprint': '0756B7CD4DFC8182BE23143FAC0642F515182CEB', 'nickname': 'rueckgrat', 'v3ident': '23D15D965BC35114467363C165C4F724B64B4F66', - 'is_bandwidth_authority': False, } self.assertEqual(stem.directory.Authority(**authority_attr), stem.directory.Authority(**authority_attr))

1 0

[stem/master] Move attribute validation into Directory constructors
by atagar＠torproject.org 08 May '18

08 May '18

commit cdffc5a9c81145877907ec0135c45d24e6676ae6 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue May 8 11:31:11 2018 -0700 Move attribute validation into Directory constructors Doing validation in the constructors not only makes sense, but lets us deduplicate this. --- stem/directory.py | 90 ++++++++++++++++------------------------ test/unit/directory/authority.py | 6 +-- test/unit/directory/fallback.py | 17 ++++---- 3 files changed, 47 insertions(+), 66 deletions(-) diff --git a/stem/directory.py b/stem/directory.py index c70cbecf..acd24a80 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -102,9 +102,22 @@ class Directory(object): """ def __init__(self, address, or_port, dir_port, fingerprint, nickname): + identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint + + if not connection.is_valid_ipv4_address(address): + raise ValueError('%s has an invalid IPv4 address: %s' % (identifier, address)) + elif not connection.is_valid_port(or_port): + raise ValueError('%s has an invalid ORPort: %s' % (identifier, or_port)) + elif not connection.is_valid_port(dir_port): + raise ValueError('%s has an invalid DirPort: %s' % (identifier, dir_port)) + elif not tor_tools.is_valid_fingerprint(fingerprint): + raise ValueError('%s has an invalid fingerprint: %s' % (identifier, fingerprint)) + elif nickname and not tor_tools.is_valid_nickname(nickname): + raise ValueError('%s has an invalid nickname: %s' % (fingerprint, nickname)) + self.address = address - self.or_port = or_port - self.dir_port = dir_port + self.or_port = int(or_port) + self.dir_port = int(dir_port) self.fingerprint = fingerprint self.nickname = nickname @@ -183,6 +196,11 @@ class Authority(Directory): def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False): super(Authority, self).__init__(address, or_port, dir_port, fingerprint, nickname) + identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint + + if v3ident and not tor_tools.is_valid_fingerprint(v3ident): + raise ValueError('%s has an invalid v3ident: %s' % (identifier, v3ident)) + self.v3ident = v3ident self.is_bandwidth_authority = is_bandwidth_authority @@ -253,33 +271,14 @@ class Authority(Directory): nickname, or_port = matches.get(AUTHORITY_NAME) v3ident = matches.get(AUTHORITY_V3IDENT) - orport_v6 = matches.get(AUTHORITY_IPV6) # TODO: add this to stem's data? + # orport_v6 = matches.get(AUTHORITY_IPV6) # TODO: add this to stem's data? address, dir_port, fingerprint = matches.get(AUTHORITY_ADDR) - fingerprint = fingerprint.replace(' ', '') - - if not connection.is_valid_ipv4_address(address): - raise ValueError('%s has an invalid IPv4 address: %s' % (nickname, address)) - elif not connection.is_valid_port(or_port): - raise ValueError('%s has an invalid or_port: %s' % (nickname, or_port)) - elif not connection.is_valid_port(dir_port): - raise ValueError('%s has an invalid dir_port: %s' % (nickname, dir_port)) - elif not tor_tools.is_valid_fingerprint(fingerprint): - raise ValueError('%s has an invalid fingerprint: %s' % (nickname, fingerprint)) - elif nickname and not tor_tools.is_valid_nickname(nickname): - raise ValueError('%s has an invalid nickname: %s' % (nickname, nickname)) - elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): - raise ValueError('%s has an invalid IPv6 address: %s' % (nickname, orport_v6[0])) - elif orport_v6 and not connection.is_valid_port(orport_v6[1]): - raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (nickname, orport_v6[1])) - elif v3ident and not tor_tools.is_valid_fingerprint(v3ident): - raise ValueError('%s has an invalid v3ident: %s' % (nickname, v3ident)) - return Authority( address = address, - or_port = int(or_port), - dir_port = int(dir_port), - fingerprint = fingerprint, + or_port = or_port, + dir_port = dir_port, + fingerprint = fingerprint.replace(' ', ''), nickname = nickname, v3ident = v3ident, ) @@ -353,9 +352,18 @@ class Fallback(Directory): def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, has_extrainfo = False, orport_v6 = None, header = None): super(Fallback, self).__init__(address, or_port, dir_port, fingerprint, nickname) + identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint + + if orport_v6: + if not isinstance(orport_v6, tuple) or len(orport_v6) != 2: + raise ValueError('%s orport_v6 should be a two value tuple: %s' % (identifier, str(orport_v6))) + elif not connection.is_valid_ipv6_address(orport_v6[0]): + raise ValueError('%s has an invalid IPv6 address: %s' % (identifier, orport_v6[0])) + elif not connection.is_valid_port(orport_v6[1]): + raise ValueError('%s has an invalid IPv6 port: %s' % (identifier, orport_v6[1])) self.has_extrainfo = has_extrainfo - self.orport_v6 = orport_v6 + self.orport_v6 = (orport_v6[0], int(orport_v6[1])) if orport_v6 else None self.header = header if header else OrderedDict() @staticmethod @@ -379,19 +387,6 @@ class Fallback(Directory): if not attr[attr_name] and attr_name not in ('nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): raise IOError("'%s' is missing from %s" % (key, FALLBACK_CACHE_PATH)) - if not connection.is_valid_ipv4_address(attr['address']): - raise IOError("'%s.address' was an invalid IPv4 address (%s)" % (fingerprint, attr['address'])) - elif not connection.is_valid_port(attr['or_port']): - raise IOError("'%s.or_port' was an invalid port (%s)" % (fingerprint, attr['or_port'])) - elif not connection.is_valid_port(attr['dir_port']): - raise IOError("'%s.dir_port' was an invalid port (%s)" % (fingerprint, attr['dir_port'])) - elif attr['nickname'] and not tor_tools.is_valid_nickname(attr['nickname']): - raise IOError("'%s.nickname' was an invalid nickname (%s)" % (fingerprint, attr['nickname'])) - elif attr['orport6_address'] and not connection.is_valid_ipv6_address(attr['orport6_address']): - raise IOError("'%s.orport6_address' was an invalid IPv6 address (%s)" % (fingerprint, attr['orport6_address'])) - elif attr['orport6_port'] and not connection.is_valid_port(attr['orport6_port']): - raise IOError("'%s.orport6_port' was an invalid port (%s)" % (fingerprint, attr['orport6_port'])) - if attr['orport6_address'] and attr['orport6_port']: orport_v6 = (attr['orport6_address'], int(attr['orport6_port'])) else: @@ -496,21 +491,6 @@ class Fallback(Directory): has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1' orport_v6 = matches.get(FALLBACK_IPV6) - if not connection.is_valid_ipv4_address(address): - raise ValueError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) - elif not connection.is_valid_port(or_port): - raise ValueError('%s has an invalid or_port: %s' % (fingerprint, or_port)) - elif not connection.is_valid_port(dir_port): - raise ValueError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) - elif not tor_tools.is_valid_fingerprint(fingerprint): - raise ValueError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) - elif nickname and not tor_tools.is_valid_nickname(nickname): - raise ValueError('%s has an invalid nickname: %s' % (fingerprint, nickname)) - elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): - raise ValueError('%s has an invalid IPv6 address: %s' % (fingerprint, orport_v6[0])) - elif orport_v6 and not connection.is_valid_port(orport_v6[1]): - raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, orport_v6[1])) - return Fallback( address = address, or_port = int(or_port), @@ -518,7 +498,7 @@ class Fallback(Directory): fingerprint = fingerprint, nickname = nickname, has_extrainfo = has_extrainfo, - orport_v6 = (orport_v6[0], int(orport_v6[1])) if orport_v6 else None, + orport_v6 = (orport_v6[0], orport_v6[1]) if orport_v6 else None, ) @staticmethod diff --git a/test/unit/directory/authority.py b/test/unit/directory/authority.py index 431cbf52..8fd70880 100644 --- a/test/unit/directory/authority.py +++ b/test/unit/directory/authority.py @@ -42,9 +42,9 @@ class TestAuthority(unittest.TestCase): for attr in authority_attr: for value in (None, 'something else'): - second_authority = dict(authority_attr) - second_authority[attr] = value - self.assertNotEqual(stem.directory.Authority(**authority_attr), stem.directory.Authority(**second_authority)) + second_authority = stem.directory.Authority(**authority_attr) + setattr(second_authority, attr, value) + self.assertNotEqual(stem.directory.Authority(**authority_attr), second_authority) def test_from_cache(self): authorities = stem.directory.Authority.from_cache() diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py index 1cbff9a3..c2d66173 100644 --- a/test/unit/directory/fallback.py +++ b/test/unit/directory/fallback.py @@ -3,6 +3,7 @@ Unit tests for stem.directory.Fallback. """ import io +import re import tempfile import unittest @@ -91,9 +92,9 @@ class TestFallback(unittest.TestCase): for attr in fallback_attr: for value in (None, 'something else'): - second_fallback = dict(fallback_attr) - second_fallback[attr] = value - self.assertNotEqual(stem.directory.Fallback(**fallback_attr), stem.directory.Fallback(**second_fallback)) + second_fallback = stem.directory.Fallback(**fallback_attr) + setattr(second_fallback, attr, value) + self.assertNotEqual(stem.directory.Fallback(**fallback_attr), second_fallback) def test_from_cache(self): fallbacks = stem.directory.Fallback.from_cache() @@ -155,15 +156,15 @@ class TestFallback(unittest.TestCase): def test_from_str_malformed(self): test_values = { FALLBACK_ENTRY.replace(b'id=0756B7CD4DFC8182BE23143FAC0642F515182CEB', b''): 'Malformed fallback address line:', - FALLBACK_ENTRY.replace(b'5.9.110.236', b'5.9.110'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv4 address: 5.9.110', - FALLBACK_ENTRY.replace(b':9030', b':7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid dir_port: 7814713228', - FALLBACK_ENTRY.replace(b'orport=9001', b'orport=7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid or_port: 7814713228', - FALLBACK_ENTRY.replace(b'ipv6=[2a01', b'ipv6=[:::'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv6 address: ::::4f8:162:51e2::2', + FALLBACK_ENTRY.replace(b'5.9.110.236', b'5.9.110'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid IPv4 address: 5.9.110', + FALLBACK_ENTRY.replace(b':9030', b':7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid DirPort: 7814713228', + FALLBACK_ENTRY.replace(b'orport=9001', b'orport=7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid ORPort: 7814713228', + FALLBACK_ENTRY.replace(b'ipv6=[2a01', b'ipv6=[:::'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid IPv6 address: ::::4f8:162:51e2::2', FALLBACK_ENTRY.replace(b'nickname=rueckgrat', b'nickname=invalid~nickname'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid nickname: invalid~nickname', } for entry, expected in test_values.items(): - self.assertRaisesRegexp(ValueError, expected, stem.directory.Fallback._from_str, entry) + self.assertRaisesRegexp(ValueError, re.escape(expected), stem.directory.Fallback._from_str, entry) def test_persistence(self): expected = {

1 0