May 2018 - tor-commits - lists.torproject.org

[stem/master] Deduplicate more of from_remote()
by atagar＠torproject.org 08 May '18

08 May '18

commit 4e360223f7567af932b3dd49e28f176c92f78bc8 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue May 8 13:10:08 2018 -0700 Deduplicate more of from_remote() Just tidying up boilerplate so this is more readable. --- stem/directory.py | 116 ++++++++++++++++++++++++++---------------------------- 1 file changed, 56 insertions(+), 60 deletions(-) diff --git a/stem/directory.py b/stem/directory.py index 014d07a5..5d1a352d 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -108,6 +108,14 @@ def _match_with(lines, regexes, required = None): return matches +def _directory_entries(lines, pop_section_func, regexes, required = None): + next_section = pop_section_func(lines) + + while next_section: + yield _match_with(next_section, regexes, required) + next_section = pop_section_func(lines) + + class Directory(object): """ Relay we can contact for descriptor information. @@ -261,34 +269,30 @@ class Authority(Directory): if not lines: raise IOError('%s did not have any content' % GITWEB_AUTHORITY_URL) - results = {} + # Entries look like... + # + # "moria1 orport=9101 " + # "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + # "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", - while lines: - # Entries look like... - # - # "moria1 orport=9101 " - # "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " - # "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", - - section = Authority._pop_section(lines) - - if section: - try: - matches = _match_with(section, (AUTHORITY_NAME, AUTHORITY_V3IDENT, AUTHORITY_IPV6, AUTHORITY_ADDR), required = (AUTHORITY_NAME, AUTHORITY_ADDR)) - nickname, or_port = matches.get(AUTHORITY_NAME) - address, dir_port, fingerprint = matches.get(AUTHORITY_ADDR) - - results[nickname] = Authority( - address = address, - or_port = or_port, - dir_port = dir_port, - fingerprint = fingerprint.replace(' ', ''), - nickname = nickname, - orport_v6 = matches.get(AUTHORITY_IPV6), - v3ident = matches.get(AUTHORITY_V3IDENT), - ) - except ValueError as exc: - raise IOError(str(exc)) + try: + results = {} + + for matches in _directory_entries(lines, Authority._pop_section, (AUTHORITY_NAME, AUTHORITY_V3IDENT, AUTHORITY_IPV6, AUTHORITY_ADDR), required = (AUTHORITY_NAME, AUTHORITY_ADDR)): + nickname, or_port = matches.get(AUTHORITY_NAME) + address, dir_port, fingerprint = matches.get(AUTHORITY_ADDR) + + results[nickname] = Authority( + address = address, + or_port = or_port, + dir_port = dir_port, + fingerprint = fingerprint.replace(' ', ''), + nickname = nickname, + orport_v6 = matches.get(AUTHORITY_IPV6), + v3ident = matches.get(AUTHORITY_V3IDENT), + ) + except ValueError as exc: + raise IOError(str(exc)) return results @@ -425,41 +429,33 @@ class Fallback(Directory): else: raise IOError('Malformed fallback directory header line: %s' % line) - # human readable comments + Fallback._pop_section(lines) # skip human readable comments - Fallback._pop_section(lines) + # Entries look like... + # + # "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" + # " ipv6=[2a01:4f8:162:51e2::2]:9001" + # /* nickname=rueckgrat */ + # /* extrainfo=1 */ - # content, everything remaining are fallback directories - - results = {} - - while lines: - # Entries look like... - # - # "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" - # " ipv6=[2a01:4f8:162:51e2::2]:9001" - # /* nickname=rueckgrat */ - # /* extrainfo=1 */ - - section = Fallback._pop_section(lines) - - if section: - try: - matches = _match_with(section, (FALLBACK_ADDR, FALLBACK_NICKNAME, FALLBACK_EXTRAINFO, FALLBACK_IPV6), required = (FALLBACK_ADDR,)) - address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR] - - results[fingerprint] = Fallback( - address = address, - or_port = int(or_port), - dir_port = int(dir_port), - fingerprint = fingerprint, - nickname = matches.get(FALLBACK_NICKNAME), - has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1', - orport_v6 = matches.get(FALLBACK_IPV6), - header = header, - ) - except ValueError as exc: - raise IOError(str(exc)) + try: + results = {} + + for matches in _directory_entries(lines, Fallback._pop_section, (FALLBACK_ADDR, FALLBACK_NICKNAME, FALLBACK_EXTRAINFO, FALLBACK_IPV6), required = (FALLBACK_ADDR,)): + address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR] + + results[fingerprint] = Fallback( + address = address, + or_port = int(or_port), + dir_port = int(dir_port), + fingerprint = fingerprint, + nickname = matches.get(FALLBACK_NICKNAME), + has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1', + orport_v6 = matches.get(FALLBACK_IPV6), + header = header, + ) + except ValueError as exc: + raise IOError(str(exc)) return results

1 0

[stem/master] Add stem.directory module
by atagar＠torproject.org 08 May '18

08 May '18

commit 65a5097ff063bd163c4e1e6ce92b4caf4090ec82 Merge: ffb3ade8 4e360223 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue May 8 13:12:02 2018 -0700 Add stem.directory module When dirauths were a small constant it made sense to co-locate them in the remote module (the sole spot they were used). But they've grown. With the addition of fallback directories and from_remote() parsers they're now the bulk of the remote module and way past the point where they deserve their own module. Adding a stem.directory module with aliases for backward compatability. Also shortening the names. With this the module is a *lot* nicer to use. For instance to list the authorities we're going from... for authority in stem.descriptor.remote.DirectoryAuthority.from_cache(): print(authority.nickname) ... to... for authority in stem.directory.Authority.from_cache(): print(authority.nickname) This branch also makes a few additions... * Authority class now has the from_cache() and from_remote() function. * Added the orport_v6 attribute to Authorities and removed is_bandwidth_authority. * Expanded test coverage. docs/_static/example/compare_flags.py | 15 +- .../example/votes_by_bandwidth_authorities.py | 21 +- docs/api/directory.rst | 5 + docs/change_log.rst | 9 +- setup.py | 3 +- stem/__init__.py | 1 + ...llback_directories.cfg => cached_fallbacks.cfg} | 0 ...ched_tor_manual.sqlite => cached_manual.sqlite} | Bin stem/descriptor/remote.py | 846 ++++----------------- stem/directory.py | 657 ++++++++++++++++ stem/manual.py | 2 +- stem/prereq.py | 51 +- stem/util/__init__.py | 8 + test/integ/descriptor/remote.py | 53 +- test/integ/directory/__init__.py | 8 + test/integ/directory/authority.py | 18 + test/integ/directory/fallback.py | 54 ++ test/settings.cfg | 5 + test/unit/__init__.py | 1 + test/unit/descriptor/remote.py | 192 +---- test/unit/directory/__init__.py | 8 + test/unit/directory/authority.py | 81 ++ test/unit/directory/fallback.py | 200 +++++ test/unit/installation.py | 3 +- test/unit/tutorial_examples.py | 14 +- 25 files changed, 1276 insertions(+), 979 deletions(-)

1 0

[stem/master] Replace sys.exc_info() calls
by atagar＠torproject.org 08 May '18

08 May '18

commit 8d2440c859ff58a1250abfbc8ce932c8ee75b04b Author: Damian Johnson <atagar(a)torproject.org> Date: Tue May 8 12:48:48 2018 -0700 Replace sys.exc_info() calls Huh. Not sure why we did this. Probably overzealous copy-paste. --- stem/directory.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/stem/directory.py b/stem/directory.py index e360ead6..cab0f30c 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -40,7 +40,6 @@ as follows... import os import re -import sys import stem.util.conf @@ -259,8 +258,7 @@ class Authority(Directory): def from_remote(timeout = 60): try: lines = str_tools._to_unicode(urllib.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() - except: - exc = sys.exc_info()[1] + except Exception as exc: raise IOError("Unable to download tor's directory authorities from %s: %s" % (GITWEB_AUTHORITY_URL, exc)) if not lines: @@ -410,8 +408,7 @@ class Fallback(Directory): def from_remote(timeout = 60): try: lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() - except: - exc = sys.exc_info()[1] + except Exception as exc: raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc)) if not lines:

1 0

[stem/master] Example of v3 hidden services in create_ephemeral_hidden_service docs
by atagar＠torproject.org 08 May '18

08 May '18

commit 07bf3c00a26a77a14a1ba6202c5d00b2927e6161 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 6 11:45:44 2018 -0700 Example of v3 hidden services in create_ephemeral_hidden_service docs Our 'key_type / key_content' distinction is confusing. I'm just surfacing tor's fields and I see what they're going for, but none the less as a user this field overloading is weird. Providing an example for how to create a new v3 hidden service. --- stem/control.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/stem/control.py b/stem/control.py index 83304cea..fd9aa8dc 100644 --- a/stem/control.py +++ b/stem/control.py @@ -2874,9 +2874,19 @@ class Controller(BaseController): }) To create a **version 3** service simply specify **ED25519-V3** as the - key_type, and to create a **version 2** service use **RSA1024**. The + our key type, and to create a **version 2** service use **RSA1024**. The default version of newly created hidden services is based on the - **HiddenServiceVersion** value in your torrc. + **HiddenServiceVersion** value in your torrc... + + :: + + response = controller.create_ephemeral_hidden_service( + 80, + key_content = 'ED25519-V3', + await_publication = True, + ) + + print('service established at %s.onion' % response.service_id) .. versionadded:: 1.4.0

1 0

[stem/master] Drop _from_str() helpers
by atagar＠torproject.org 08 May '18

08 May '18

commit 6240c44e89867de2ea436a614a5458ff31ffb3d9 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue May 8 12:46:01 2018 -0700 Drop _from_str() helpers Our two Directory subclasses had almost identical _from_str() helper. Replacing them with a single more generic _match_with() helper. --- stem/directory.py | 183 +++++++++++++++++----------------------- test/unit/directory/fallback.py | 37 ++------ 2 files changed, 85 insertions(+), 135 deletions(-) diff --git a/stem/directory.py b/stem/directory.py index b638a354..e360ead6 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -76,6 +76,42 @@ FALLBACK_EXTRAINFO = re.compile('/\* extrainfo=([0-1]) \*/') FALLBACK_IPV6 = re.compile('" ipv6=\[([\da-f:]+)\]:(\d+)"') +def _match_with(content, regexes, required = None): + """ + Scans the given content against a series of regex matchers, providing back a + mapping of regexes to their capture groups. This maping is with the value if + the regex has just a single capture group, and a tuple otherwise. + + :param str content: text to parse + :param list regexes: regexes to match against + :param list required: matches that must be in the content + + :returns: **dict** mapping matchers against their capture groups + + :raises: **ValueError** if a required match is not present + """ + + if isinstance(content, bytes): + content = str_tools._to_unicode(content) + + matches = {} + + for line in content.splitlines(): + for matcher in regexes: + m = matcher.search(line) + + if m: + match_groups = m.groups() + matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] + + if required: + for required_matcher in required: + if required_matcher not in matches: + raise ValueError('Failed to parse mandatory data from:\n\n%s' % content) + + return matches + + class Directory(object): """ Relay we can contact for descriptor information. @@ -207,9 +243,9 @@ class Authority(Directory): def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, orport_v6 = None, v3ident = None, is_bandwidth_authority = False): super(Authority, self).__init__(address, or_port, dir_port, fingerprint, nickname, orport_v6) - identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint if v3ident and not tor_tools.is_valid_fingerprint(v3ident): + identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint raise ValueError('%s has an invalid v3ident: %s' % (identifier, v3ident)) self.v3ident = v3ident @@ -233,69 +269,35 @@ class Authority(Directory): results = {} while lines: + # Entries look like... + # + # "moria1 orport=9101 " + # "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + # "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + section = Authority._pop_section(lines) if section: try: - authority = Authority._from_str('\n'.join(section)) - results[authority.nickname] = authority + matches = _match_with('\n'.join(section), (AUTHORITY_NAME, AUTHORITY_V3IDENT, AUTHORITY_IPV6, AUTHORITY_ADDR), required = (AUTHORITY_NAME, AUTHORITY_ADDR)) + nickname, or_port = matches.get(AUTHORITY_NAME) + address, dir_port, fingerprint = matches.get(AUTHORITY_ADDR) + + results[nickname] = Authority( + address = address, + or_port = or_port, + dir_port = dir_port, + fingerprint = fingerprint.replace(' ', ''), + nickname = nickname, + orport_v6 = matches.get(AUTHORITY_IPV6), + v3ident = matches.get(AUTHORITY_V3IDENT), + ) except ValueError as exc: raise IOError(str(exc)) return results @staticmethod - def _from_str(content): - """ - Parses authority from its textual representation. For example... - - :: - - "moria1 orport=9101 " - "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " - "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", - - :param str content: text to parse - - :returns: :class:`~stem.directory.Authority` in the text - - :raises: **ValueError** if content is malformed - """ - - if isinstance(content, bytes): - content = str_tools._to_unicode(content) - - matches = {} - - for line in content.splitlines(): - for matcher in (AUTHORITY_NAME, AUTHORITY_V3IDENT, AUTHORITY_IPV6, AUTHORITY_ADDR): - m = matcher.match(line.strip()) - - if m: - match_groups = m.groups() - matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] - - if AUTHORITY_NAME not in matches: - raise ValueError('Unable to parse the name and orport from:\n\n%s' % content) - elif AUTHORITY_ADDR not in matches: - raise ValueError('Unable to parse the address and fingerprint from:\n\n%s' % content) - - nickname, or_port = matches.get(AUTHORITY_NAME) - v3ident = matches.get(AUTHORITY_V3IDENT) - orport_v6 = matches.get(AUTHORITY_IPV6) - address, dir_port, fingerprint = matches.get(AUTHORITY_ADDR) - - return Authority( - address = address, - or_port = or_port, - dir_port = dir_port, - fingerprint = fingerprint.replace(' ', ''), - nickname = nickname, - orport_v6 = orport_v6, - v3ident = v3ident, - ) - - @staticmethod def _pop_section(lines): """ Provides the next authority entry. @@ -438,69 +440,36 @@ class Fallback(Directory): results = {} while lines: + # Entries look like... + # + # "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" + # " ipv6=[2a01:4f8:162:51e2::2]:9001" + # /* nickname=rueckgrat */ + # /* extrainfo=1 */ + section = Fallback._pop_section(lines) if section: try: - fallback = Fallback._from_str('\n'.join(section)) - fallback.header = header - results[fallback.fingerprint] = fallback + matches = _match_with('\n'.join(section), (FALLBACK_ADDR, FALLBACK_NICKNAME, FALLBACK_EXTRAINFO, FALLBACK_IPV6), required = (FALLBACK_ADDR,)) + address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR] + + results[fingerprint] = Fallback( + address = address, + or_port = int(or_port), + dir_port = int(dir_port), + fingerprint = fingerprint, + nickname = matches.get(FALLBACK_NICKNAME), + has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1', + orport_v6 = matches.get(FALLBACK_IPV6), + header = header, + ) except ValueError as exc: raise IOError(str(exc)) return results @staticmethod - def _from_str(content): - """ - Parses a fallback from its textual representation. For example... - - :: - - "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" - " ipv6=[2a01:4f8:162:51e2::2]:9001" - /* nickname=rueckgrat */ - /* extrainfo=1 */ - - :param str content: text to parse - - :returns: :class:`~stem.directory.Fallback` in the text - - :raises: **ValueError** if content is malformed - """ - - if isinstance(content, bytes): - content = str_tools._to_unicode(content) - - matches = {} - - for line in content.splitlines(): - for matcher in (FALLBACK_ADDR, FALLBACK_NICKNAME, FALLBACK_EXTRAINFO, FALLBACK_IPV6): - m = matcher.match(line) - - if m: - match_groups = m.groups() - matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] - - if FALLBACK_ADDR not in matches: - raise ValueError('Malformed fallback address line:\n\n%s' % content) - - address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR] - nickname = matches.get(FALLBACK_NICKNAME) - has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1' - orport_v6 = matches.get(FALLBACK_IPV6) - - return Fallback( - address = address, - or_port = int(or_port), - dir_port = int(dir_port), - fingerprint = fingerprint, - nickname = nickname, - has_extrainfo = has_extrainfo, - orport_v6 = (orport_v6[0], orport_v6[1]) if orport_v6 else None, - ) - - @staticmethod def _pop_section(lines): """ Provides lines up through the next divider. This excludes lines with just a diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py index c2d66173..c523013d 100644 --- a/test/unit/directory/fallback.py +++ b/test/unit/directory/fallback.py @@ -57,13 +57,6 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=30-&flag=V2Dir&type=rela /* ===== */ """ -FALLBACK_ENTRY = b"""\ -"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" -" ipv6=[2a01:4f8:162:51e2::2]:9001" -/* nickname=rueckgrat */ -/* extrainfo=1 */ -""" - HEADER = OrderedDict(( ('type', 'fallback'), ('version', '2.0.0'), @@ -140,31 +133,19 @@ class TestFallback(unittest.TestCase): def test_from_remote_malformed_header(self): self.assertRaisesRegexp(IOError, 'Malformed fallback directory header line: /\* version \*/', stem.directory.Fallback.from_remote) - def test_from_str(self): - expected = stem.directory.Fallback( - address = '5.9.110.236', - or_port = 9001, - dir_port = 9030, - fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', - nickname = 'rueckgrat', - has_extrainfo = True, - orport_v6 = ('2a01:4f8:162:51e2::2', 9001), - ) - - self.assertEqual(expected, stem.directory.Fallback._from_str(FALLBACK_ENTRY)) - - def test_from_str_malformed(self): + def test_from_remote_malformed(self): test_values = { - FALLBACK_ENTRY.replace(b'id=0756B7CD4DFC8182BE23143FAC0642F515182CEB', b''): 'Malformed fallback address line:', - FALLBACK_ENTRY.replace(b'5.9.110.236', b'5.9.110'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid IPv4 address: 5.9.110', - FALLBACK_ENTRY.replace(b':9030', b':7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid DirPort: 7814713228', - FALLBACK_ENTRY.replace(b'orport=9001', b'orport=7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid ORPort: 7814713228', - FALLBACK_ENTRY.replace(b'ipv6=[2a01', b'ipv6=[:::'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid IPv6 address: ::::4f8:162:51e2::2', - FALLBACK_ENTRY.replace(b'nickname=rueckgrat', b'nickname=invalid~nickname'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid nickname: invalid~nickname', + FALLBACK_GITWEB_CONTENT.replace(b'id=0756B7CD4DFC8182BE23143FAC0642F515182CEB', b''): 'Failed to parse mandatory data from:', + FALLBACK_GITWEB_CONTENT.replace(b'5.9.110.236', b'5.9.110'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid IPv4 address: 5.9.110', + FALLBACK_GITWEB_CONTENT.replace(b':9030', b':7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid DirPort: 7814713228', + FALLBACK_GITWEB_CONTENT.replace(b'orport=9001', b'orport=7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid ORPort: 7814713228', + FALLBACK_GITWEB_CONTENT.replace(b'ipv6=[2a01', b'ipv6=[:::'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB (rueckgrat) has an invalid IPv6 address: ::::4f8:162:51e2::2', + FALLBACK_GITWEB_CONTENT.replace(b'nickname=rueckgrat', b'nickname=invalid~nickname'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid nickname: invalid~nickname', } for entry, expected in test_values.items(): - self.assertRaisesRegexp(ValueError, re.escape(expected), stem.directory.Fallback._from_str, entry) + with patch(URL_OPEN, Mock(return_value = io.BytesIO(entry))): + self.assertRaisesRegexp(IOError, re.escape(expected), stem.directory.Fallback.from_remote) def test_persistence(self): expected = {

1 0

[stem/master] Move zstd and lzma checks to prereq module
by atagar＠torproject.org 08 May '18

08 May '18

commit bb260c0ab9bd7d8fad08657b21a36a186e7a5fe0 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 13:41:36 2018 -0700 Move zstd and lzma checks to prereq module This is the spot where we do these prereq checks. Yay, remote module's finally under the 1k line threshold. --- stem/descriptor/remote.py | 38 +++++-------------------------- stem/prereq.py | 51 ++++++++++++++++++++++++++++++++++++++++-- test/settings.cfg | 1 + test/unit/descriptor/remote.py | 12 +++++----- 4 files changed, 62 insertions(+), 40 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 301e89d9..1b6a7ea6 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -113,29 +113,6 @@ try: except ImportError: import urllib2 as urllib -try: - # added in python 3.3 - import lzma - LZMA_SUPPORTED = True -except ImportError: - LZMA_SUPPORTED = False - -try: - # We use the suggested python zstd library... - # - # https://pypi.python.org/pypi/zstandard - # - # Unfortunately this installs as a zstd module which can be confused with... - # - # https://pypi.python.org/pypi/zstd - # - # As such checking for the specific decompression class we'll need. - - import zstd - ZSTD_SUPPORTED = hasattr(zstd, 'ZstdDecompressor') -except ImportError: - ZSTD_SUPPORTED = False - Compression = stem.util.enum.Enum( ('PLAINTEXT', 'identity'), ('GZIP', 'gzip'), # can also be 'deflate' @@ -143,9 +120,6 @@ Compression = stem.util.enum.Enum( ('LZMA', 'x-tor-lzma'), ) -ZSTD_UNAVAILABLE_MSG = 'ZSTD compression requires the zstandard module (https://pypi.python.org/pypi/zstandard)' -LZMA_UNAVAILABLE_MSG = 'LZMA compression requires the lzma module (https://docs.python.org/3/library/lzma.html)' - # Tor has a limited number of descriptors we can fetch explicitly by their # fingerprint or hashes due to a limit on the url length by squid proxies. @@ -340,9 +314,10 @@ def _decompress(data, encoding): elif encoding in (Compression.GZIP, 'deflate'): return zlib.decompress(data, zlib.MAX_WBITS | 32).strip() elif encoding == Compression.ZSTD: - if not ZSTD_SUPPORTED: + if not stem.prereq.is_zstd_available(): raise ImportError('Decompressing zstd data requires https://pypi.python.org/pypi/zstandard') + import zstd output_buffer = io.BytesIO() with zstd.ZstdDecompressor().write_to(output_buffer) as decompressor: @@ -350,9 +325,10 @@ def _decompress(data, encoding): return output_buffer.getvalue().strip() elif encoding == Compression.LZMA: - if not LZMA_SUPPORTED: + if not stem.prereq.is_lzma_available(): raise ImportError('Decompressing lzma data requires https://docs.python.org/3/library/lzma.html') + import lzma return lzma.decompress(data).strip() else: raise ValueError("'%s' isn't a recognized type of encoding" % encoding) @@ -528,12 +504,10 @@ class Query(object): if isinstance(compression, str): compression = [compression] # caller provided only a single option - if Compression.ZSTD in compression and not ZSTD_SUPPORTED: - log.log_once('stem.descriptor.remote.zstd_unavailable', log.INFO, ZSTD_UNAVAILABLE_MSG) + if Compression.ZSTD in compression and not stem.prereq.is_zstd_available(): compression.remove(Compression.ZSTD) - if Compression.LZMA in compression and not LZMA_SUPPORTED: - log.log_once('stem.descriptor.remote.lzma_unavailable', log.INFO, LZMA_UNAVAILABLE_MSG) + if Compression.LZMA in compression and not stem.prereq.is_lzma_available(): compression.remove(Compression.LZMA) if not compression: diff --git a/stem/prereq.py b/stem/prereq.py index 16437c23..6e230007 100644 --- a/stem/prereq.py +++ b/stem/prereq.py @@ -16,6 +16,8 @@ Checks for stem dependencies. We require python 2.6 or greater (including the is_python_3 - checks if python 3.0 or later is available is_sqlite_available - checks if the sqlite3 module is available is_crypto_available - checks if the cryptography module is available + is_zstd_available - checks if the zstd module is available + is_lzma_available - checks if the lzma module is available is_mock_available - checks if the mock module is available """ @@ -29,6 +31,8 @@ except ImportError: from stem.util.lru_cache import lru_cache CRYPTO_UNAVAILABLE = "Unable to import the cryptography module. Because of this we'll be unable to verify descriptor signature integrity. You can get cryptography from: https://pypi.python.org/pypi/cryptography" +ZSTD_UNAVAILABLE = 'ZSTD compression requires the zstandard module (https://pypi.python.org/pypi/zstandard)' +LZMA_UNAVAILABLE = 'LZMA compression requires the lzma module (https://docs.python.org/3/library/lzma.html)' PYNACL_UNAVAILABLE = "Unable to import the pynacl module. Because of this we'll be unable to verify descriptor ed25519 certificate integrity. You can get pynacl from https://pypi.python.org/pypi/PyNaCl/" @@ -113,8 +117,6 @@ def is_crypto_available(): otherwise """ - from stem.util import log - try: from cryptography.utils import int_from_bytes, int_to_bytes from cryptography.hazmat.backends import default_backend @@ -127,11 +129,56 @@ def is_crypto_available(): return True except ImportError: + from stem.util import log log.log_once('stem.prereq.is_crypto_available', log.INFO, CRYPTO_UNAVAILABLE) return False @lru_cache() +def is_zstd_available(): + """ + Checks if the `zstd module <https://pypi.python.org/pypi/zstandard>`_ is + available. + + .. versionadded:: 1.7.0 + + :returns: **True** if we can use the zstd module and **False** otherwise + """ + + try: + # Unfortunately the zstandard module uses the same namespace as another + # zstd module (https://pypi.python.org/pypi/zstd) so we need to + # differentiate them. + + import zstd + return hasattr(zstd, 'ZstdDecompressor') + except ImportError: + from stem.util import log + log.log_once('stem.prereq.is_zstd_available', log.INFO, ZSTD_UNAVAILABLE) + return False + + +@lru_cache() +def is_lzma_available(): + """ + Checks if the `lzma module <https://docs.python.org/3/library/lzma.html>`_ is + available. This was added as a builtin in Python 3.3. + + .. versionadded:: 1.7.0 + + :returns: **True** if we can use the lzma module and **False** otherwise + """ + + try: + import lzma + return True + except ImportError: + from stem.util import log + log.log_once('stem.prereq.is_lzma_available', log.INFO, LZMA_UNAVAILABLE) + return False + + +@lru_cache() def is_mock_available(): """ Checks if the mock module is available. In python 3.3 and up it is a builtin diff --git a/test/settings.cfg b/test/settings.cfg index f1cbd381..60224463 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -171,6 +171,7 @@ pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.serialization. pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.modes' imported but unused pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.Cipher' imported but unused pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.algorithms' imported but unused +pyflakes.ignore stem/prereq.py => 'lzma' imported but unused pyflakes.ignore stem/prereq.py => 'nacl.encoding' imported but unused pyflakes.ignore stem/prereq.py => 'nacl.signing' imported but unused pyflakes.ignore stem/interpreter/__init__.py => undefined name 'raw_input' diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index a02a1171..fd831e1e 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -207,20 +207,20 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual(TEST_RESOURCE, query.resource) def test_zstd_support_check(self): - with patch('stem.descriptor.remote.ZSTD_SUPPORTED', True): + with patch('stem.prereq.is_zstd_available', Mock(return_value = True)): query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.ZSTD, start = False) self.assertEqual([Compression.ZSTD], query.compression) - with patch('stem.descriptor.remote.ZSTD_SUPPORTED', False): + with patch('stem.prereq.is_zstd_available', Mock(return_value = False)): query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.ZSTD, start = False) self.assertEqual([Compression.PLAINTEXT], query.compression) def test_lzma_support_check(self): - with patch('stem.descriptor.remote.LZMA_SUPPORTED', True): + with patch('stem.prereq.is_lzma_available', Mock(return_value = True)): query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.LZMA, start = False) self.assertEqual([Compression.LZMA], query.compression) - with patch('stem.descriptor.remote.LZMA_SUPPORTED', False): + with patch('stem.prereq.is_lzma_available', Mock(return_value = False)): query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.LZMA, start = False) self.assertEqual([Compression.PLAINTEXT], query.compression) @@ -260,7 +260,7 @@ class TestDescriptorDownloader(unittest.TestCase): Download a zstd compressed descriptor. """ - if not stem.descriptor.remote.ZSTD_SUPPORTED: + if not stem.prereq.is_zstd_available(): self.skipTest('(requires zstd module)') return @@ -279,7 +279,7 @@ class TestDescriptorDownloader(unittest.TestCase): Download a lzma compressed descriptor. """ - if not stem.descriptor.remote.LZMA_SUPPORTED: + if not stem.prereq.is_lzma_available(): self.skipTest('(requires lzma module)') return

1 0

[stem/master] Replace get_authorities() with DirectoryAuthority.from_cache()
by atagar＠torproject.org 08 May '18

08 May '18

commit fa26e4aad86b5d52a0cb4002afd302cfccebf768 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri May 4 12:48:35 2018 -0700 Replace get_authorities() with DirectoryAuthority.from_cache() Replacing our function for vending directory authority data. Simplifies our API by making us more consistant across our Directory subclasses. --- docs/change_log.rst | 1 + stem/descriptor/remote.py | 143 +++++++++++++++++++++++----------------------- 2 files changed, 71 insertions(+), 73 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index d10597ea..d5505821 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -57,6 +57,7 @@ The following are only available within Stem's `git repository * `stem.descriptor.remote <api/descriptor/remote.html>`_ can now download from relay ORPorts * Zstd and lzma compression support (:spec:`1cb56af`) + * Added :func:`~stem.descriptor.remote.Directory.from_cache` and :func:`~stem.descriptor.remote.Directory.from_remote` to the :class:`~stem.descriptor.remote.DirectoryAuthority` subclass. * `Fallback directory v2 support <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_, which adds *nickname* and *extrainfo* * Added server descriptor's new is_hidden_service_dir attribute * Don't retry downloading descriptors when we've timed out diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index a656e354..e0b4c21d 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -50,13 +50,12 @@ content. For example... |- get_extrainfo_descriptors - provides present extrainfo descriptors +- get_consensus - provides the present consensus or router status entries - get_authorities - Provides tor directory information. - Directory - Relay we can retrieve directory information from + | |- from_cache - Provides fallback directories cached with Stem. + | +- from_remote - Retrieves fallback directories remotely from tor's latest commit. + | |- DirectoryAuthority - Information about a tor directory authority +- FallbackDirectory - Directory mirror tor uses when authories are unavailable - |- from_cache - Provides fallback directories cached with Stem. - +- from_remote - Retrieves fallback directories remotely from tor's latest commit. Query - Asynchronous request to download tor descriptors |- start - issues the query if it isn't already running @@ -1022,9 +1021,18 @@ class DescriptorDownloader(object): class Directory(object): """ - Relay we can contact for directory information + Relay we can contact for directory information. - .. versionadded:: 1.5.0 + Our :func:`~stem.descriptor.remote.Directory.from_cache` and + :func:`~stem.descriptor.remote.Directory.from_remote` functions key off a + different identifier based on our subclass... + + * **DirectoryAuthority** keys off the nickname. + * **FallbackDirectory** keys off fingerprints. + + This is because authorities are highly static and canonically known by their + names, whereas fallbacks vary more and don't necessarily have a nickname to + key off of. .. versionchanged:: 1.3.0 Moved nickname from subclasses to this base class. @@ -1043,6 +1051,53 @@ class Directory(object): self.fingerprint = fingerprint self.nickname = nickname + @staticmethod + def from_cache(): + """ + Provides cached Tor directory information. This information is hardcoded + into Tor and occasionally changes, so the information this provides might + not necessarily match your version of tor. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 1.7.0 + Support added to the :class:`~stem.descriptor.remote.DirectoryAuthority` class. + + :returns: **dict** of **str** identifiers to + :class:`~stem.descriptor.remote.Directory` instances + """ + + raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') + + @staticmethod + def from_remote(timeout = 60): + """ + Reads and parses tor's directory data `from gitweb.torproject.org <https://gitweb.torproject.org/>`_. + Note that while convenient, this reliance on GitWeb means you should alway + call with a fallback, such as... + + :: + + try: + authorities = DirectoryAuthority.from_remote() + except IOError: + authorities = DirectoryAuthority.from_cache() + + .. versionadded:: 1.5.0 + + .. versionchanged:: 1.7.0 + Support added to the :class:`~stem.descriptor.remote.DirectoryAuthority` class. + + :param int timeout: seconds to wait before timing out the request + + :returns: **dict** of **str** identifiers to their + :class:`~stem.descriptor.remote.Directory` + + :raises: **IOError** if unable to retrieve the fallback directories + """ + + raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') + def __hash__(self): return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint') @@ -1056,7 +1111,7 @@ class Directory(object): class DirectoryAuthority(Directory): """ Tor directory authority, a special type of relay `hardcoded into tor - <https://gitweb.torproject.org/tor.git/tree/src/or/config.c#n819>`_ + <https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc>`_ that enumerates the other relays within the network. At a very high level tor works as follows... @@ -1095,34 +1150,11 @@ class DirectoryAuthority(Directory): self.is_bandwidth_authority = is_bandwidth_authority @staticmethod - def from_remote(timeout = 60): - """ - Reads and parses tor's latest directory authority data `from - gitweb.torproject.org - <https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc>`_. - Note that while convenient, this reliance on GitWeb means you should alway - call with a fallback, such as... - - :: - - try: - fallback_directories = DirectoryAuthority.from_remote() - except IOError: - fallback_directories = get_authorities() - - Authorities provided through this method will not have a - **is_bandwidth_authority** value set. - - .. versionadded:: 1.7.0 - - :param int timeout: seconds to wait before timing out the request - - :returns: **dict** of **str** nicknames to their - :class:`~stem.descriptor.remote.DirectoryAuthority` - - :raises: **IOError** if unable to retrieve the directory authorities - """ + def from_cache(): + return dict(DIRECTORY_AUTHORITIES) + @staticmethod + def from_remote(timeout = 60): try: lines = str_tools._to_unicode(urllib.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() except: @@ -1341,10 +1373,13 @@ def get_authorities(): information hardcoded into Tor and occasionally changes, so the information this provides might not necessarily match your version of tor. + .. deprecated:: 1.7.0 + Use stem.descriptor.remote.DirectoryAuthority.from_cache() instead. + :returns: **dict** of **str** nicknames to :class:`~stem.descriptor.remote.DirectoryAuthority` instances """ - return dict(DIRECTORY_AUTHORITIES) + return DirectoryAuthority.from_cache() class FallbackDirectory(Directory): @@ -1398,22 +1433,6 @@ class FallbackDirectory(Directory): @staticmethod def from_cache(path = CACHE_PATH): - """ - Provides fallback directory information cached with Stem. Unlike - :func:`~stem.descriptor.remote.FallbackDirectory.from_remote` this doesn't - have any system requirements, and is faster too. Only drawback is that - these fallback directories are only as up to date as the Stem release we're - using. - - .. versionchanged:: 1.7.0 - Added the path argument. - - :param str path: cache file to load from - - :returns: **dict** of **str** fingerprints to their - :class:`~stem.descriptor.remote.FallbackDirectory` - """ - conf = stem.util.conf.Config() conf.load(path) headers = OrderedDict([(k.split('.', 1)[1], conf.get(k)) for k in conf.keys() if k.startswith('header.')]) @@ -1466,28 +1485,6 @@ class FallbackDirectory(Directory): @staticmethod def from_remote(timeout = 60): - """ - Reads and parses tor's latest fallback directories `from - gitweb.torproject.org - <https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc>`_. - Note that while convenient, this reliance on GitWeb means you should alway - call with a fallback, such as... - - :: - - try: - fallback_directories = FallbackDirectory.from_remote() - except IOError: - fallback_directories = FallbackDirectory.from_cache() - - :param int timeout: seconds to wait before timing out the request - - :returns: **dict** of **str** fingerprints to their - :class:`~stem.descriptor.remote.FallbackDirectory` - - :raises: **IOError** if unable to retrieve the fallback directories - """ - try: lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() except:

1 0

[stem/master] Move Directory into its own module
by atagar＠torproject.org 08 May '18

08 May '18

commit 609a411d05fc3288efd27d419fd478906e08bc34 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 12:57:58 2018 -0700 Move Directory into its own module Directory authorities started as a small, simple constant so it made sense to co-locate them in the remote module (the sole spot they were used). But they've grown. With the addition of fallback directories and from_remote() parsers they're now the bulk of the remote module and way past the point where they deserve their own module. Adding a stem.directory module with aliases for backward compatability. Also shortening the names. With this the module is a *lot* nicer to use. For instance to list the authorities we're going from... for authority in stem.descriptor.remote.DirectoryAuthority.from_cache(): print(authority.nickname) ... to... for authority in stem.directory.Authority.from_cache(): print(authority.nickname) Quite a few other improvements I'd like to follow this up with, but starting with just a simple move. --- docs/api/directory.rst | 5 + docs/change_log.rst | 9 +- setup.py | 3 +- stem/__init__.py | 1 + stem/descriptor/remote.py | 710 +----------------------- stem/directory.py | 725 +++++++++++++++++++++++++ stem/{descriptor => }/fallback_directories.cfg | 0 test/unit/tutorial_examples.py | 2 +- 8 files changed, 746 insertions(+), 709 deletions(-) diff --git a/docs/api/directory.rst b/docs/api/directory.rst new file mode 100644 index 00000000..befd1d59 --- /dev/null +++ b/docs/api/directory.rst @@ -0,0 +1,5 @@ +Directory +========= + +.. automodule:: stem.directory + diff --git a/docs/change_log.rst b/docs/change_log.rst index d5505821..67359a95 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -57,15 +57,16 @@ The following are only available within Stem's `git repository * `stem.descriptor.remote <api/descriptor/remote.html>`_ can now download from relay ORPorts * Zstd and lzma compression support (:spec:`1cb56af`) - * Added :func:`~stem.descriptor.remote.Directory.from_cache` and :func:`~stem.descriptor.remote.Directory.from_remote` to the :class:`~stem.descriptor.remote.DirectoryAuthority` subclass. + * Moved the Directory classes into their own `stem.directory <api/directory.html>`_ module + * Added :func:`~stem.descriptor.remote.Directory.from_cache` and :func:`~stem.descriptor.remote.Directory.from_remote` to the :class:`~stem.descriptor.remote.DirectoryAuthority` subclass * `Fallback directory v2 support <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_, which adds *nickname* and *extrainfo* * Added server descriptor's new is_hidden_service_dir attribute * Don't retry downloading descriptors when we've timed out - * Don't download from tor26 and Bifroest, which are authorities that frequently timeout. - * `stem.descriptor.remote <api/descriptor/remote.html>`_ now consistently defaults **fall_back_to_authority** to false. + * Don't download from tor26 and Bifroest, which are authorities that frequently timeout + * `stem.descriptor.remote <api/descriptor/remote.html>`_ now consistently defaults **fall_back_to_authority** to false * Added :func:`~stem.descriptor.remote.their_server_descriptor` * Added the reply_headers attribute to :class:`~stem.descriptor.remote.Query` - * Supplying a User-Agent when downloading descriptors. + * Supplying a User-Agent when downloading descriptors * Reduced maximum descriptors fetched by the remote module to match tor's new limit (:trac:`24743`) * Consensus **shared_randomness_*_reveal_count** attributes undocumented, and unavailable if retrieved before their corresponding shared_randomness_*_value attribute (:trac:`25046`) * Allow 'proto' line to have blank values (:spec:`a8455f4`) diff --git a/setup.py b/setup.py index 87a766b5..9e200bf1 100644 --- a/setup.py +++ b/setup.py @@ -106,8 +106,7 @@ try: keywords = 'tor onion controller', scripts = ['tor-prompt'], package_data = { - 'stem': ['cached_tor_manual.sqlite', 'settings.cfg'], - 'stem.descriptor': ['fallback_directories.cfg'], + 'stem': ['cached_tor_manual.sqlite', 'fallback_directories.cfg', 'settings.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg'], }, classifiers = [ diff --git a/stem/__init__.py b/stem/__init__.py index c7432d86..0104d3b2 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -492,6 +492,7 @@ __all__ = [ 'util', 'connection', 'control', + 'directory', 'exit_policy', 'prereq', 'process', diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index e0b4c21d..301e89d9 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -50,13 +50,6 @@ content. For example... |- get_extrainfo_descriptors - provides present extrainfo descriptors +- get_consensus - provides the present consensus or router status entries - Directory - Relay we can retrieve directory information from - | |- from_cache - Provides fallback directories cached with Stem. - | +- from_remote - Retrieves fallback directories remotely from tor's latest commit. - | - |- DirectoryAuthority - Information about a tor directory authority - +- FallbackDirectory - Directory mirror tor uses when authories are unavailable - Query - Asynchronous request to download tor descriptors |- start - issues the query if it isn't already running +- run - blocks until the request is finished and provides the results @@ -99,9 +92,7 @@ content. For example... """ import io -import os import random -import re import sys import threading import time @@ -110,16 +101,11 @@ import zlib import stem import stem.client import stem.descriptor +import stem.directory import stem.prereq import stem.util.enum -from stem.util import _hash_attr, connection, log, str_tools, tor_tools - -try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict +from stem.util import log, str_tools try: # account for urllib's change between python 2.x and 3.x @@ -166,23 +152,6 @@ LZMA_UNAVAILABLE_MSG = 'LZMA compression requires the lzma module (https://docs. MAX_FINGERPRINTS = 96 MAX_MICRODESCRIPTOR_HASHES = 90 -GITWEB_AUTHORITY_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc' -GITWEB_FALLBACK_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc' -CACHE_PATH = os.path.join(os.path.dirname(__file__), 'fallback_directories.cfg') - -AUTHORITY_NAME = re.compile('"(\S+) orport=(\d+) .*"') -AUTHORITY_V3IDENT = re.compile('"v3ident=([\dA-F]{40}) "') -AUTHORITY_IPV6 = re.compile('"ipv6=\[([\da-f:]+)\]:(\d+) "') -AUTHORITY_ADDR = re.compile('"([\d\.]+):(\d+) ([\dA-F ]{49})",') - -FALLBACK_DIV = '/* ===== */' -FALLBACK_MAPPING = re.compile('/\*\s+(\S+)=(\S*)\s+\*/') - -FALLBACK_ADDR = re.compile('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*') -FALLBACK_NICKNAME = re.compile('/\* nickname=(\S+) \*/') -FALLBACK_EXTRAINFO = re.compile('/\* extrainfo=([0-1]) \*/') -FALLBACK_IPV6 = re.compile('" ipv6=\[([\da-f:]+)\]:(\d+)"') - SINGLETON_DOWNLOADER = None @@ -1019,354 +988,6 @@ class DescriptorDownloader(object): return Query(resource, **args) -class Directory(object): - """ - Relay we can contact for directory information. - - Our :func:`~stem.descriptor.remote.Directory.from_cache` and - :func:`~stem.descriptor.remote.Directory.from_remote` functions key off a - different identifier based on our subclass... - - * **DirectoryAuthority** keys off the nickname. - * **FallbackDirectory** keys off fingerprints. - - This is because authorities are highly static and canonically known by their - names, whereas fallbacks vary more and don't necessarily have a nickname to - key off of. - - .. versionchanged:: 1.3.0 - Moved nickname from subclasses to this base class. - - :var str address: IPv4 address of the directory - :var int or_port: port on which the relay services relay traffic - :var int dir_port: port on which directory information is available - :var str fingerprint: relay fingerprint - :var str nickname: relay nickname - """ - - def __init__(self, address, or_port, dir_port, fingerprint, nickname): - self.address = address - self.or_port = or_port - self.dir_port = dir_port - self.fingerprint = fingerprint - self.nickname = nickname - - @staticmethod - def from_cache(): - """ - Provides cached Tor directory information. This information is hardcoded - into Tor and occasionally changes, so the information this provides might - not necessarily match your version of tor. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 1.7.0 - Support added to the :class:`~stem.descriptor.remote.DirectoryAuthority` class. - - :returns: **dict** of **str** identifiers to - :class:`~stem.descriptor.remote.Directory` instances - """ - - raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') - - @staticmethod - def from_remote(timeout = 60): - """ - Reads and parses tor's directory data `from gitweb.torproject.org <https://gitweb.torproject.org/>`_. - Note that while convenient, this reliance on GitWeb means you should alway - call with a fallback, such as... - - :: - - try: - authorities = DirectoryAuthority.from_remote() - except IOError: - authorities = DirectoryAuthority.from_cache() - - .. versionadded:: 1.5.0 - - .. versionchanged:: 1.7.0 - Support added to the :class:`~stem.descriptor.remote.DirectoryAuthority` class. - - :param int timeout: seconds to wait before timing out the request - - :returns: **dict** of **str** identifiers to their - :class:`~stem.descriptor.remote.Directory` - - :raises: **IOError** if unable to retrieve the fallback directories - """ - - raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') - - def __hash__(self): - return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint') - - def __eq__(self, other): - return hash(self) == hash(other) if isinstance(other, Directory) else False - - def __ne__(self, other): - return not self == other - - -class DirectoryAuthority(Directory): - """ - Tor directory authority, a special type of relay `hardcoded into tor - <https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc>`_ - that enumerates the other relays within the network. - - At a very high level tor works as follows... - - 1. A volunteer starts up a new tor relay, during which it sends a `server - descriptor <server_descriptor.html>`_ to each of the directory - authorities. - - 2. Each hour the directory authorities make a `vote <networkstatus.html>`_ - that says who they think the active relays are in the network and some - attributes about them. - - 3. The directory authorities send each other their votes, and compile that - into the `consensus <networkstatus.html>`_. This document is very similar - to the votes, the only difference being that the majority of the - authorities agree upon and sign this document. The idividual relay entries - in the vote or consensus is called `router status entries - <router_status_entry.html>`_. - - 4. Tor clients (people using the service) download the consensus from one of - the authorities or a mirror to determine the active relays within the - network. They in turn use this to construct their circuits and use the - network. - - .. versionchanged:: 1.3.0 - Added the is_bandwidth_authority attribute. - - :var str v3ident: identity key fingerprint used to sign votes and consensus - :var bool is_bandwidth_authority: **True** if this is a bandwidth authority, - **False** otherwise - """ - - def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False): - super(DirectoryAuthority, self).__init__(address, or_port, dir_port, fingerprint, nickname) - self.v3ident = v3ident - self.is_bandwidth_authority = is_bandwidth_authority - - @staticmethod - def from_cache(): - return dict(DIRECTORY_AUTHORITIES) - - @staticmethod - def from_remote(timeout = 60): - try: - lines = str_tools._to_unicode(urllib.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() - except: - exc = sys.exc_info()[1] - raise IOError("Unable to download tor's directory authorities from %s: %s" % (GITWEB_AUTHORITY_URL, exc)) - - if not lines: - raise IOError('%s did not have any content' % GITWEB_AUTHORITY_URL) - - results = {} - - while lines: - section = DirectoryAuthority._pop_section(lines) - - if section: - try: - authority = DirectoryAuthority._from_str('\n'.join(section)) - results[authority.nickname] = authority - except ValueError as exc: - raise IOError(str(exc)) - - return results - - @staticmethod - def _from_str(content): - """ - Parses authority from its textual representation. For example... - - :: - - "moria1 orport=9101 " - "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " - "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", - - :param str content: text to parse - - :returns: :class:`~stem.descriptor.remote.DirectoryAuthority` in the text - - :raises: **ValueError** if content is malformed - """ - - if isinstance(content, bytes): - content = str_tools._to_unicode(content) - - matches = {} - - for line in content.splitlines(): - for matcher in (AUTHORITY_NAME, AUTHORITY_V3IDENT, AUTHORITY_IPV6, AUTHORITY_ADDR): - m = matcher.match(line.strip()) - - if m: - match_groups = m.groups() - matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] - - if AUTHORITY_NAME not in matches: - raise ValueError('Unable to parse the name and orport from:\n\n%s' % content) - elif AUTHORITY_ADDR not in matches: - raise ValueError('Unable to parse the address and fingerprint from:\n\n%s' % content) - - nickname, or_port = matches.get(AUTHORITY_NAME) - v3ident = matches.get(AUTHORITY_V3IDENT) - orport_v6 = matches.get(AUTHORITY_IPV6) # TODO: add this to stem's data? - address, dir_port, fingerprint = matches.get(AUTHORITY_ADDR) - - fingerprint = fingerprint.replace(' ', '') - - if not connection.is_valid_ipv4_address(address): - raise ValueError('%s has an invalid IPv4 address: %s' % (nickname, address)) - elif not connection.is_valid_port(or_port): - raise ValueError('%s has an invalid or_port: %s' % (nickname, or_port)) - elif not connection.is_valid_port(dir_port): - raise ValueError('%s has an invalid dir_port: %s' % (nickname, dir_port)) - elif not tor_tools.is_valid_fingerprint(fingerprint): - raise ValueError('%s has an invalid fingerprint: %s' % (nickname, fingerprint)) - elif nickname and not tor_tools.is_valid_nickname(nickname): - raise ValueError('%s has an invalid nickname: %s' % (nickname, nickname)) - elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): - raise ValueError('%s has an invalid IPv6 address: %s' % (nickname, orport_v6[0])) - elif orport_v6 and not connection.is_valid_port(orport_v6[1]): - raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (nickname, orport_v6[1])) - elif v3ident and not tor_tools.is_valid_fingerprint(v3ident): - raise ValueError('%s has an invalid v3ident: %s' % (nickname, v3ident)) - - return DirectoryAuthority( - address = address, - or_port = int(or_port), - dir_port = int(dir_port), - fingerprint = fingerprint, - nickname = nickname, - v3ident = v3ident, - ) - - @staticmethod - def _pop_section(lines): - """ - Provides the next authority entry. - """ - - section_lines = [] - - if lines: - section_lines.append(lines.pop(0)) - - while lines and lines[0].startswith(' '): - section_lines.append(lines.pop(0)) - - return section_lines - - def __hash__(self): - return _hash_attr(self, 'nickname', 'v3ident', 'is_bandwidth_authority', parent = Directory) - - def __eq__(self, other): - return hash(self) == hash(other) if isinstance(other, DirectoryAuthority) else False - - def __ne__(self, other): - return not self == other - - -DIRECTORY_AUTHORITIES = { - 'moria1': DirectoryAuthority( - nickname = 'moria1', - address = '128.31.0.39', - or_port = 9101, - dir_port = 9131, - is_bandwidth_authority = True, - fingerprint = '9695DFC35FFEB861329B9F1AB04C46397020CE31', - v3ident = 'D586D18309DED4CD6D57C18FDB97EFA96D330566', - ), - 'tor26': DirectoryAuthority( - nickname = 'tor26', - address = '86.59.21.38', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = False, - fingerprint = '847B1F850344D7876491A54892F904934E4EB85D', - v3ident = '14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', - ), - 'dizum': DirectoryAuthority( - nickname = 'dizum', - address = '194.109.206.212', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = False, - fingerprint = '7EA6EAD6FD83083C538F44038BBFA077587DD755', - v3ident = 'E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58', - ), - 'gabelmoo': DirectoryAuthority( - nickname = 'gabelmoo', - address = '131.188.40.189', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = True, - fingerprint = 'F2044413DAC2E02E3D6BCF4735A19BCA1DE97281', - v3ident = 'ED03BB616EB2F60BEC80151114BB25CEF515B226', - ), - 'dannenberg': DirectoryAuthority( - nickname = 'dannenberg', - address = '193.23.244.244', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = False, - fingerprint = '7BE683E65D48141321C5ED92F075C55364AC7123', - v3ident = '0232AF901C31A04EE9848595AF9BB7620D4C5B2E', - ), - 'maatuska': DirectoryAuthority( - nickname = 'maatuska', - address = '171.25.193.9', - or_port = 80, - dir_port = 443, - is_bandwidth_authority = True, - fingerprint = 'BD6A829255CB08E66FBE7D3748363586E46B3810', - v3ident = '49015F787433103580E3B66A1707A00E60F2D15B', - ), - 'Faravahar': DirectoryAuthority( - nickname = 'Faravahar', - address = '154.35.175.225', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = True, - fingerprint = 'CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC', - v3ident = 'EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97', - ), - 'longclaw': DirectoryAuthority( - nickname = 'longclaw', - address = '199.58.81.140', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = False, - fingerprint = '74A910646BCEEFBCD2E874FC1DC997430F968145', - v3ident = '23D15D965BC35114467363C165C4F724B64B4F66', - ), - 'bastet': DirectoryAuthority( - nickname = 'bastet', - address = '204.13.164.118', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = True, - fingerprint = '24E2F139121D4394C54B5BCC368B3B411857C413', - v3ident = '27102BC123E7AF1D4741AE047E160C91ADC76B21', - ), - 'Bifroest': DirectoryAuthority( - nickname = 'Bifroest', - address = '37.218.247.217', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = False, - fingerprint = '1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1', - v3ident = None, # does not vote in the consensus - ), -} - - def get_authorities(): """ Provides cached Tor directory authority information. The directory @@ -1374,331 +995,16 @@ def get_authorities(): this provides might not necessarily match your version of tor. .. deprecated:: 1.7.0 - Use stem.descriptor.remote.DirectoryAuthority.from_cache() instead. + Use stem.directory.Authority.from_cache() instead. - :returns: **dict** of **str** nicknames to :class:`~stem.descriptor.remote.DirectoryAuthority` instances + :returns: **dict** of **str** nicknames to :class:`~stem.directory.Authority` instances """ return DirectoryAuthority.from_cache() -class FallbackDirectory(Directory): - """ - Particularly stable relays tor can instead of authorities when - bootstrapping. These relays are `hardcoded in tor - <https://gitweb.torproject.org/tor.git/tree/src/or/fallback_dirs.inc>`_. - - For example, the following checks the performance of tor's fallback directories... - - :: - - import time - from stem.descriptor.remote import DescriptorDownloader, FallbackDirectory - - downloader = DescriptorDownloader() - - for fallback_directory in FallbackDirectory.from_cache().values(): - start = time.time() - downloader.get_consensus(endpoints = [(fallback_directory.address, fallback_directory.dir_port)]).run() - print('Downloading the consensus took %0.2f from %s' % (time.time() - start, fallback_directory.fingerprint)) - - :: - - % python example.py - Downloading the consensus took 5.07 from 0AD3FA884D18F89EEA2D89C019379E0E7FD94417 - Downloading the consensus took 3.59 from C871C91489886D5E2E94C13EA1A5FDC4B6DC5204 - Downloading the consensus took 4.16 from 74A910646BCEEFBCD2E874FC1DC997430F968145 - ... - - .. versionadded:: 1.5.0 - - .. versionchanged:: 1.7.0 - Added the nickname, has_extrainfo, and header attributes which are part of - the `second version of the fallback directories - <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_. - - :var bool has_extrainfo: **True** if the relay should be able to provide - extrainfo descriptors, **False** otherwise. - :var str orport_v6: **(address, port)** tuple for the directory's IPv6 - ORPort, or **None** if it doesn't have one - :var dict header: metadata about the fallback directory file this originated from - """ - - def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, has_extrainfo = False, orport_v6 = None, header = None): - super(FallbackDirectory, self).__init__(address, or_port, dir_port, fingerprint, nickname) - - self.has_extrainfo = has_extrainfo - self.orport_v6 = orport_v6 - self.header = header if header else OrderedDict() - - @staticmethod - def from_cache(path = CACHE_PATH): - conf = stem.util.conf.Config() - conf.load(path) - headers = OrderedDict([(k.split('.', 1)[1], conf.get(k)) for k in conf.keys() if k.startswith('header.')]) - - results = {} - - for fingerprint in set([key.split('.')[0] for key in conf.keys()]): - if fingerprint in ('tor_commit', 'stem_commit', 'header'): - continue - - attr = {} - - for attr_name in ('address', 'or_port', 'dir_port', 'nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): - key = '%s.%s' % (fingerprint, attr_name) - attr[attr_name] = conf.get(key) - - if not attr[attr_name] and attr_name not in ('nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): - raise IOError("'%s' is missing from %s" % (key, CACHE_PATH)) - - if not connection.is_valid_ipv4_address(attr['address']): - raise IOError("'%s.address' was an invalid IPv4 address (%s)" % (fingerprint, attr['address'])) - elif not connection.is_valid_port(attr['or_port']): - raise IOError("'%s.or_port' was an invalid port (%s)" % (fingerprint, attr['or_port'])) - elif not connection.is_valid_port(attr['dir_port']): - raise IOError("'%s.dir_port' was an invalid port (%s)" % (fingerprint, attr['dir_port'])) - elif attr['nickname'] and not tor_tools.is_valid_nickname(attr['nickname']): - raise IOError("'%s.nickname' was an invalid nickname (%s)" % (fingerprint, attr['nickname'])) - elif attr['orport6_address'] and not connection.is_valid_ipv6_address(attr['orport6_address']): - raise IOError("'%s.orport6_address' was an invalid IPv6 address (%s)" % (fingerprint, attr['orport6_address'])) - elif attr['orport6_port'] and not connection.is_valid_port(attr['orport6_port']): - raise IOError("'%s.orport6_port' was an invalid port (%s)" % (fingerprint, attr['orport6_port'])) - - if attr['orport6_address'] and attr['orport6_port']: - orport_v6 = (attr['orport6_address'], int(attr['orport6_port'])) - else: - orport_v6 = None - - results[fingerprint] = FallbackDirectory( - address = attr['address'], - or_port = int(attr['or_port']), - dir_port = int(attr['dir_port']), - fingerprint = fingerprint, - nickname = attr['nickname'], - has_extrainfo = attr['has_extrainfo'] == 'true', - orport_v6 = orport_v6, - header = headers, - ) - - return results - - @staticmethod - def from_remote(timeout = 60): - try: - lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() - except: - exc = sys.exc_info()[1] - raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc)) - - if not lines: - raise IOError('%s did not have any content' % GITWEB_FALLBACK_URL) - elif lines[0] != '/* type=fallback */': - raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_URL) - - # header metadata - - header = {} - - for line in FallbackDirectory._pop_section(lines): - mapping = FALLBACK_MAPPING.match(line) - - if mapping: - header[mapping.group(1)] = mapping.group(2) - else: - raise IOError('Malformed fallback directory header line: %s' % line) - - # human readable comments - - FallbackDirectory._pop_section(lines) - - # content, everything remaining are fallback directories - - results = {} - - while lines: - section = FallbackDirectory._pop_section(lines) - - if section: - try: - fallback = FallbackDirectory._from_str('\n'.join(section)) - fallback.header = header - results[fallback.fingerprint] = fallback - except ValueError as exc: - raise IOError(str(exc)) - - return results - - @staticmethod - def _from_str(content): - """ - Parses a fallback from its textual representation. For example... - - :: - - "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" - " ipv6=[2a01:4f8:162:51e2::2]:9001" - /* nickname=rueckgrat */ - /* extrainfo=1 */ - - :param str content: text to parse - - :returns: :class:`~stem.descriptor.remote.FallbackDirectory` in the text - - :raises: **ValueError** if content is malformed - """ - - if isinstance(content, bytes): - content = str_tools._to_unicode(content) - - matches = {} - - for line in content.splitlines(): - for matcher in (FALLBACK_ADDR, FALLBACK_NICKNAME, FALLBACK_EXTRAINFO, FALLBACK_IPV6): - m = matcher.match(line) - - if m: - match_groups = m.groups() - matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] - - if FALLBACK_ADDR not in matches: - raise ValueError('Malformed fallback address line:\n\n%s' % content) - - address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR] - nickname = matches.get(FALLBACK_NICKNAME) - has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1' - orport_v6 = matches.get(FALLBACK_IPV6) - - if not connection.is_valid_ipv4_address(address): - raise ValueError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) - elif not connection.is_valid_port(or_port): - raise ValueError('%s has an invalid or_port: %s' % (fingerprint, or_port)) - elif not connection.is_valid_port(dir_port): - raise ValueError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) - elif not tor_tools.is_valid_fingerprint(fingerprint): - raise ValueError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) - elif nickname and not tor_tools.is_valid_nickname(nickname): - raise ValueError('%s has an invalid nickname: %s' % (fingerprint, nickname)) - elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): - raise ValueError('%s has an invalid IPv6 address: %s' % (fingerprint, orport_v6[0])) - elif orport_v6 and not connection.is_valid_port(orport_v6[1]): - raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, orport_v6[1])) - - return FallbackDirectory( - address = address, - or_port = int(or_port), - dir_port = int(dir_port), - fingerprint = fingerprint, - nickname = nickname, - has_extrainfo = has_extrainfo, - orport_v6 = (orport_v6[0], int(orport_v6[1])) if orport_v6 else None, - ) - - @staticmethod - def _pop_section(lines): - """ - Provides lines up through the next divider. This excludes lines with just a - comma since they're an artifact of these being C strings. - """ - - section_lines = [] - - if lines: - line = lines.pop(0) - - while lines and line != FALLBACK_DIV: - if line.strip() != ',': - section_lines.append(line) - - line = lines.pop(0) - - return section_lines - - @staticmethod - def _write(fallbacks, tor_commit, stem_commit, headers, path = CACHE_PATH): - """ - Persists fallback directories to a location in a way that can be read by - from_cache(). - - :param dict fallbacks: mapping of fingerprints to their fallback directory - :param str tor_commit: tor commit the fallbacks came from - :param str stem_commit: stem commit the fallbacks came from - :param dict headers: metadata about the file these came from - :param str path: location fallbacks will be persisted to - """ - - conf = stem.util.conf.Config() - conf.set('tor_commit', tor_commit) - conf.set('stem_commit', stem_commit) - - for k, v in headers.items(): - conf.set('header.%s' % k, v) - - for directory in sorted(fallbacks.values(), key = lambda x: x.fingerprint): - fingerprint = directory.fingerprint - conf.set('%s.address' % fingerprint, directory.address) - conf.set('%s.or_port' % fingerprint, str(directory.or_port)) - conf.set('%s.dir_port' % fingerprint, str(directory.dir_port)) - conf.set('%s.nickname' % fingerprint, directory.nickname) - conf.set('%s.has_extrainfo' % fingerprint, 'true' if directory.has_extrainfo else 'false') - - if directory.orport_v6: - conf.set('%s.orport6_address' % fingerprint, str(directory.orport_v6[0])) - conf.set('%s.orport6_port' % fingerprint, str(directory.orport_v6[1])) - - conf.save(path) - - def __hash__(self): - return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'has_extrainfo', 'orport_v6', 'header', parent = Directory) - - def __eq__(self, other): - return hash(self) == hash(other) if isinstance(other, FallbackDirectory) else False - - def __ne__(self, other): - return not self == other - - -def _fallback_directory_differences(previous_directories, new_directories): - """ - Provides a description of how fallback directories differ. - """ - - lines = [] - - added_fp = set(new_directories.keys()).difference(previous_directories.keys()) - removed_fp = set(previous_directories.keys()).difference(new_directories.keys()) - - for fp in added_fp: - directory = new_directories[fp] - orport_v6 = '%s:%s' % directory.orport_v6 if directory.orport_v6 else '[none]' - - lines += [ - '* Added %s as a new fallback directory:' % directory.fingerprint, - ' address: %s' % directory.address, - ' or_port: %s' % directory.or_port, - ' dir_port: %s' % directory.dir_port, - ' nickname: %s' % directory.nickname, - ' has_extrainfo: %s' % directory.has_extrainfo, - ' orport_v6: %s' % orport_v6, - '', - ] - - for fp in removed_fp: - lines.append('* Removed %s as a fallback directory' % fp) - - for fp in new_directories: - if fp in added_fp or fp in removed_fp: - continue # already discussed these - - previous_directory = previous_directories[fp] - new_directory = new_directories[fp] - - if previous_directory != new_directory: - for attr in ('address', 'or_port', 'dir_port', 'fingerprint', 'orport_v6'): - old_attr = getattr(previous_directory, attr) - new_attr = getattr(new_directory, attr) - - if old_attr != new_attr: - lines.append('* Changed the %s of %s from %s to %s' % (attr, fp, old_attr, new_attr)) +# TODO: drop aliases in stem 2.0 - return '\n'.join(lines) +Directory = stem.directory.Directory +DirectoryAuthority = stem.directory.Authority +FallbackDirectory = stem.directory.Fallback diff --git a/stem/directory.py b/stem/directory.py new file mode 100644 index 00000000..b7bc201f --- /dev/null +++ b/stem/directory.py @@ -0,0 +1,725 @@ +# Copyright 2018, Damian Johnson and The Tor Project +# See LICENSE for licensing information + +""" +Directories with Tor descriptor information. + +:: + + Directory - Relay we can retrieve directory information from + | |- from_cache - Provides fallback directories cached with Stem. + | +- from_remote - Retrieves fallback directories remotely from tor's latest commit. + | + |- Authority - Information about a tor directory authority + +- Fallback - Directory mirror tor uses when authories are unavailable + +.. versionadded:: 1.7.0 +""" + +import os +import re +import sys + +import stem.util.conf + +from stem.util import _hash_attr, connection, str_tools, tor_tools + +try: + # added in python 2.7 + from collections import OrderedDict +except ImportError: + from stem.util.ordereddict import OrderedDict + +try: + # account for urllib's change between python 2.x and 3.x + import urllib.request as urllib +except ImportError: + import urllib2 as urllib + +GITWEB_AUTHORITY_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc' +GITWEB_FALLBACK_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc' +CACHE_PATH = os.path.join(os.path.dirname(__file__), 'fallback_directories.cfg') + +AUTHORITY_NAME = re.compile('"(\S+) orport=(\d+) .*"') +AUTHORITY_V3IDENT = re.compile('"v3ident=([\dA-F]{40}) "') +AUTHORITY_IPV6 = re.compile('"ipv6=\[([\da-f:]+)\]:(\d+) "') +AUTHORITY_ADDR = re.compile('"([\d\.]+):(\d+) ([\dA-F ]{49})",') + +FALLBACK_DIV = '/* ===== */' +FALLBACK_MAPPING = re.compile('/\*\s+(\S+)=(\S*)\s+\*/') + +FALLBACK_ADDR = re.compile('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*') +FALLBACK_NICKNAME = re.compile('/\* nickname=(\S+) \*/') +FALLBACK_EXTRAINFO = re.compile('/\* extrainfo=([0-1]) \*/') +FALLBACK_IPV6 = re.compile('" ipv6=\[([\da-f:]+)\]:(\d+)"') + + +class Directory(object): + """ + Relay we can contact for directory information. + + Our :func:`~stem.directory.Directory.from_cache` and + :func:`~stem.directory.Directory.from_remote` functions key off a + different identifier based on our subclass... + + * **Authority** keys off the nickname. + * **Fallback** keys off fingerprints. + + This is because authorities are highly static and canonically known by their + names, whereas fallbacks vary more and don't necessarily have a nickname to + key off of. + + .. versionchanged:: 1.3.0 + Moved nickname from subclasses to this base class. + + :var str address: IPv4 address of the directory + :var int or_port: port on which the relay services relay traffic + :var int dir_port: port on which directory information is available + :var str fingerprint: relay fingerprint + :var str nickname: relay nickname + """ + + def __init__(self, address, or_port, dir_port, fingerprint, nickname): + self.address = address + self.or_port = or_port + self.dir_port = dir_port + self.fingerprint = fingerprint + self.nickname = nickname + + @staticmethod + def from_cache(): + """ + Provides cached Tor directory information. This information is hardcoded + into Tor and occasionally changes, so the information this provides might + not necessarily match your version of tor. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 1.7.0 + Support added to the :class:`~stem.directory.Authority` class. + + :returns: **dict** of **str** identifiers to + :class:`~stem.directory.Directory` instances + """ + + raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') + + @staticmethod + def from_remote(timeout = 60): + """ + Reads and parses tor's directory data `from gitweb.torproject.org <https://gitweb.torproject.org/>`_. + Note that while convenient, this reliance on GitWeb means you should alway + call with a fallback, such as... + + :: + + try: + authorities = stem.directory.Authority.from_remote() + except IOError: + authorities = stem.directory.Authority.from_cache() + + .. versionadded:: 1.5.0 + + .. versionchanged:: 1.7.0 + Support added to the :class:`~stem.directory.Authority` class. + + :param int timeout: seconds to wait before timing out the request + + :returns: **dict** of **str** identifiers to their + :class:`~stem.directory.Directory` + + :raises: **IOError** if unable to retrieve the fallback directories + """ + + raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') + + def __hash__(self): + return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint') + + def __eq__(self, other): + return hash(self) == hash(other) if isinstance(other, Directory) else False + + def __ne__(self, other): + return not self == other + + +class Authority(Directory): + """ + Tor directory authority, a special type of relay `hardcoded into tor + <https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc>`_ + that enumerates the other relays within the network. + + At a very high level tor works as follows... + + 1. A volunteer starts up a new tor relay, during which it sends a `server + descriptor <server_descriptor.html>`_ to each of the directory + authorities. + + 2. Each hour the directory authorities make a `vote <networkstatus.html>`_ + that says who they think the active relays are in the network and some + attributes about them. + + 3. The directory authorities send each other their votes, and compile that + into the `consensus <networkstatus.html>`_. This document is very similar + to the votes, the only difference being that the majority of the + authorities agree upon and sign this document. The idividual relay entries + in the vote or consensus is called `router status entries + <router_status_entry.html>`_. + + 4. Tor clients (people using the service) download the consensus from one of + the authorities or a mirror to determine the active relays within the + network. They in turn use this to construct their circuits and use the + network. + + .. versionchanged:: 1.3.0 + Added the is_bandwidth_authority attribute. + + :var str v3ident: identity key fingerprint used to sign votes and consensus + :var bool is_bandwidth_authority: **True** if this is a bandwidth authority, + **False** otherwise + """ + + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False): + super(Authority, self).__init__(address, or_port, dir_port, fingerprint, nickname) + self.v3ident = v3ident + self.is_bandwidth_authority = is_bandwidth_authority + + @staticmethod + def from_cache(): + return dict(DIRECTORY_AUTHORITIES) + + @staticmethod + def from_remote(timeout = 60): + try: + lines = str_tools._to_unicode(urllib.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() + except: + exc = sys.exc_info()[1] + raise IOError("Unable to download tor's directory authorities from %s: %s" % (GITWEB_AUTHORITY_URL, exc)) + + if not lines: + raise IOError('%s did not have any content' % GITWEB_AUTHORITY_URL) + + results = {} + + while lines: + section = Authority._pop_section(lines) + + if section: + try: + authority = Authority._from_str('\n'.join(section)) + results[authority.nickname] = authority + except ValueError as exc: + raise IOError(str(exc)) + + return results + + @staticmethod + def _from_str(content): + """ + Parses authority from its textual representation. For example... + + :: + + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + + :param str content: text to parse + + :returns: :class:`~stem.directory.Authority` in the text + + :raises: **ValueError** if content is malformed + """ + + if isinstance(content, bytes): + content = str_tools._to_unicode(content) + + matches = {} + + for line in content.splitlines(): + for matcher in (AUTHORITY_NAME, AUTHORITY_V3IDENT, AUTHORITY_IPV6, AUTHORITY_ADDR): + m = matcher.match(line.strip()) + + if m: + match_groups = m.groups() + matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] + + if AUTHORITY_NAME not in matches: + raise ValueError('Unable to parse the name and orport from:\n\n%s' % content) + elif AUTHORITY_ADDR not in matches: + raise ValueError('Unable to parse the address and fingerprint from:\n\n%s' % content) + + nickname, or_port = matches.get(AUTHORITY_NAME) + v3ident = matches.get(AUTHORITY_V3IDENT) + orport_v6 = matches.get(AUTHORITY_IPV6) # TODO: add this to stem's data? + address, dir_port, fingerprint = matches.get(AUTHORITY_ADDR) + + fingerprint = fingerprint.replace(' ', '') + + if not connection.is_valid_ipv4_address(address): + raise ValueError('%s has an invalid IPv4 address: %s' % (nickname, address)) + elif not connection.is_valid_port(or_port): + raise ValueError('%s has an invalid or_port: %s' % (nickname, or_port)) + elif not connection.is_valid_port(dir_port): + raise ValueError('%s has an invalid dir_port: %s' % (nickname, dir_port)) + elif not tor_tools.is_valid_fingerprint(fingerprint): + raise ValueError('%s has an invalid fingerprint: %s' % (nickname, fingerprint)) + elif nickname and not tor_tools.is_valid_nickname(nickname): + raise ValueError('%s has an invalid nickname: %s' % (nickname, nickname)) + elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): + raise ValueError('%s has an invalid IPv6 address: %s' % (nickname, orport_v6[0])) + elif orport_v6 and not connection.is_valid_port(orport_v6[1]): + raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (nickname, orport_v6[1])) + elif v3ident and not tor_tools.is_valid_fingerprint(v3ident): + raise ValueError('%s has an invalid v3ident: %s' % (nickname, v3ident)) + + return Authority( + address = address, + or_port = int(or_port), + dir_port = int(dir_port), + fingerprint = fingerprint, + nickname = nickname, + v3ident = v3ident, + ) + + @staticmethod + def _pop_section(lines): + """ + Provides the next authority entry. + """ + + section_lines = [] + + if lines: + section_lines.append(lines.pop(0)) + + while lines and lines[0].startswith(' '): + section_lines.append(lines.pop(0)) + + return section_lines + + def __hash__(self): + return _hash_attr(self, 'nickname', 'v3ident', 'is_bandwidth_authority', parent = Directory) + + def __eq__(self, other): + return hash(self) == hash(other) if isinstance(other, Authority) else False + + def __ne__(self, other): + return not self == other + + +class Fallback(Directory): + """ + Particularly stable relays tor can instead of authorities when + bootstrapping. These relays are `hardcoded in tor + <https://gitweb.torproject.org/tor.git/tree/src/or/fallback_dirs.inc>`_. + + For example, the following checks the performance of tor's fallback directories... + + :: + + import time + from stem.descriptor.remote import DescriptorDownloader + from stem.directory import Fallback + + downloader = DescriptorDownloader() + + for fallback in Fallback.from_cache().values(): + start = time.time() + downloader.get_consensus(endpoints = [(fallback.address, fallback.dir_port)]).run() + print('Downloading the consensus took %0.2f from %s' % (time.time() - start, fallback.fingerprint)) + + :: + + % python example.py + Downloading the consensus took 5.07 from 0AD3FA884D18F89EEA2D89C019379E0E7FD94417 + Downloading the consensus took 3.59 from C871C91489886D5E2E94C13EA1A5FDC4B6DC5204 + Downloading the consensus took 4.16 from 74A910646BCEEFBCD2E874FC1DC997430F968145 + ... + + .. versionadded:: 1.5.0 + + .. versionchanged:: 1.7.0 + Added the nickname, has_extrainfo, and header attributes which are part of + the `second version of the fallback directories + <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_. + + :var bool has_extrainfo: **True** if the relay should be able to provide + extrainfo descriptors, **False** otherwise. + :var str orport_v6: **(address, port)** tuple for the directory's IPv6 + ORPort, or **None** if it doesn't have one + :var dict header: metadata about the fallback directory file this originated from + """ + + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, has_extrainfo = False, orport_v6 = None, header = None): + super(Fallback, self).__init__(address, or_port, dir_port, fingerprint, nickname) + + self.has_extrainfo = has_extrainfo + self.orport_v6 = orport_v6 + self.header = header if header else OrderedDict() + + @staticmethod + def from_cache(path = CACHE_PATH): + conf = stem.util.conf.Config() + conf.load(path) + headers = OrderedDict([(k.split('.', 1)[1], conf.get(k)) for k in conf.keys() if k.startswith('header.')]) + + results = {} + + for fingerprint in set([key.split('.')[0] for key in conf.keys()]): + if fingerprint in ('tor_commit', 'stem_commit', 'header'): + continue + + attr = {} + + for attr_name in ('address', 'or_port', 'dir_port', 'nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): + key = '%s.%s' % (fingerprint, attr_name) + attr[attr_name] = conf.get(key) + + if not attr[attr_name] and attr_name not in ('nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): + raise IOError("'%s' is missing from %s" % (key, CACHE_PATH)) + + if not connection.is_valid_ipv4_address(attr['address']): + raise IOError("'%s.address' was an invalid IPv4 address (%s)" % (fingerprint, attr['address'])) + elif not connection.is_valid_port(attr['or_port']): + raise IOError("'%s.or_port' was an invalid port (%s)" % (fingerprint, attr['or_port'])) + elif not connection.is_valid_port(attr['dir_port']): + raise IOError("'%s.dir_port' was an invalid port (%s)" % (fingerprint, attr['dir_port'])) + elif attr['nickname'] and not tor_tools.is_valid_nickname(attr['nickname']): + raise IOError("'%s.nickname' was an invalid nickname (%s)" % (fingerprint, attr['nickname'])) + elif attr['orport6_address'] and not connection.is_valid_ipv6_address(attr['orport6_address']): + raise IOError("'%s.orport6_address' was an invalid IPv6 address (%s)" % (fingerprint, attr['orport6_address'])) + elif attr['orport6_port'] and not connection.is_valid_port(attr['orport6_port']): + raise IOError("'%s.orport6_port' was an invalid port (%s)" % (fingerprint, attr['orport6_port'])) + + if attr['orport6_address'] and attr['orport6_port']: + orport_v6 = (attr['orport6_address'], int(attr['orport6_port'])) + else: + orport_v6 = None + + results[fingerprint] = Fallback( + address = attr['address'], + or_port = int(attr['or_port']), + dir_port = int(attr['dir_port']), + fingerprint = fingerprint, + nickname = attr['nickname'], + has_extrainfo = attr['has_extrainfo'] == 'true', + orport_v6 = orport_v6, + header = headers, + ) + + return results + + @staticmethod + def from_remote(timeout = 60): + try: + lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() + except: + exc = sys.exc_info()[1] + raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc)) + + if not lines: + raise IOError('%s did not have any content' % GITWEB_FALLBACK_URL) + elif lines[0] != '/* type=fallback */': + raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_URL) + + # header metadata + + header = {} + + for line in Fallback._pop_section(lines): + mapping = FALLBACK_MAPPING.match(line) + + if mapping: + header[mapping.group(1)] = mapping.group(2) + else: + raise IOError('Malformed fallback directory header line: %s' % line) + + # human readable comments + + Fallback._pop_section(lines) + + # content, everything remaining are fallback directories + + results = {} + + while lines: + section = Fallback._pop_section(lines) + + if section: + try: + fallback = Fallback._from_str('\n'.join(section)) + fallback.header = header + results[fallback.fingerprint] = fallback + except ValueError as exc: + raise IOError(str(exc)) + + return results + + @staticmethod + def _from_str(content): + """ + Parses a fallback from its textual representation. For example... + + :: + + "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" + " ipv6=[2a01:4f8:162:51e2::2]:9001" + /* nickname=rueckgrat */ + /* extrainfo=1 */ + + :param str content: text to parse + + :returns: :class:`~stem.directory.Fallback` in the text + + :raises: **ValueError** if content is malformed + """ + + if isinstance(content, bytes): + content = str_tools._to_unicode(content) + + matches = {} + + for line in content.splitlines(): + for matcher in (FALLBACK_ADDR, FALLBACK_NICKNAME, FALLBACK_EXTRAINFO, FALLBACK_IPV6): + m = matcher.match(line) + + if m: + match_groups = m.groups() + matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] + + if FALLBACK_ADDR not in matches: + raise ValueError('Malformed fallback address line:\n\n%s' % content) + + address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR] + nickname = matches.get(FALLBACK_NICKNAME) + has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1' + orport_v6 = matches.get(FALLBACK_IPV6) + + if not connection.is_valid_ipv4_address(address): + raise ValueError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) + elif not connection.is_valid_port(or_port): + raise ValueError('%s has an invalid or_port: %s' % (fingerprint, or_port)) + elif not connection.is_valid_port(dir_port): + raise ValueError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) + elif not tor_tools.is_valid_fingerprint(fingerprint): + raise ValueError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) + elif nickname and not tor_tools.is_valid_nickname(nickname): + raise ValueError('%s has an invalid nickname: %s' % (fingerprint, nickname)) + elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): + raise ValueError('%s has an invalid IPv6 address: %s' % (fingerprint, orport_v6[0])) + elif orport_v6 and not connection.is_valid_port(orport_v6[1]): + raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, orport_v6[1])) + + return Fallback( + address = address, + or_port = int(or_port), + dir_port = int(dir_port), + fingerprint = fingerprint, + nickname = nickname, + has_extrainfo = has_extrainfo, + orport_v6 = (orport_v6[0], int(orport_v6[1])) if orport_v6 else None, + ) + + @staticmethod + def _pop_section(lines): + """ + Provides lines up through the next divider. This excludes lines with just a + comma since they're an artifact of these being C strings. + """ + + section_lines = [] + + if lines: + line = lines.pop(0) + + while lines and line != FALLBACK_DIV: + if line.strip() != ',': + section_lines.append(line) + + line = lines.pop(0) + + return section_lines + + @staticmethod + def _write(fallbacks, tor_commit, stem_commit, headers, path = CACHE_PATH): + """ + Persists fallback directories to a location in a way that can be read by + from_cache(). + + :param dict fallbacks: mapping of fingerprints to their fallback directory + :param str tor_commit: tor commit the fallbacks came from + :param str stem_commit: stem commit the fallbacks came from + :param dict headers: metadata about the file these came from + :param str path: location fallbacks will be persisted to + """ + + conf = stem.util.conf.Config() + conf.set('tor_commit', tor_commit) + conf.set('stem_commit', stem_commit) + + for k, v in headers.items(): + conf.set('header.%s' % k, v) + + for directory in sorted(fallbacks.values(), key = lambda x: x.fingerprint): + fingerprint = directory.fingerprint + conf.set('%s.address' % fingerprint, directory.address) + conf.set('%s.or_port' % fingerprint, str(directory.or_port)) + conf.set('%s.dir_port' % fingerprint, str(directory.dir_port)) + conf.set('%s.nickname' % fingerprint, directory.nickname) + conf.set('%s.has_extrainfo' % fingerprint, 'true' if directory.has_extrainfo else 'false') + + if directory.orport_v6: + conf.set('%s.orport6_address' % fingerprint, str(directory.orport_v6[0])) + conf.set('%s.orport6_port' % fingerprint, str(directory.orport_v6[1])) + + conf.save(path) + + def __hash__(self): + return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'has_extrainfo', 'orport_v6', 'header', parent = Directory) + + def __eq__(self, other): + return hash(self) == hash(other) if isinstance(other, Fallback) else False + + def __ne__(self, other): + return not self == other + + +def _fallback_directory_differences(previous_directories, new_directories): + """ + Provides a description of how fallback directories differ. + """ + + lines = [] + + added_fp = set(new_directories.keys()).difference(previous_directories.keys()) + removed_fp = set(previous_directories.keys()).difference(new_directories.keys()) + + for fp in added_fp: + directory = new_directories[fp] + orport_v6 = '%s:%s' % directory.orport_v6 if directory.orport_v6 else '[none]' + + lines += [ + '* Added %s as a new fallback directory:' % directory.fingerprint, + ' address: %s' % directory.address, + ' or_port: %s' % directory.or_port, + ' dir_port: %s' % directory.dir_port, + ' nickname: %s' % directory.nickname, + ' has_extrainfo: %s' % directory.has_extrainfo, + ' orport_v6: %s' % orport_v6, + '', + ] + + for fp in removed_fp: + lines.append('* Removed %s as a fallback directory' % fp) + + for fp in new_directories: + if fp in added_fp or fp in removed_fp: + continue # already discussed these + + previous_directory = previous_directories[fp] + new_directory = new_directories[fp] + + if previous_directory != new_directory: + for attr in ('address', 'or_port', 'dir_port', 'fingerprint', 'orport_v6'): + old_attr = getattr(previous_directory, attr) + new_attr = getattr(new_directory, attr) + + if old_attr != new_attr: + lines.append('* Changed the %s of %s from %s to %s' % (attr, fp, old_attr, new_attr)) + + return '\n'.join(lines) + + +DIRECTORY_AUTHORITIES = { + 'moria1': Authority( + nickname = 'moria1', + address = '128.31.0.39', + or_port = 9101, + dir_port = 9131, + is_bandwidth_authority = True, + fingerprint = '9695DFC35FFEB861329B9F1AB04C46397020CE31', + v3ident = 'D586D18309DED4CD6D57C18FDB97EFA96D330566', + ), + 'tor26': Authority( + nickname = 'tor26', + address = '86.59.21.38', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = False, + fingerprint = '847B1F850344D7876491A54892F904934E4EB85D', + v3ident = '14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', + ), + 'dizum': Authority( + nickname = 'dizum', + address = '194.109.206.212', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = False, + fingerprint = '7EA6EAD6FD83083C538F44038BBFA077587DD755', + v3ident = 'E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58', + ), + 'gabelmoo': Authority( + nickname = 'gabelmoo', + address = '131.188.40.189', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = True, + fingerprint = 'F2044413DAC2E02E3D6BCF4735A19BCA1DE97281', + v3ident = 'ED03BB616EB2F60BEC80151114BB25CEF515B226', + ), + 'dannenberg': Authority( + nickname = 'dannenberg', + address = '193.23.244.244', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = False, + fingerprint = '7BE683E65D48141321C5ED92F075C55364AC7123', + v3ident = '0232AF901C31A04EE9848595AF9BB7620D4C5B2E', + ), + 'maatuska': Authority( + nickname = 'maatuska', + address = '171.25.193.9', + or_port = 80, + dir_port = 443, + is_bandwidth_authority = True, + fingerprint = 'BD6A829255CB08E66FBE7D3748363586E46B3810', + v3ident = '49015F787433103580E3B66A1707A00E60F2D15B', + ), + 'Faravahar': Authority( + nickname = 'Faravahar', + address = '154.35.175.225', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = True, + fingerprint = 'CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC', + v3ident = 'EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97', + ), + 'longclaw': Authority( + nickname = 'longclaw', + address = '199.58.81.140', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = False, + fingerprint = '74A910646BCEEFBCD2E874FC1DC997430F968145', + v3ident = '23D15D965BC35114467363C165C4F724B64B4F66', + ), + 'bastet': Authority( + nickname = 'bastet', + address = '204.13.164.118', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = True, + fingerprint = '24E2F139121D4394C54B5BCC368B3B411857C413', + v3ident = '27102BC123E7AF1D4741AE047E160C91ADC76B21', + ), + 'Bifroest': Authority( + nickname = 'Bifroest', + address = '37.218.247.217', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = False, + fingerprint = '1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1', + v3ident = None, # does not vote in the consensus + ), +} diff --git a/stem/descriptor/fallback_directories.cfg b/stem/fallback_directories.cfg similarity index 100% rename from stem/descriptor/fallback_directories.cfg rename to stem/fallback_directories.cfg diff --git a/test/unit/tutorial_examples.py b/test/unit/tutorial_examples.py index ecc9b5ca..d94da5bf 100644 --- a/test/unit/tutorial_examples.py +++ b/test/unit/tutorial_examples.py @@ -17,9 +17,9 @@ import stem.prereq from stem.control import Controller from stem.descriptor.networkstatus import NetworkStatusDocumentV3 -from stem.descriptor.remote import DIRECTORY_AUTHORITIES from stem.descriptor.router_status_entry import RouterStatusEntryV3 from stem.descriptor.server_descriptor import RelayDescriptor +from stem.directory import DIRECTORY_AUTHORITIES from stem.response import ControlMessage from stem.util import str_type

1 0

[stem/master] Separate fallback unit tests
by atagar＠torproject.org 08 May '18

08 May '18

commit cf2b82b06688b4355a848029592ccdbea7fb3ebd Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 14:30:57 2018 -0700 Separate fallback unit tests Now that directories have their own module we can separate the fallback unit tests into their own module. We should add similar tests for authorities. --- test/settings.cfg | 1 + test/unit/__init__.py | 1 + test/unit/descriptor/remote.py | 176 ------------------------------------ test/unit/directory/__init__.py | 7 ++ test/unit/directory/fallback.py | 194 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 203 insertions(+), 176 deletions(-) diff --git a/test/settings.cfg b/test/settings.cfg index 60224463..2206c7e7 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -220,6 +220,7 @@ test.unit_tests |test.unit.endpoint.TestEndpoint |test.unit.version.TestVersion |test.unit.manual.TestManual +|test.unit.directory.fallback.TestFallback |test.unit.tutorial.TestTutorial |test.unit.tutorial_examples.TestTutorialExamples |test.unit.response.add_onion.TestAddOnionResponse diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 34fcbb19..1b16bc99 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -10,6 +10,7 @@ __all__ = [ 'connection', 'control', 'descriptor', + 'directory', 'exit_policy', 'socket', 'util', diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index fd831e1e..478a7143 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -5,13 +5,11 @@ Unit tests for stem.descriptor.remote. import io import re import socket -import tempfile import time import unittest import stem.descriptor.remote import stem.prereq -import stem.util.conf import stem.util.str_tools from stem.descriptor.remote import Compression @@ -23,12 +21,6 @@ except ImportError: from httplib import HTTPMessage # python2 try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - -try: # added in python 3.3 from unittest.mock import patch, Mock, MagicMock except ImportError: @@ -79,46 +71,6 @@ iO3EUE0AEYah2W9gdz8t+i3Dtr0zgqLS841GC/TyDKCm+MKmN8d098qnwK0NGF9q -----END SIGNATURE----- """ -FALLBACK_DIR_CONTENT = b"""\ -/* type=fallback */ -/* version=2.0.0 */ -/* timestamp=20170526090242 */ -/* ===== */ -/* Whitelist & blacklist excluded 1326 of 1513 candidates. */ -/* Checked IPv4 DirPorts served a consensus within 15.0s. */ -/* -Final Count: 151 (Eligible 187, Target 392 (1963 * 0.20), Max 200) -Excluded: 36 (Same Operator 27, Failed/Skipped Download 9, Excess 0) -Bandwidth Range: 1.3 - 40.0 MByte/s -*/ -/* -Onionoo Source: details Date: 2017-05-16 07:00:00 Version: 4.0 -URL: https:onionoo.torproject.orgdetails?fields=fingerprint%2Cnickname%2Ccontact%2Clast_changed_address_or_port%2Cconsensus_weight%2Cadvertised_bandwidth%2Cor_addresses%2Cdir_address%2Crecommended_version%2Cflags%2Ceffective_family%2Cplatform&flag=V2Dir&type=relay&last_seen_days=-0&first_seen_days=30- -*/ -/* -Onionoo Source: uptime Date: 2017-05-16 07:00:00 Version: 4.0 -URL: https:onionoo.torproject.orguptime?first_seen_days=30-&flag=V2Dir&type=relay&last_seen_days=-0 -*/ -/* ===== */ -"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" -" ipv6=[2a01:4f8:162:51e2::2]:9001" -/* nickname=rueckgrat */ -/* extrainfo=1 */ -/* ===== */ -, -"193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524" -/* nickname= */ -/* extrainfo=0 */ -/* ===== */ -""" - -FALLBACK_ENTRY = b"""\ -"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" -" ipv6=[2a01:4f8:162:51e2::2]:9001" -/* nickname=rueckgrat */ -/* extrainfo=1 */ -""" - HEADER = '\r\n'.join([ 'Date: Fri, 13 Apr 2018 16:35:50 GMT', 'Content-Type: application/octet-stream', @@ -421,131 +373,3 @@ class TestDescriptorDownloader(unittest.TestCase): def test_using_authorities_in_hash(self): # ensure our DirectoryAuthority instances can be used in hashes {stem.descriptor.remote.get_authorities()['moria1']: 'hello'} - - def test_fallback_directories_from_cache(self): - # quick sanity test that we can load cached content - fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache() - self.assertTrue(len(fallback_directories) > 10) - self.assertEqual('5.39.92.199', fallback_directories['0BEA4A88D069753218EAAAD6D22EA87B9A1319D6'].address) - - @patch(URL_OPEN, _dirport_mock(FALLBACK_DIR_CONTENT)) - def test_fallback_directories_from_remote(self): - fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() - header = OrderedDict((('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'))) - - expected = { - '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.descriptor.remote.FallbackDirectory( - address = '5.9.110.236', - or_port = 9001, - dir_port = 9030, - fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', - nickname = 'rueckgrat', - has_extrainfo = True, - orport_v6 = ('2a01:4f8:162:51e2::2', 9001), - header = header, - ), - '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.descriptor.remote.FallbackDirectory( - address = '193.171.202.146', - or_port = 9001, - dir_port = 9030, - fingerprint = '01A9258A46E97FF8B2CAC7910577862C14F2C524', - nickname = None, - has_extrainfo = False, - orport_v6 = None, - header = header, - ), - } - - self.assertEqual(expected, fallback_directories) - - def test_fallback_persistence(self): - header = OrderedDict((('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'))) - - expected = { - '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.descriptor.remote.FallbackDirectory( - address = '5.9.110.236', - or_port = 9001, - dir_port = 9030, - fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', - nickname = 'rueckgrat', - has_extrainfo = True, - orport_v6 = ('2a01:4f8:162:51e2::2', 9001), - header = header, - ), - '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.descriptor.remote.FallbackDirectory( - address = '193.171.202.146', - or_port = 9001, - dir_port = 9030, - fingerprint = '01A9258A46E97FF8B2CAC7910577862C14F2C524', - nickname = None, - has_extrainfo = False, - orport_v6 = None, - header = header, - ), - } - - excepted_config = { - 'tor_commit': ['abc'], - 'stem_commit': ['def'], - 'header.type': ['fallback'], - 'header.version': ['2.0.0'], - 'header.timestamp': ['20170526090242'], - '01A9258A46E97FF8B2CAC7910577862C14F2C524.address': ['193.171.202.146'], - '01A9258A46E97FF8B2CAC7910577862C14F2C524.or_port': ['9001'], - '01A9258A46E97FF8B2CAC7910577862C14F2C524.dir_port': ['9030'], - '01A9258A46E97FF8B2CAC7910577862C14F2C524.has_extrainfo': ['false'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.address': ['5.9.110.236'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.or_port': ['9001'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.dir_port': ['9030'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.nickname': ['rueckgrat'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.has_extrainfo': ['true'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.orport6_address': ['2a01:4f8:162:51e2::2'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.orport6_port': ['9001'], - } - - with tempfile.NamedTemporaryFile(prefix = 'fallbacks.') as tmp: - stem.descriptor.remote.FallbackDirectory._write(expected, 'abc', 'def', header, tmp.name) - - conf = stem.util.conf.Config() - conf.load(tmp.name) - self.assertEqual(excepted_config, dict(conf)) - - self.assertEqual(expected, stem.descriptor.remote.FallbackDirectory.from_cache(tmp.name)) - - @patch(URL_OPEN, _dirport_mock(b'')) - def test_fallback_directories_from_remote_empty(self): - self.assertRaisesRegexp(IOError, 'did not have any content', stem.descriptor.remote.FallbackDirectory.from_remote) - - @patch(URL_OPEN, _dirport_mock(b'\n'.join(FALLBACK_DIR_CONTENT.splitlines()[1:]))) - def test_fallback_directories_from_remote_no_header(self): - self.assertRaisesRegexp(IOError, 'does not have a type field indicating it is fallback directory metadata', stem.descriptor.remote.FallbackDirectory.from_remote) - - @patch(URL_OPEN, _dirport_mock(FALLBACK_DIR_CONTENT.replace(b'version=2.0.0', b'version'))) - def test_fallback_directories_from_remote_malformed_header(self): - self.assertRaisesRegexp(IOError, 'Malformed fallback directory header line: /\* version \*/', stem.descriptor.remote.FallbackDirectory.from_remote) - - def test_fallback_directories_from_str(self): - expected = stem.descriptor.remote.FallbackDirectory( - address = '5.9.110.236', - or_port = 9001, - dir_port = 9030, - fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', - nickname = 'rueckgrat', - has_extrainfo = True, - orport_v6 = ('2a01:4f8:162:51e2::2', 9001), - ) - - self.assertEqual(expected, stem.descriptor.remote.FallbackDirectory._from_str(FALLBACK_ENTRY)) - - def test_fallback_directories_from_str_malformed(self): - test_values = { - FALLBACK_ENTRY.replace(b'id=0756B7CD4DFC8182BE23143FAC0642F515182CEB', b''): 'Malformed fallback address line:', - FALLBACK_ENTRY.replace(b'5.9.110.236', b'5.9.110'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv4 address: 5.9.110', - FALLBACK_ENTRY.replace(b':9030', b':7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid dir_port: 7814713228', - FALLBACK_ENTRY.replace(b'orport=9001', b'orport=7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid or_port: 7814713228', - FALLBACK_ENTRY.replace(b'ipv6=[2a01', b'ipv6=[:::'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv6 address: ::::4f8:162:51e2::2', - FALLBACK_ENTRY.replace(b'nickname=rueckgrat', b'nickname=invalid~nickname'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid nickname: invalid~nickname', - } - - for entry, expected in test_values.items(): - self.assertRaisesRegexp(ValueError, expected, stem.descriptor.remote.FallbackDirectory._from_str, entry) diff --git a/test/unit/directory/__init__.py b/test/unit/directory/__init__.py new file mode 100644 index 00000000..78e42d7c --- /dev/null +++ b/test/unit/directory/__init__.py @@ -0,0 +1,7 @@ +""" +Unit tests for stem.directory. +""" + +__all__ = [ + 'fallback', +] diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py new file mode 100644 index 00000000..c95d74b7 --- /dev/null +++ b/test/unit/directory/fallback.py @@ -0,0 +1,194 @@ +""" +Unit tests for stem.directory.Fallback. +""" + +import io +import tempfile +import unittest + +import stem.directory +import stem.util.conf + +try: + # added in python 2.7 + from collections import OrderedDict +except ImportError: + from stem.util.ordereddict import OrderedDict + +try: + # added in python 3.3 + from unittest.mock import patch, Mock +except ImportError: + from mock import patch, Mock + +URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' + +FALLBACK_DIR_CONTENT = b"""\ +/* type=fallback */ +/* version=2.0.0 */ +/* timestamp=20170526090242 */ +/* ===== */ +/* Whitelist & blacklist excluded 1326 of 1513 candidates. */ +/* Checked IPv4 DirPorts served a consensus within 15.0s. */ +/* +Final Count: 151 (Eligible 187, Target 392 (1963 * 0.20), Max 200) +Excluded: 36 (Same Operator 27, Failed/Skipped Download 9, Excess 0) +Bandwidth Range: 1.3 - 40.0 MByte/s +*/ +/* +Onionoo Source: details Date: 2017-05-16 07:00:00 Version: 4.0 +URL: https:onionoo.torproject.orgdetails?fields=fingerprint%2Cnickname%2Ccontact%2Clast_changed_address_or_port%2Cconsensus_weight%2Cadvertised_bandwidth%2Cor_addresses%2Cdir_address%2Crecommended_version%2Cflags%2Ceffective_family%2Cplatform&flag=V2Dir&type=relay&last_seen_days=-0&first_seen_days=30- +*/ +/* +Onionoo Source: uptime Date: 2017-05-16 07:00:00 Version: 4.0 +URL: https:onionoo.torproject.orguptime?first_seen_days=30-&flag=V2Dir&type=relay&last_seen_days=-0 +*/ +/* ===== */ +"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" +" ipv6=[2a01:4f8:162:51e2::2]:9001" +/* nickname=rueckgrat */ +/* extrainfo=1 */ +/* ===== */ +, +"193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524" +/* nickname= */ +/* extrainfo=0 */ +/* ===== */ +""" + +FALLBACK_ENTRY = b"""\ +"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" +" ipv6=[2a01:4f8:162:51e2::2]:9001" +/* nickname=rueckgrat */ +/* extrainfo=1 */ +""" + + +class TestFallback(unittest.TestCase): + def test_from_cache(self): + # quick sanity test that we can load cached content + fallback_directories = stem.directory.Fallback.from_cache() + self.assertTrue(len(fallback_directories) > 10) + self.assertEqual('5.39.92.199', fallback_directories['0BEA4A88D069753218EAAAD6D22EA87B9A1319D6'].address) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(FALLBACK_DIR_CONTENT))) + def test_from_remote(self): + fallback_directories = stem.directory.Fallback.from_remote() + header = OrderedDict((('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'))) + + expected = { + '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.directory.Fallback( + address = '5.9.110.236', + or_port = 9001, + dir_port = 9030, + fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', + nickname = 'rueckgrat', + has_extrainfo = True, + orport_v6 = ('2a01:4f8:162:51e2::2', 9001), + header = header, + ), + '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.directory.Fallback( + address = '193.171.202.146', + or_port = 9001, + dir_port = 9030, + fingerprint = '01A9258A46E97FF8B2CAC7910577862C14F2C524', + nickname = None, + has_extrainfo = False, + orport_v6 = None, + header = header, + ), + } + + self.assertEqual(expected, fallback_directories) + + def test_persistence(self): + header = OrderedDict((('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'))) + + expected = { + '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.directory.Fallback( + address = '5.9.110.236', + or_port = 9001, + dir_port = 9030, + fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', + nickname = 'rueckgrat', + has_extrainfo = True, + orport_v6 = ('2a01:4f8:162:51e2::2', 9001), + header = header, + ), + '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.directory.Fallback( + address = '193.171.202.146', + or_port = 9001, + dir_port = 9030, + fingerprint = '01A9258A46E97FF8B2CAC7910577862C14F2C524', + nickname = None, + has_extrainfo = False, + orport_v6 = None, + header = header, + ), + } + + excepted_config = { + 'tor_commit': ['abc'], + 'stem_commit': ['def'], + 'header.type': ['fallback'], + 'header.version': ['2.0.0'], + 'header.timestamp': ['20170526090242'], + '01A9258A46E97FF8B2CAC7910577862C14F2C524.address': ['193.171.202.146'], + '01A9258A46E97FF8B2CAC7910577862C14F2C524.or_port': ['9001'], + '01A9258A46E97FF8B2CAC7910577862C14F2C524.dir_port': ['9030'], + '01A9258A46E97FF8B2CAC7910577862C14F2C524.has_extrainfo': ['false'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.address': ['5.9.110.236'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.or_port': ['9001'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.dir_port': ['9030'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.nickname': ['rueckgrat'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.has_extrainfo': ['true'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.orport6_address': ['2a01:4f8:162:51e2::2'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.orport6_port': ['9001'], + } + + with tempfile.NamedTemporaryFile(prefix = 'fallbacks.') as tmp: + stem.directory.Fallback._write(expected, 'abc', 'def', header, tmp.name) + + conf = stem.util.conf.Config() + conf.load(tmp.name) + self.assertEqual(excepted_config, dict(conf)) + + self.assertEqual(expected, stem.directory.Fallback.from_cache(tmp.name)) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) + def test_from_remote_empty(self): + self.assertRaisesRegexp(IOError, 'did not have any content', stem.directory.Fallback.from_remote) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'\n'.join(FALLBACK_DIR_CONTENT.splitlines()[1:])))) + def test_from_remote_no_header(self): + self.assertRaisesRegexp(IOError, 'does not have a type field indicating it is fallback directory metadata', stem.directory.Fallback.from_remote) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(FALLBACK_DIR_CONTENT.replace(b'version=2.0.0', b'version')))) + def test_from_remote_malformed_header(self): + self.assertRaisesRegexp(IOError, 'Malformed fallback directory header line: /\* version \*/', stem.directory.Fallback.from_remote) + + def test_from_str(self): + expected = stem.directory.Fallback( + address = '5.9.110.236', + or_port = 9001, + dir_port = 9030, + fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', + nickname = 'rueckgrat', + has_extrainfo = True, + orport_v6 = ('2a01:4f8:162:51e2::2', 9001), + ) + + self.assertEqual(expected, stem.directory.Fallback._from_str(FALLBACK_ENTRY)) + + def test_from_str_malformed(self): + test_values = { + FALLBACK_ENTRY.replace(b'id=0756B7CD4DFC8182BE23143FAC0642F515182CEB', b''): 'Malformed fallback address line:', + FALLBACK_ENTRY.replace(b'5.9.110.236', b'5.9.110'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv4 address: 5.9.110', + FALLBACK_ENTRY.replace(b':9030', b':7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid dir_port: 7814713228', + FALLBACK_ENTRY.replace(b'orport=9001', b'orport=7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid or_port: 7814713228', + FALLBACK_ENTRY.replace(b'ipv6=[2a01', b'ipv6=[:::'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv6 address: ::::4f8:162:51e2::2', + FALLBACK_ENTRY.replace(b'nickname=rueckgrat', b'nickname=invalid~nickname'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid nickname: invalid~nickname', + } + + for entry, expected in test_values.items(): + self.assertRaisesRegexp(ValueError, expected, stem.directory.Fallback._from_str, entry)

1 0

[stem/master] Rename cache files
by atagar＠torproject.org 08 May '18

08 May '18

commit 972f4a1b773b9b4bfb9a6127f1d7184e3cade5be Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 13:11:25 2018 -0700 Rename cache files Shorter and more consistant names. --- setup.py | 2 +- stem/{fallback_directories.cfg => cached_fallbacks.cfg} | 0 stem/{cached_tor_manual.sqlite => cached_manual.sqlite} | Bin stem/directory.py | 2 +- stem/manual.py | 2 +- test/unit/installation.py | 3 +-- 6 files changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 9e200bf1..1d674e13 100644 --- a/setup.py +++ b/setup.py @@ -106,7 +106,7 @@ try: keywords = 'tor onion controller', scripts = ['tor-prompt'], package_data = { - 'stem': ['cached_tor_manual.sqlite', 'fallback_directories.cfg', 'settings.cfg'], + 'stem': ['cached_fallbacks.cfg', 'cached_manual.sqlite', 'settings.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg'], }, classifiers = [ diff --git a/stem/fallback_directories.cfg b/stem/cached_fallbacks.cfg similarity index 100% rename from stem/fallback_directories.cfg rename to stem/cached_fallbacks.cfg diff --git a/stem/cached_tor_manual.sqlite b/stem/cached_manual.sqlite similarity index 100% rename from stem/cached_tor_manual.sqlite rename to stem/cached_manual.sqlite diff --git a/stem/directory.py b/stem/directory.py index b7bc201f..4190e885 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -38,7 +38,7 @@ except ImportError: GITWEB_AUTHORITY_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc' GITWEB_FALLBACK_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc' -CACHE_PATH = os.path.join(os.path.dirname(__file__), 'fallback_directories.cfg') +CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_fallbacks.cfg') AUTHORITY_NAME = re.compile('"(\S+) orport=(\d+) .*"') AUTHORITY_V3IDENT = re.compile('"v3ident=([\dA-F]{40}) "') diff --git a/stem/manual.py b/stem/manual.py index 3d734026..f8e7ea96 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -81,7 +81,7 @@ except ImportError: Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'RELAY', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'DENIAL_OF_SERVICE', 'TESTING', 'UNKNOWN') GITWEB_MANUAL_URL = 'https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt' -CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_tor_manual.sqlite') +CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_manual.sqlite') DATABASE = None # cache database connections HAS_ENCODING_ARG = not stem.util.system.is_mac() and not stem.util.system.is_bsd() and not stem.util.system.is_slackware() diff --git a/test/unit/installation.py b/test/unit/installation.py index 15d9fe85..feb52d89 100644 --- a/test/unit/installation.py +++ b/test/unit/installation.py @@ -54,8 +54,7 @@ class TestInstallation(unittest.TestCase): # Checking that we have all non-source files. Data looks like... # # package_data = { - # 'stem': ['cached_tor_manual.cfg', 'settings.cfg'], - # 'stem.descriptor': ['fallback_directories.cfg'], + # 'stem': ['cached_fallbacks.cfg', 'cached_manual.cfg', 'settings.cfg'], # }, package_data = {}

1 0