commit 609a411d05fc3288efd27d419fd478906e08bc34 Author: Damian Johnson atagar@torproject.org Date: Sat May 5 12:57:58 2018 -0700
Move Directory into its own module
Directory authorities started as a small, simple constant so it made sense to co-locate them in the remote module (the sole spot they were used). But they've grown. With the addition of fallback directories and from_remote() parsers they're now the bulk of the remote module and way past the point where they deserve their own module.
Adding a stem.directory module with aliases for backward compatability. Also shortening the names. With this the module is a *lot* nicer to use. For instance to list the authorities we're going from...
for authority in stem.descriptor.remote.DirectoryAuthority.from_cache(): print(authority.nickname)
... to...
for authority in stem.directory.Authority.from_cache(): print(authority.nickname)
Quite a few other improvements I'd like to follow this up with, but starting with just a simple move. --- docs/api/directory.rst | 5 + docs/change_log.rst | 9 +- setup.py | 3 +- stem/__init__.py | 1 + stem/descriptor/remote.py | 710 +----------------------- stem/directory.py | 725 +++++++++++++++++++++++++ stem/{descriptor => }/fallback_directories.cfg | 0 test/unit/tutorial_examples.py | 2 +- 8 files changed, 746 insertions(+), 709 deletions(-)
diff --git a/docs/api/directory.rst b/docs/api/directory.rst new file mode 100644 index 00000000..befd1d59 --- /dev/null +++ b/docs/api/directory.rst @@ -0,0 +1,5 @@ +Directory +========= + +.. automodule:: stem.directory + diff --git a/docs/change_log.rst b/docs/change_log.rst index d5505821..67359a95 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -57,15 +57,16 @@ The following are only available within Stem's `git repository
* `stem.descriptor.remote <api/descriptor/remote.html>`_ can now download from relay ORPorts * Zstd and lzma compression support (:spec:`1cb56af`) - * Added :func:`~stem.descriptor.remote.Directory.from_cache` and :func:`~stem.descriptor.remote.Directory.from_remote` to the :class:`~stem.descriptor.remote.DirectoryAuthority` subclass. + * Moved the Directory classes into their own `stem.directory <api/directory.html>`_ module + * Added :func:`~stem.descriptor.remote.Directory.from_cache` and :func:`~stem.descriptor.remote.Directory.from_remote` to the :class:`~stem.descriptor.remote.DirectoryAuthority` subclass * `Fallback directory v2 support https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html`_, which adds *nickname* and *extrainfo* * Added server descriptor's new is_hidden_service_dir attribute * Don't retry downloading descriptors when we've timed out - * Don't download from tor26 and Bifroest, which are authorities that frequently timeout. - * `stem.descriptor.remote <api/descriptor/remote.html>`_ now consistently defaults **fall_back_to_authority** to false. + * Don't download from tor26 and Bifroest, which are authorities that frequently timeout + * `stem.descriptor.remote <api/descriptor/remote.html>`_ now consistently defaults **fall_back_to_authority** to false * Added :func:`~stem.descriptor.remote.their_server_descriptor` * Added the reply_headers attribute to :class:`~stem.descriptor.remote.Query` - * Supplying a User-Agent when downloading descriptors. + * Supplying a User-Agent when downloading descriptors * Reduced maximum descriptors fetched by the remote module to match tor's new limit (:trac:`24743`) * Consensus **shared_randomness_*_reveal_count** attributes undocumented, and unavailable if retrieved before their corresponding shared_randomness_*_value attribute (:trac:`25046`) * Allow 'proto' line to have blank values (:spec:`a8455f4`) diff --git a/setup.py b/setup.py index 87a766b5..9e200bf1 100644 --- a/setup.py +++ b/setup.py @@ -106,8 +106,7 @@ try: keywords = 'tor onion controller', scripts = ['tor-prompt'], package_data = { - 'stem': ['cached_tor_manual.sqlite', 'settings.cfg'], - 'stem.descriptor': ['fallback_directories.cfg'], + 'stem': ['cached_tor_manual.sqlite', 'fallback_directories.cfg', 'settings.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg'], }, classifiers = [ diff --git a/stem/__init__.py b/stem/__init__.py index c7432d86..0104d3b2 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -492,6 +492,7 @@ __all__ = [ 'util', 'connection', 'control', + 'directory', 'exit_policy', 'prereq', 'process', diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index e0b4c21d..301e89d9 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -50,13 +50,6 @@ content. For example... |- get_extrainfo_descriptors - provides present extrainfo descriptors +- get_consensus - provides the present consensus or router status entries
- Directory - Relay we can retrieve directory information from - | |- from_cache - Provides fallback directories cached with Stem. - | +- from_remote - Retrieves fallback directories remotely from tor's latest commit. - | - |- DirectoryAuthority - Information about a tor directory authority - +- FallbackDirectory - Directory mirror tor uses when authories are unavailable - Query - Asynchronous request to download tor descriptors |- start - issues the query if it isn't already running +- run - blocks until the request is finished and provides the results @@ -99,9 +92,7 @@ content. For example... """
import io -import os import random -import re import sys import threading import time @@ -110,16 +101,11 @@ import zlib import stem import stem.client import stem.descriptor +import stem.directory import stem.prereq import stem.util.enum
-from stem.util import _hash_attr, connection, log, str_tools, tor_tools - -try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict +from stem.util import log, str_tools
try: # account for urllib's change between python 2.x and 3.x @@ -166,23 +152,6 @@ LZMA_UNAVAILABLE_MSG = 'LZMA compression requires the lzma module (https://docs. MAX_FINGERPRINTS = 96 MAX_MICRODESCRIPTOR_HASHES = 90
-GITWEB_AUTHORITY_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc' -GITWEB_FALLBACK_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc' -CACHE_PATH = os.path.join(os.path.dirname(__file__), 'fallback_directories.cfg') - -AUTHORITY_NAME = re.compile('"(\S+) orport=(\d+) .*"') -AUTHORITY_V3IDENT = re.compile('"v3ident=([\dA-F]{40}) "') -AUTHORITY_IPV6 = re.compile('"ipv6=[([\da-f:]+)]:(\d+) "') -AUTHORITY_ADDR = re.compile('"([\d.]+):(\d+) ([\dA-F ]{49})",') - -FALLBACK_DIV = '/* ===== */' -FALLBACK_MAPPING = re.compile('/*\s+(\S+)=(\S*)\s+*/') - -FALLBACK_ADDR = re.compile('"([\d.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*') -FALLBACK_NICKNAME = re.compile('/* nickname=(\S+) */') -FALLBACK_EXTRAINFO = re.compile('/* extrainfo=([0-1]) */') -FALLBACK_IPV6 = re.compile('" ipv6=[([\da-f:]+)]:(\d+)"') - SINGLETON_DOWNLOADER = None
@@ -1019,354 +988,6 @@ class DescriptorDownloader(object): return Query(resource, **args)
-class Directory(object): - """ - Relay we can contact for directory information. - - Our :func:`~stem.descriptor.remote.Directory.from_cache` and - :func:`~stem.descriptor.remote.Directory.from_remote` functions key off a - different identifier based on our subclass... - - * **DirectoryAuthority** keys off the nickname. - * **FallbackDirectory** keys off fingerprints. - - This is because authorities are highly static and canonically known by their - names, whereas fallbacks vary more and don't necessarily have a nickname to - key off of. - - .. versionchanged:: 1.3.0 - Moved nickname from subclasses to this base class. - - :var str address: IPv4 address of the directory - :var int or_port: port on which the relay services relay traffic - :var int dir_port: port on which directory information is available - :var str fingerprint: relay fingerprint - :var str nickname: relay nickname - """ - - def __init__(self, address, or_port, dir_port, fingerprint, nickname): - self.address = address - self.or_port = or_port - self.dir_port = dir_port - self.fingerprint = fingerprint - self.nickname = nickname - - @staticmethod - def from_cache(): - """ - Provides cached Tor directory information. This information is hardcoded - into Tor and occasionally changes, so the information this provides might - not necessarily match your version of tor. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 1.7.0 - Support added to the :class:`~stem.descriptor.remote.DirectoryAuthority` class. - - :returns: **dict** of **str** identifiers to - :class:`~stem.descriptor.remote.Directory` instances - """ - - raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') - - @staticmethod - def from_remote(timeout = 60): - """ - Reads and parses tor's directory data `from gitweb.torproject.org https://gitweb.torproject.org/`_. - Note that while convenient, this reliance on GitWeb means you should alway - call with a fallback, such as... - - :: - - try: - authorities = DirectoryAuthority.from_remote() - except IOError: - authorities = DirectoryAuthority.from_cache() - - .. versionadded:: 1.5.0 - - .. versionchanged:: 1.7.0 - Support added to the :class:`~stem.descriptor.remote.DirectoryAuthority` class. - - :param int timeout: seconds to wait before timing out the request - - :returns: **dict** of **str** identifiers to their - :class:`~stem.descriptor.remote.Directory` - - :raises: **IOError** if unable to retrieve the fallback directories - """ - - raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') - - def __hash__(self): - return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint') - - def __eq__(self, other): - return hash(self) == hash(other) if isinstance(other, Directory) else False - - def __ne__(self, other): - return not self == other - - -class DirectoryAuthority(Directory): - """ - Tor directory authority, a special type of relay `hardcoded into tor - https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc`_ - that enumerates the other relays within the network. - - At a very high level tor works as follows... - - 1. A volunteer starts up a new tor relay, during which it sends a `server - descriptor <server_descriptor.html>`_ to each of the directory - authorities. - - 2. Each hour the directory authorities make a `vote <networkstatus.html>`_ - that says who they think the active relays are in the network and some - attributes about them. - - 3. The directory authorities send each other their votes, and compile that - into the `consensus <networkstatus.html>`_. This document is very similar - to the votes, the only difference being that the majority of the - authorities agree upon and sign this document. The idividual relay entries - in the vote or consensus is called `router status entries - <router_status_entry.html>`_. - - 4. Tor clients (people using the service) download the consensus from one of - the authorities or a mirror to determine the active relays within the - network. They in turn use this to construct their circuits and use the - network. - - .. versionchanged:: 1.3.0 - Added the is_bandwidth_authority attribute. - - :var str v3ident: identity key fingerprint used to sign votes and consensus - :var bool is_bandwidth_authority: **True** if this is a bandwidth authority, - **False** otherwise - """ - - def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False): - super(DirectoryAuthority, self).__init__(address, or_port, dir_port, fingerprint, nickname) - self.v3ident = v3ident - self.is_bandwidth_authority = is_bandwidth_authority - - @staticmethod - def from_cache(): - return dict(DIRECTORY_AUTHORITIES) - - @staticmethod - def from_remote(timeout = 60): - try: - lines = str_tools._to_unicode(urllib.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() - except: - exc = sys.exc_info()[1] - raise IOError("Unable to download tor's directory authorities from %s: %s" % (GITWEB_AUTHORITY_URL, exc)) - - if not lines: - raise IOError('%s did not have any content' % GITWEB_AUTHORITY_URL) - - results = {} - - while lines: - section = DirectoryAuthority._pop_section(lines) - - if section: - try: - authority = DirectoryAuthority._from_str('\n'.join(section)) - results[authority.nickname] = authority - except ValueError as exc: - raise IOError(str(exc)) - - return results - - @staticmethod - def _from_str(content): - """ - Parses authority from its textual representation. For example... - - :: - - "moria1 orport=9101 " - "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " - "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", - - :param str content: text to parse - - :returns: :class:`~stem.descriptor.remote.DirectoryAuthority` in the text - - :raises: **ValueError** if content is malformed - """ - - if isinstance(content, bytes): - content = str_tools._to_unicode(content) - - matches = {} - - for line in content.splitlines(): - for matcher in (AUTHORITY_NAME, AUTHORITY_V3IDENT, AUTHORITY_IPV6, AUTHORITY_ADDR): - m = matcher.match(line.strip()) - - if m: - match_groups = m.groups() - matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] - - if AUTHORITY_NAME not in matches: - raise ValueError('Unable to parse the name and orport from:\n\n%s' % content) - elif AUTHORITY_ADDR not in matches: - raise ValueError('Unable to parse the address and fingerprint from:\n\n%s' % content) - - nickname, or_port = matches.get(AUTHORITY_NAME) - v3ident = matches.get(AUTHORITY_V3IDENT) - orport_v6 = matches.get(AUTHORITY_IPV6) # TODO: add this to stem's data? - address, dir_port, fingerprint = matches.get(AUTHORITY_ADDR) - - fingerprint = fingerprint.replace(' ', '') - - if not connection.is_valid_ipv4_address(address): - raise ValueError('%s has an invalid IPv4 address: %s' % (nickname, address)) - elif not connection.is_valid_port(or_port): - raise ValueError('%s has an invalid or_port: %s' % (nickname, or_port)) - elif not connection.is_valid_port(dir_port): - raise ValueError('%s has an invalid dir_port: %s' % (nickname, dir_port)) - elif not tor_tools.is_valid_fingerprint(fingerprint): - raise ValueError('%s has an invalid fingerprint: %s' % (nickname, fingerprint)) - elif nickname and not tor_tools.is_valid_nickname(nickname): - raise ValueError('%s has an invalid nickname: %s' % (nickname, nickname)) - elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): - raise ValueError('%s has an invalid IPv6 address: %s' % (nickname, orport_v6[0])) - elif orport_v6 and not connection.is_valid_port(orport_v6[1]): - raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (nickname, orport_v6[1])) - elif v3ident and not tor_tools.is_valid_fingerprint(v3ident): - raise ValueError('%s has an invalid v3ident: %s' % (nickname, v3ident)) - - return DirectoryAuthority( - address = address, - or_port = int(or_port), - dir_port = int(dir_port), - fingerprint = fingerprint, - nickname = nickname, - v3ident = v3ident, - ) - - @staticmethod - def _pop_section(lines): - """ - Provides the next authority entry. - """ - - section_lines = [] - - if lines: - section_lines.append(lines.pop(0)) - - while lines and lines[0].startswith(' '): - section_lines.append(lines.pop(0)) - - return section_lines - - def __hash__(self): - return _hash_attr(self, 'nickname', 'v3ident', 'is_bandwidth_authority', parent = Directory) - - def __eq__(self, other): - return hash(self) == hash(other) if isinstance(other, DirectoryAuthority) else False - - def __ne__(self, other): - return not self == other - - -DIRECTORY_AUTHORITIES = { - 'moria1': DirectoryAuthority( - nickname = 'moria1', - address = '128.31.0.39', - or_port = 9101, - dir_port = 9131, - is_bandwidth_authority = True, - fingerprint = '9695DFC35FFEB861329B9F1AB04C46397020CE31', - v3ident = 'D586D18309DED4CD6D57C18FDB97EFA96D330566', - ), - 'tor26': DirectoryAuthority( - nickname = 'tor26', - address = '86.59.21.38', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = False, - fingerprint = '847B1F850344D7876491A54892F904934E4EB85D', - v3ident = '14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', - ), - 'dizum': DirectoryAuthority( - nickname = 'dizum', - address = '194.109.206.212', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = False, - fingerprint = '7EA6EAD6FD83083C538F44038BBFA077587DD755', - v3ident = 'E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58', - ), - 'gabelmoo': DirectoryAuthority( - nickname = 'gabelmoo', - address = '131.188.40.189', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = True, - fingerprint = 'F2044413DAC2E02E3D6BCF4735A19BCA1DE97281', - v3ident = 'ED03BB616EB2F60BEC80151114BB25CEF515B226', - ), - 'dannenberg': DirectoryAuthority( - nickname = 'dannenberg', - address = '193.23.244.244', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = False, - fingerprint = '7BE683E65D48141321C5ED92F075C55364AC7123', - v3ident = '0232AF901C31A04EE9848595AF9BB7620D4C5B2E', - ), - 'maatuska': DirectoryAuthority( - nickname = 'maatuska', - address = '171.25.193.9', - or_port = 80, - dir_port = 443, - is_bandwidth_authority = True, - fingerprint = 'BD6A829255CB08E66FBE7D3748363586E46B3810', - v3ident = '49015F787433103580E3B66A1707A00E60F2D15B', - ), - 'Faravahar': DirectoryAuthority( - nickname = 'Faravahar', - address = '154.35.175.225', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = True, - fingerprint = 'CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC', - v3ident = 'EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97', - ), - 'longclaw': DirectoryAuthority( - nickname = 'longclaw', - address = '199.58.81.140', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = False, - fingerprint = '74A910646BCEEFBCD2E874FC1DC997430F968145', - v3ident = '23D15D965BC35114467363C165C4F724B64B4F66', - ), - 'bastet': DirectoryAuthority( - nickname = 'bastet', - address = '204.13.164.118', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = True, - fingerprint = '24E2F139121D4394C54B5BCC368B3B411857C413', - v3ident = '27102BC123E7AF1D4741AE047E160C91ADC76B21', - ), - 'Bifroest': DirectoryAuthority( - nickname = 'Bifroest', - address = '37.218.247.217', - or_port = 443, - dir_port = 80, - is_bandwidth_authority = False, - fingerprint = '1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1', - v3ident = None, # does not vote in the consensus - ), -} - - def get_authorities(): """ Provides cached Tor directory authority information. The directory @@ -1374,331 +995,16 @@ def get_authorities(): this provides might not necessarily match your version of tor.
.. deprecated:: 1.7.0 - Use stem.descriptor.remote.DirectoryAuthority.from_cache() instead. + Use stem.directory.Authority.from_cache() instead.
- :returns: **dict** of **str** nicknames to :class:`~stem.descriptor.remote.DirectoryAuthority` instances + :returns: **dict** of **str** nicknames to :class:`~stem.directory.Authority` instances """
return DirectoryAuthority.from_cache()
-class FallbackDirectory(Directory): - """ - Particularly stable relays tor can instead of authorities when - bootstrapping. These relays are `hardcoded in tor - https://gitweb.torproject.org/tor.git/tree/src/or/fallback_dirs.inc`_. - - For example, the following checks the performance of tor's fallback directories... - - :: - - import time - from stem.descriptor.remote import DescriptorDownloader, FallbackDirectory - - downloader = DescriptorDownloader() - - for fallback_directory in FallbackDirectory.from_cache().values(): - start = time.time() - downloader.get_consensus(endpoints = [(fallback_directory.address, fallback_directory.dir_port)]).run() - print('Downloading the consensus took %0.2f from %s' % (time.time() - start, fallback_directory.fingerprint)) - - :: - - % python example.py - Downloading the consensus took 5.07 from 0AD3FA884D18F89EEA2D89C019379E0E7FD94417 - Downloading the consensus took 3.59 from C871C91489886D5E2E94C13EA1A5FDC4B6DC5204 - Downloading the consensus took 4.16 from 74A910646BCEEFBCD2E874FC1DC997430F968145 - ... - - .. versionadded:: 1.5.0 - - .. versionchanged:: 1.7.0 - Added the nickname, has_extrainfo, and header attributes which are part of - the `second version of the fallback directories - https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html`_. - - :var bool has_extrainfo: **True** if the relay should be able to provide - extrainfo descriptors, **False** otherwise. - :var str orport_v6: **(address, port)** tuple for the directory's IPv6 - ORPort, or **None** if it doesn't have one - :var dict header: metadata about the fallback directory file this originated from - """ - - def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, has_extrainfo = False, orport_v6 = None, header = None): - super(FallbackDirectory, self).__init__(address, or_port, dir_port, fingerprint, nickname) - - self.has_extrainfo = has_extrainfo - self.orport_v6 = orport_v6 - self.header = header if header else OrderedDict() - - @staticmethod - def from_cache(path = CACHE_PATH): - conf = stem.util.conf.Config() - conf.load(path) - headers = OrderedDict([(k.split('.', 1)[1], conf.get(k)) for k in conf.keys() if k.startswith('header.')]) - - results = {} - - for fingerprint in set([key.split('.')[0] for key in conf.keys()]): - if fingerprint in ('tor_commit', 'stem_commit', 'header'): - continue - - attr = {} - - for attr_name in ('address', 'or_port', 'dir_port', 'nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): - key = '%s.%s' % (fingerprint, attr_name) - attr[attr_name] = conf.get(key) - - if not attr[attr_name] and attr_name not in ('nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): - raise IOError("'%s' is missing from %s" % (key, CACHE_PATH)) - - if not connection.is_valid_ipv4_address(attr['address']): - raise IOError("'%s.address' was an invalid IPv4 address (%s)" % (fingerprint, attr['address'])) - elif not connection.is_valid_port(attr['or_port']): - raise IOError("'%s.or_port' was an invalid port (%s)" % (fingerprint, attr['or_port'])) - elif not connection.is_valid_port(attr['dir_port']): - raise IOError("'%s.dir_port' was an invalid port (%s)" % (fingerprint, attr['dir_port'])) - elif attr['nickname'] and not tor_tools.is_valid_nickname(attr['nickname']): - raise IOError("'%s.nickname' was an invalid nickname (%s)" % (fingerprint, attr['nickname'])) - elif attr['orport6_address'] and not connection.is_valid_ipv6_address(attr['orport6_address']): - raise IOError("'%s.orport6_address' was an invalid IPv6 address (%s)" % (fingerprint, attr['orport6_address'])) - elif attr['orport6_port'] and not connection.is_valid_port(attr['orport6_port']): - raise IOError("'%s.orport6_port' was an invalid port (%s)" % (fingerprint, attr['orport6_port'])) - - if attr['orport6_address'] and attr['orport6_port']: - orport_v6 = (attr['orport6_address'], int(attr['orport6_port'])) - else: - orport_v6 = None - - results[fingerprint] = FallbackDirectory( - address = attr['address'], - or_port = int(attr['or_port']), - dir_port = int(attr['dir_port']), - fingerprint = fingerprint, - nickname = attr['nickname'], - has_extrainfo = attr['has_extrainfo'] == 'true', - orport_v6 = orport_v6, - header = headers, - ) - - return results - - @staticmethod - def from_remote(timeout = 60): - try: - lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() - except: - exc = sys.exc_info()[1] - raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc)) - - if not lines: - raise IOError('%s did not have any content' % GITWEB_FALLBACK_URL) - elif lines[0] != '/* type=fallback */': - raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_URL) - - # header metadata - - header = {} - - for line in FallbackDirectory._pop_section(lines): - mapping = FALLBACK_MAPPING.match(line) - - if mapping: - header[mapping.group(1)] = mapping.group(2) - else: - raise IOError('Malformed fallback directory header line: %s' % line) - - # human readable comments - - FallbackDirectory._pop_section(lines) - - # content, everything remaining are fallback directories - - results = {} - - while lines: - section = FallbackDirectory._pop_section(lines) - - if section: - try: - fallback = FallbackDirectory._from_str('\n'.join(section)) - fallback.header = header - results[fallback.fingerprint] = fallback - except ValueError as exc: - raise IOError(str(exc)) - - return results - - @staticmethod - def _from_str(content): - """ - Parses a fallback from its textual representation. For example... - - :: - - "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" - " ipv6=[2a01:4f8:162:51e2::2]:9001" - /* nickname=rueckgrat */ - /* extrainfo=1 */ - - :param str content: text to parse - - :returns: :class:`~stem.descriptor.remote.FallbackDirectory` in the text - - :raises: **ValueError** if content is malformed - """ - - if isinstance(content, bytes): - content = str_tools._to_unicode(content) - - matches = {} - - for line in content.splitlines(): - for matcher in (FALLBACK_ADDR, FALLBACK_NICKNAME, FALLBACK_EXTRAINFO, FALLBACK_IPV6): - m = matcher.match(line) - - if m: - match_groups = m.groups() - matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] - - if FALLBACK_ADDR not in matches: - raise ValueError('Malformed fallback address line:\n\n%s' % content) - - address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR] - nickname = matches.get(FALLBACK_NICKNAME) - has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1' - orport_v6 = matches.get(FALLBACK_IPV6) - - if not connection.is_valid_ipv4_address(address): - raise ValueError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) - elif not connection.is_valid_port(or_port): - raise ValueError('%s has an invalid or_port: %s' % (fingerprint, or_port)) - elif not connection.is_valid_port(dir_port): - raise ValueError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) - elif not tor_tools.is_valid_fingerprint(fingerprint): - raise ValueError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) - elif nickname and not tor_tools.is_valid_nickname(nickname): - raise ValueError('%s has an invalid nickname: %s' % (fingerprint, nickname)) - elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): - raise ValueError('%s has an invalid IPv6 address: %s' % (fingerprint, orport_v6[0])) - elif orport_v6 and not connection.is_valid_port(orport_v6[1]): - raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, orport_v6[1])) - - return FallbackDirectory( - address = address, - or_port = int(or_port), - dir_port = int(dir_port), - fingerprint = fingerprint, - nickname = nickname, - has_extrainfo = has_extrainfo, - orport_v6 = (orport_v6[0], int(orport_v6[1])) if orport_v6 else None, - ) - - @staticmethod - def _pop_section(lines): - """ - Provides lines up through the next divider. This excludes lines with just a - comma since they're an artifact of these being C strings. - """ - - section_lines = [] - - if lines: - line = lines.pop(0) - - while lines and line != FALLBACK_DIV: - if line.strip() != ',': - section_lines.append(line) - - line = lines.pop(0) - - return section_lines - - @staticmethod - def _write(fallbacks, tor_commit, stem_commit, headers, path = CACHE_PATH): - """ - Persists fallback directories to a location in a way that can be read by - from_cache(). - - :param dict fallbacks: mapping of fingerprints to their fallback directory - :param str tor_commit: tor commit the fallbacks came from - :param str stem_commit: stem commit the fallbacks came from - :param dict headers: metadata about the file these came from - :param str path: location fallbacks will be persisted to - """ - - conf = stem.util.conf.Config() - conf.set('tor_commit', tor_commit) - conf.set('stem_commit', stem_commit) - - for k, v in headers.items(): - conf.set('header.%s' % k, v) - - for directory in sorted(fallbacks.values(), key = lambda x: x.fingerprint): - fingerprint = directory.fingerprint - conf.set('%s.address' % fingerprint, directory.address) - conf.set('%s.or_port' % fingerprint, str(directory.or_port)) - conf.set('%s.dir_port' % fingerprint, str(directory.dir_port)) - conf.set('%s.nickname' % fingerprint, directory.nickname) - conf.set('%s.has_extrainfo' % fingerprint, 'true' if directory.has_extrainfo else 'false') - - if directory.orport_v6: - conf.set('%s.orport6_address' % fingerprint, str(directory.orport_v6[0])) - conf.set('%s.orport6_port' % fingerprint, str(directory.orport_v6[1])) - - conf.save(path) - - def __hash__(self): - return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'has_extrainfo', 'orport_v6', 'header', parent = Directory) - - def __eq__(self, other): - return hash(self) == hash(other) if isinstance(other, FallbackDirectory) else False - - def __ne__(self, other): - return not self == other - - -def _fallback_directory_differences(previous_directories, new_directories): - """ - Provides a description of how fallback directories differ. - """ - - lines = [] - - added_fp = set(new_directories.keys()).difference(previous_directories.keys()) - removed_fp = set(previous_directories.keys()).difference(new_directories.keys()) - - for fp in added_fp: - directory = new_directories[fp] - orport_v6 = '%s:%s' % directory.orport_v6 if directory.orport_v6 else '[none]' - - lines += [ - '* Added %s as a new fallback directory:' % directory.fingerprint, - ' address: %s' % directory.address, - ' or_port: %s' % directory.or_port, - ' dir_port: %s' % directory.dir_port, - ' nickname: %s' % directory.nickname, - ' has_extrainfo: %s' % directory.has_extrainfo, - ' orport_v6: %s' % orport_v6, - '', - ] - - for fp in removed_fp: - lines.append('* Removed %s as a fallback directory' % fp) - - for fp in new_directories: - if fp in added_fp or fp in removed_fp: - continue # already discussed these - - previous_directory = previous_directories[fp] - new_directory = new_directories[fp] - - if previous_directory != new_directory: - for attr in ('address', 'or_port', 'dir_port', 'fingerprint', 'orport_v6'): - old_attr = getattr(previous_directory, attr) - new_attr = getattr(new_directory, attr) - - if old_attr != new_attr: - lines.append('* Changed the %s of %s from %s to %s' % (attr, fp, old_attr, new_attr)) +# TODO: drop aliases in stem 2.0
- return '\n'.join(lines) +Directory = stem.directory.Directory +DirectoryAuthority = stem.directory.Authority +FallbackDirectory = stem.directory.Fallback diff --git a/stem/directory.py b/stem/directory.py new file mode 100644 index 00000000..b7bc201f --- /dev/null +++ b/stem/directory.py @@ -0,0 +1,725 @@ +# Copyright 2018, Damian Johnson and The Tor Project +# See LICENSE for licensing information + +""" +Directories with Tor descriptor information. + +:: + + Directory - Relay we can retrieve directory information from + | |- from_cache - Provides fallback directories cached with Stem. + | +- from_remote - Retrieves fallback directories remotely from tor's latest commit. + | + |- Authority - Information about a tor directory authority + +- Fallback - Directory mirror tor uses when authories are unavailable + +.. versionadded:: 1.7.0 +""" + +import os +import re +import sys + +import stem.util.conf + +from stem.util import _hash_attr, connection, str_tools, tor_tools + +try: + # added in python 2.7 + from collections import OrderedDict +except ImportError: + from stem.util.ordereddict import OrderedDict + +try: + # account for urllib's change between python 2.x and 3.x + import urllib.request as urllib +except ImportError: + import urllib2 as urllib + +GITWEB_AUTHORITY_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc' +GITWEB_FALLBACK_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc' +CACHE_PATH = os.path.join(os.path.dirname(__file__), 'fallback_directories.cfg') + +AUTHORITY_NAME = re.compile('"(\S+) orport=(\d+) .*"') +AUTHORITY_V3IDENT = re.compile('"v3ident=([\dA-F]{40}) "') +AUTHORITY_IPV6 = re.compile('"ipv6=[([\da-f:]+)]:(\d+) "') +AUTHORITY_ADDR = re.compile('"([\d.]+):(\d+) ([\dA-F ]{49})",') + +FALLBACK_DIV = '/* ===== */' +FALLBACK_MAPPING = re.compile('/*\s+(\S+)=(\S*)\s+*/') + +FALLBACK_ADDR = re.compile('"([\d.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*') +FALLBACK_NICKNAME = re.compile('/* nickname=(\S+) */') +FALLBACK_EXTRAINFO = re.compile('/* extrainfo=([0-1]) */') +FALLBACK_IPV6 = re.compile('" ipv6=[([\da-f:]+)]:(\d+)"') + + +class Directory(object): + """ + Relay we can contact for directory information. + + Our :func:`~stem.directory.Directory.from_cache` and + :func:`~stem.directory.Directory.from_remote` functions key off a + different identifier based on our subclass... + + * **Authority** keys off the nickname. + * **Fallback** keys off fingerprints. + + This is because authorities are highly static and canonically known by their + names, whereas fallbacks vary more and don't necessarily have a nickname to + key off of. + + .. versionchanged:: 1.3.0 + Moved nickname from subclasses to this base class. + + :var str address: IPv4 address of the directory + :var int or_port: port on which the relay services relay traffic + :var int dir_port: port on which directory information is available + :var str fingerprint: relay fingerprint + :var str nickname: relay nickname + """ + + def __init__(self, address, or_port, dir_port, fingerprint, nickname): + self.address = address + self.or_port = or_port + self.dir_port = dir_port + self.fingerprint = fingerprint + self.nickname = nickname + + @staticmethod + def from_cache(): + """ + Provides cached Tor directory information. This information is hardcoded + into Tor and occasionally changes, so the information this provides might + not necessarily match your version of tor. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 1.7.0 + Support added to the :class:`~stem.directory.Authority` class. + + :returns: **dict** of **str** identifiers to + :class:`~stem.directory.Directory` instances + """ + + raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') + + @staticmethod + def from_remote(timeout = 60): + """ + Reads and parses tor's directory data `from gitweb.torproject.org https://gitweb.torproject.org/`_. + Note that while convenient, this reliance on GitWeb means you should alway + call with a fallback, such as... + + :: + + try: + authorities = stem.directory.Authority.from_remote() + except IOError: + authorities = stem.directory.Authority.from_cache() + + .. versionadded:: 1.5.0 + + .. versionchanged:: 1.7.0 + Support added to the :class:`~stem.directory.Authority` class. + + :param int timeout: seconds to wait before timing out the request + + :returns: **dict** of **str** identifiers to their + :class:`~stem.directory.Directory` + + :raises: **IOError** if unable to retrieve the fallback directories + """ + + raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass') + + def __hash__(self): + return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint') + + def __eq__(self, other): + return hash(self) == hash(other) if isinstance(other, Directory) else False + + def __ne__(self, other): + return not self == other + + +class Authority(Directory): + """ + Tor directory authority, a special type of relay `hardcoded into tor + https://gitweb.torproject.org/tor.git/plain/src/or/auth_dirs.inc`_ + that enumerates the other relays within the network. + + At a very high level tor works as follows... + + 1. A volunteer starts up a new tor relay, during which it sends a `server + descriptor <server_descriptor.html>`_ to each of the directory + authorities. + + 2. Each hour the directory authorities make a `vote <networkstatus.html>`_ + that says who they think the active relays are in the network and some + attributes about them. + + 3. The directory authorities send each other their votes, and compile that + into the `consensus <networkstatus.html>`_. This document is very similar + to the votes, the only difference being that the majority of the + authorities agree upon and sign this document. The idividual relay entries + in the vote or consensus is called `router status entries + <router_status_entry.html>`_. + + 4. Tor clients (people using the service) download the consensus from one of + the authorities or a mirror to determine the active relays within the + network. They in turn use this to construct their circuits and use the + network. + + .. versionchanged:: 1.3.0 + Added the is_bandwidth_authority attribute. + + :var str v3ident: identity key fingerprint used to sign votes and consensus + :var bool is_bandwidth_authority: **True** if this is a bandwidth authority, + **False** otherwise + """ + + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False): + super(Authority, self).__init__(address, or_port, dir_port, fingerprint, nickname) + self.v3ident = v3ident + self.is_bandwidth_authority = is_bandwidth_authority + + @staticmethod + def from_cache(): + return dict(DIRECTORY_AUTHORITIES) + + @staticmethod + def from_remote(timeout = 60): + try: + lines = str_tools._to_unicode(urllib.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() + except: + exc = sys.exc_info()[1] + raise IOError("Unable to download tor's directory authorities from %s: %s" % (GITWEB_AUTHORITY_URL, exc)) + + if not lines: + raise IOError('%s did not have any content' % GITWEB_AUTHORITY_URL) + + results = {} + + while lines: + section = Authority._pop_section(lines) + + if section: + try: + authority = Authority._from_str('\n'.join(section)) + results[authority.nickname] = authority + except ValueError as exc: + raise IOError(str(exc)) + + return results + + @staticmethod + def _from_str(content): + """ + Parses authority from its textual representation. For example... + + :: + + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + + :param str content: text to parse + + :returns: :class:`~stem.directory.Authority` in the text + + :raises: **ValueError** if content is malformed + """ + + if isinstance(content, bytes): + content = str_tools._to_unicode(content) + + matches = {} + + for line in content.splitlines(): + for matcher in (AUTHORITY_NAME, AUTHORITY_V3IDENT, AUTHORITY_IPV6, AUTHORITY_ADDR): + m = matcher.match(line.strip()) + + if m: + match_groups = m.groups() + matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] + + if AUTHORITY_NAME not in matches: + raise ValueError('Unable to parse the name and orport from:\n\n%s' % content) + elif AUTHORITY_ADDR not in matches: + raise ValueError('Unable to parse the address and fingerprint from:\n\n%s' % content) + + nickname, or_port = matches.get(AUTHORITY_NAME) + v3ident = matches.get(AUTHORITY_V3IDENT) + orport_v6 = matches.get(AUTHORITY_IPV6) # TODO: add this to stem's data? + address, dir_port, fingerprint = matches.get(AUTHORITY_ADDR) + + fingerprint = fingerprint.replace(' ', '') + + if not connection.is_valid_ipv4_address(address): + raise ValueError('%s has an invalid IPv4 address: %s' % (nickname, address)) + elif not connection.is_valid_port(or_port): + raise ValueError('%s has an invalid or_port: %s' % (nickname, or_port)) + elif not connection.is_valid_port(dir_port): + raise ValueError('%s has an invalid dir_port: %s' % (nickname, dir_port)) + elif not tor_tools.is_valid_fingerprint(fingerprint): + raise ValueError('%s has an invalid fingerprint: %s' % (nickname, fingerprint)) + elif nickname and not tor_tools.is_valid_nickname(nickname): + raise ValueError('%s has an invalid nickname: %s' % (nickname, nickname)) + elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): + raise ValueError('%s has an invalid IPv6 address: %s' % (nickname, orport_v6[0])) + elif orport_v6 and not connection.is_valid_port(orport_v6[1]): + raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (nickname, orport_v6[1])) + elif v3ident and not tor_tools.is_valid_fingerprint(v3ident): + raise ValueError('%s has an invalid v3ident: %s' % (nickname, v3ident)) + + return Authority( + address = address, + or_port = int(or_port), + dir_port = int(dir_port), + fingerprint = fingerprint, + nickname = nickname, + v3ident = v3ident, + ) + + @staticmethod + def _pop_section(lines): + """ + Provides the next authority entry. + """ + + section_lines = [] + + if lines: + section_lines.append(lines.pop(0)) + + while lines and lines[0].startswith(' '): + section_lines.append(lines.pop(0)) + + return section_lines + + def __hash__(self): + return _hash_attr(self, 'nickname', 'v3ident', 'is_bandwidth_authority', parent = Directory) + + def __eq__(self, other): + return hash(self) == hash(other) if isinstance(other, Authority) else False + + def __ne__(self, other): + return not self == other + + +class Fallback(Directory): + """ + Particularly stable relays tor can instead of authorities when + bootstrapping. These relays are `hardcoded in tor + https://gitweb.torproject.org/tor.git/tree/src/or/fallback_dirs.inc`_. + + For example, the following checks the performance of tor's fallback directories... + + :: + + import time + from stem.descriptor.remote import DescriptorDownloader + from stem.directory import Fallback + + downloader = DescriptorDownloader() + + for fallback in Fallback.from_cache().values(): + start = time.time() + downloader.get_consensus(endpoints = [(fallback.address, fallback.dir_port)]).run() + print('Downloading the consensus took %0.2f from %s' % (time.time() - start, fallback.fingerprint)) + + :: + + % python example.py + Downloading the consensus took 5.07 from 0AD3FA884D18F89EEA2D89C019379E0E7FD94417 + Downloading the consensus took 3.59 from C871C91489886D5E2E94C13EA1A5FDC4B6DC5204 + Downloading the consensus took 4.16 from 74A910646BCEEFBCD2E874FC1DC997430F968145 + ... + + .. versionadded:: 1.5.0 + + .. versionchanged:: 1.7.0 + Added the nickname, has_extrainfo, and header attributes which are part of + the `second version of the fallback directories + https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html`_. + + :var bool has_extrainfo: **True** if the relay should be able to provide + extrainfo descriptors, **False** otherwise. + :var str orport_v6: **(address, port)** tuple for the directory's IPv6 + ORPort, or **None** if it doesn't have one + :var dict header: metadata about the fallback directory file this originated from + """ + + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, has_extrainfo = False, orport_v6 = None, header = None): + super(Fallback, self).__init__(address, or_port, dir_port, fingerprint, nickname) + + self.has_extrainfo = has_extrainfo + self.orport_v6 = orport_v6 + self.header = header if header else OrderedDict() + + @staticmethod + def from_cache(path = CACHE_PATH): + conf = stem.util.conf.Config() + conf.load(path) + headers = OrderedDict([(k.split('.', 1)[1], conf.get(k)) for k in conf.keys() if k.startswith('header.')]) + + results = {} + + for fingerprint in set([key.split('.')[0] for key in conf.keys()]): + if fingerprint in ('tor_commit', 'stem_commit', 'header'): + continue + + attr = {} + + for attr_name in ('address', 'or_port', 'dir_port', 'nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): + key = '%s.%s' % (fingerprint, attr_name) + attr[attr_name] = conf.get(key) + + if not attr[attr_name] and attr_name not in ('nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): + raise IOError("'%s' is missing from %s" % (key, CACHE_PATH)) + + if not connection.is_valid_ipv4_address(attr['address']): + raise IOError("'%s.address' was an invalid IPv4 address (%s)" % (fingerprint, attr['address'])) + elif not connection.is_valid_port(attr['or_port']): + raise IOError("'%s.or_port' was an invalid port (%s)" % (fingerprint, attr['or_port'])) + elif not connection.is_valid_port(attr['dir_port']): + raise IOError("'%s.dir_port' was an invalid port (%s)" % (fingerprint, attr['dir_port'])) + elif attr['nickname'] and not tor_tools.is_valid_nickname(attr['nickname']): + raise IOError("'%s.nickname' was an invalid nickname (%s)" % (fingerprint, attr['nickname'])) + elif attr['orport6_address'] and not connection.is_valid_ipv6_address(attr['orport6_address']): + raise IOError("'%s.orport6_address' was an invalid IPv6 address (%s)" % (fingerprint, attr['orport6_address'])) + elif attr['orport6_port'] and not connection.is_valid_port(attr['orport6_port']): + raise IOError("'%s.orport6_port' was an invalid port (%s)" % (fingerprint, attr['orport6_port'])) + + if attr['orport6_address'] and attr['orport6_port']: + orport_v6 = (attr['orport6_address'], int(attr['orport6_port'])) + else: + orport_v6 = None + + results[fingerprint] = Fallback( + address = attr['address'], + or_port = int(attr['or_port']), + dir_port = int(attr['dir_port']), + fingerprint = fingerprint, + nickname = attr['nickname'], + has_extrainfo = attr['has_extrainfo'] == 'true', + orport_v6 = orport_v6, + header = headers, + ) + + return results + + @staticmethod + def from_remote(timeout = 60): + try: + lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() + except: + exc = sys.exc_info()[1] + raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc)) + + if not lines: + raise IOError('%s did not have any content' % GITWEB_FALLBACK_URL) + elif lines[0] != '/* type=fallback */': + raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_URL) + + # header metadata + + header = {} + + for line in Fallback._pop_section(lines): + mapping = FALLBACK_MAPPING.match(line) + + if mapping: + header[mapping.group(1)] = mapping.group(2) + else: + raise IOError('Malformed fallback directory header line: %s' % line) + + # human readable comments + + Fallback._pop_section(lines) + + # content, everything remaining are fallback directories + + results = {} + + while lines: + section = Fallback._pop_section(lines) + + if section: + try: + fallback = Fallback._from_str('\n'.join(section)) + fallback.header = header + results[fallback.fingerprint] = fallback + except ValueError as exc: + raise IOError(str(exc)) + + return results + + @staticmethod + def _from_str(content): + """ + Parses a fallback from its textual representation. For example... + + :: + + "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" + " ipv6=[2a01:4f8:162:51e2::2]:9001" + /* nickname=rueckgrat */ + /* extrainfo=1 */ + + :param str content: text to parse + + :returns: :class:`~stem.directory.Fallback` in the text + + :raises: **ValueError** if content is malformed + """ + + if isinstance(content, bytes): + content = str_tools._to_unicode(content) + + matches = {} + + for line in content.splitlines(): + for matcher in (FALLBACK_ADDR, FALLBACK_NICKNAME, FALLBACK_EXTRAINFO, FALLBACK_IPV6): + m = matcher.match(line) + + if m: + match_groups = m.groups() + matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] + + if FALLBACK_ADDR not in matches: + raise ValueError('Malformed fallback address line:\n\n%s' % content) + + address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR] + nickname = matches.get(FALLBACK_NICKNAME) + has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1' + orport_v6 = matches.get(FALLBACK_IPV6) + + if not connection.is_valid_ipv4_address(address): + raise ValueError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) + elif not connection.is_valid_port(or_port): + raise ValueError('%s has an invalid or_port: %s' % (fingerprint, or_port)) + elif not connection.is_valid_port(dir_port): + raise ValueError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) + elif not tor_tools.is_valid_fingerprint(fingerprint): + raise ValueError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) + elif nickname and not tor_tools.is_valid_nickname(nickname): + raise ValueError('%s has an invalid nickname: %s' % (fingerprint, nickname)) + elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): + raise ValueError('%s has an invalid IPv6 address: %s' % (fingerprint, orport_v6[0])) + elif orport_v6 and not connection.is_valid_port(orport_v6[1]): + raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, orport_v6[1])) + + return Fallback( + address = address, + or_port = int(or_port), + dir_port = int(dir_port), + fingerprint = fingerprint, + nickname = nickname, + has_extrainfo = has_extrainfo, + orport_v6 = (orport_v6[0], int(orport_v6[1])) if orport_v6 else None, + ) + + @staticmethod + def _pop_section(lines): + """ + Provides lines up through the next divider. This excludes lines with just a + comma since they're an artifact of these being C strings. + """ + + section_lines = [] + + if lines: + line = lines.pop(0) + + while lines and line != FALLBACK_DIV: + if line.strip() != ',': + section_lines.append(line) + + line = lines.pop(0) + + return section_lines + + @staticmethod + def _write(fallbacks, tor_commit, stem_commit, headers, path = CACHE_PATH): + """ + Persists fallback directories to a location in a way that can be read by + from_cache(). + + :param dict fallbacks: mapping of fingerprints to their fallback directory + :param str tor_commit: tor commit the fallbacks came from + :param str stem_commit: stem commit the fallbacks came from + :param dict headers: metadata about the file these came from + :param str path: location fallbacks will be persisted to + """ + + conf = stem.util.conf.Config() + conf.set('tor_commit', tor_commit) + conf.set('stem_commit', stem_commit) + + for k, v in headers.items(): + conf.set('header.%s' % k, v) + + for directory in sorted(fallbacks.values(), key = lambda x: x.fingerprint): + fingerprint = directory.fingerprint + conf.set('%s.address' % fingerprint, directory.address) + conf.set('%s.or_port' % fingerprint, str(directory.or_port)) + conf.set('%s.dir_port' % fingerprint, str(directory.dir_port)) + conf.set('%s.nickname' % fingerprint, directory.nickname) + conf.set('%s.has_extrainfo' % fingerprint, 'true' if directory.has_extrainfo else 'false') + + if directory.orport_v6: + conf.set('%s.orport6_address' % fingerprint, str(directory.orport_v6[0])) + conf.set('%s.orport6_port' % fingerprint, str(directory.orport_v6[1])) + + conf.save(path) + + def __hash__(self): + return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'has_extrainfo', 'orport_v6', 'header', parent = Directory) + + def __eq__(self, other): + return hash(self) == hash(other) if isinstance(other, Fallback) else False + + def __ne__(self, other): + return not self == other + + +def _fallback_directory_differences(previous_directories, new_directories): + """ + Provides a description of how fallback directories differ. + """ + + lines = [] + + added_fp = set(new_directories.keys()).difference(previous_directories.keys()) + removed_fp = set(previous_directories.keys()).difference(new_directories.keys()) + + for fp in added_fp: + directory = new_directories[fp] + orport_v6 = '%s:%s' % directory.orport_v6 if directory.orport_v6 else '[none]' + + lines += [ + '* Added %s as a new fallback directory:' % directory.fingerprint, + ' address: %s' % directory.address, + ' or_port: %s' % directory.or_port, + ' dir_port: %s' % directory.dir_port, + ' nickname: %s' % directory.nickname, + ' has_extrainfo: %s' % directory.has_extrainfo, + ' orport_v6: %s' % orport_v6, + '', + ] + + for fp in removed_fp: + lines.append('* Removed %s as a fallback directory' % fp) + + for fp in new_directories: + if fp in added_fp or fp in removed_fp: + continue # already discussed these + + previous_directory = previous_directories[fp] + new_directory = new_directories[fp] + + if previous_directory != new_directory: + for attr in ('address', 'or_port', 'dir_port', 'fingerprint', 'orport_v6'): + old_attr = getattr(previous_directory, attr) + new_attr = getattr(new_directory, attr) + + if old_attr != new_attr: + lines.append('* Changed the %s of %s from %s to %s' % (attr, fp, old_attr, new_attr)) + + return '\n'.join(lines) + + +DIRECTORY_AUTHORITIES = { + 'moria1': Authority( + nickname = 'moria1', + address = '128.31.0.39', + or_port = 9101, + dir_port = 9131, + is_bandwidth_authority = True, + fingerprint = '9695DFC35FFEB861329B9F1AB04C46397020CE31', + v3ident = 'D586D18309DED4CD6D57C18FDB97EFA96D330566', + ), + 'tor26': Authority( + nickname = 'tor26', + address = '86.59.21.38', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = False, + fingerprint = '847B1F850344D7876491A54892F904934E4EB85D', + v3ident = '14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', + ), + 'dizum': Authority( + nickname = 'dizum', + address = '194.109.206.212', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = False, + fingerprint = '7EA6EAD6FD83083C538F44038BBFA077587DD755', + v3ident = 'E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58', + ), + 'gabelmoo': Authority( + nickname = 'gabelmoo', + address = '131.188.40.189', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = True, + fingerprint = 'F2044413DAC2E02E3D6BCF4735A19BCA1DE97281', + v3ident = 'ED03BB616EB2F60BEC80151114BB25CEF515B226', + ), + 'dannenberg': Authority( + nickname = 'dannenberg', + address = '193.23.244.244', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = False, + fingerprint = '7BE683E65D48141321C5ED92F075C55364AC7123', + v3ident = '0232AF901C31A04EE9848595AF9BB7620D4C5B2E', + ), + 'maatuska': Authority( + nickname = 'maatuska', + address = '171.25.193.9', + or_port = 80, + dir_port = 443, + is_bandwidth_authority = True, + fingerprint = 'BD6A829255CB08E66FBE7D3748363586E46B3810', + v3ident = '49015F787433103580E3B66A1707A00E60F2D15B', + ), + 'Faravahar': Authority( + nickname = 'Faravahar', + address = '154.35.175.225', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = True, + fingerprint = 'CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC', + v3ident = 'EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97', + ), + 'longclaw': Authority( + nickname = 'longclaw', + address = '199.58.81.140', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = False, + fingerprint = '74A910646BCEEFBCD2E874FC1DC997430F968145', + v3ident = '23D15D965BC35114467363C165C4F724B64B4F66', + ), + 'bastet': Authority( + nickname = 'bastet', + address = '204.13.164.118', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = True, + fingerprint = '24E2F139121D4394C54B5BCC368B3B411857C413', + v3ident = '27102BC123E7AF1D4741AE047E160C91ADC76B21', + ), + 'Bifroest': Authority( + nickname = 'Bifroest', + address = '37.218.247.217', + or_port = 443, + dir_port = 80, + is_bandwidth_authority = False, + fingerprint = '1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1', + v3ident = None, # does not vote in the consensus + ), +} diff --git a/stem/descriptor/fallback_directories.cfg b/stem/fallback_directories.cfg similarity index 100% rename from stem/descriptor/fallback_directories.cfg rename to stem/fallback_directories.cfg diff --git a/test/unit/tutorial_examples.py b/test/unit/tutorial_examples.py index ecc9b5ca..d94da5bf 100644 --- a/test/unit/tutorial_examples.py +++ b/test/unit/tutorial_examples.py @@ -17,9 +17,9 @@ import stem.prereq
from stem.control import Controller from stem.descriptor.networkstatus import NetworkStatusDocumentV3 -from stem.descriptor.remote import DIRECTORY_AUTHORITIES from stem.descriptor.router_status_entry import RouterStatusEntryV3 from stem.descriptor.server_descriptor import RelayDescriptor +from stem.directory import DIRECTORY_AUTHORITIES from stem.response import ControlMessage from stem.util import str_type