commit ea55eaa28c1bf6a3c97cf678e7856f3bd8f23be8 Author: Damian Johnson atagar@torproject.org Date: Sat Dec 23 13:10:24 2017 -0800

Fallback directory v2 parsing

Adding support for the proposed v2 fallback directory format...

https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html

This isn't live yet and I won't be checking persistance support until it is, but good starting point. Remaining thoughs are...

1. The 'extrainfo=' line is optional but we're using it as a delimiter, so this won't work if it's omitted.

2. It would be nice if the document explicitly said its format version.

3. I suspect reading persisted has_extrainfo attributes mistakenly provide a str rather than bool. As mentioned above I'll be checking persistence once it's live. --- stem/descriptor/remote.py | 98 ++++++++++++++++++++++++++++++++++++++++-- test/unit/descriptor/remote.py | 32 ++++++++++++-- 2 files changed, 124 insertions(+), 6 deletions(-)

diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index dbba6857..9124a45f 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -939,12 +939,25 @@ class FallbackDirectory(Directory):

.. versionadded:: 1.5.0

+ .. versionchanged:: 1.7.0 + Added the nickname and has_extrainfo attributes. + + .. versionchanged:: 1.7.0 + Support for parsing `second version of the fallback directories + https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html`_. + + :var str nickname: relay nickname + :var bool has_extrainfo: **True** if the relay should be able to provide + extrainfo descriptors, **False** otherwise. :var str orport_v6: **(address, port)** tuple for the directory's IPv6 ORPort, or **None** if it doesn't have one """

- def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, orport_v6 = None): + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, has_extrainfo = False, orport_v6 = None): super(FallbackDirectory, self).__init__(address, or_port, dir_port, fingerprint) + + self.nickname = nickname + self.has_extrainfo = has_extrainfo self.orport_v6 = orport_v6

@staticmethod @@ -971,11 +984,11 @@ class FallbackDirectory(Directory):

attr = {}

- for attr_name in ('address', 'or_port', 'dir_port', 'orport6_address', 'orport6_port'): + for attr_name in ('address', 'or_port', 'dir_port', 'nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): key = '%s.%s' % (fingerprint, attr_name) attr[attr_name] = conf.get(key)

- if not attr[attr_name] and not attr_name.startswith('orport6_'): + if not attr[attr_name] and attr_name not in ('nickname', 'has_extrainfo', 'orport6_address', 'orport6_port'): raise IOError("'%s' is missing from %s" % (key, CACHE_PATH))

if not connection.is_valid_ipv4_address(attr['address']): @@ -984,6 +997,8 @@ class FallbackDirectory(Directory): raise IOError("'%s.or_port' was an invalid port (%s)" % (fingerprint, attr['or_port'])) elif not connection.is_valid_port(attr['dir_port']): raise IOError("'%s.dir_port' was an invalid port (%s)" % (fingerprint, attr['dir_port'])) + elif attr['nickname'] and not connection.is_valid_nickname(attr['nickname']): + raise IOError("'%s.nickname' was an invalid nickname (%s)" % (fingerprint, attr['nickname'])) elif attr['orport6_address'] and not connection.is_valid_ipv6_address(attr['orport6_address']): raise IOError("'%s.orport6_address' was an invalid IPv6 address (%s)" % (fingerprint, attr['orport6_address'])) elif attr['orport6_port'] and not connection.is_valid_port(attr['orport6_port']): @@ -1034,12 +1049,21 @@ class FallbackDirectory(Directory): exc = sys.exc_info()[1] raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_DIR_URL, exc))

+ if '/* nickname=' in fallback_dir_page: + return FallbackDirectory._parse_v2(fallback_dir_page) + else: + return FallbackDirectory._parse_v1(fallback_dir_page) + + @staticmethod + def _parse_v1(fallback_dir_page): # Example of an entry... # # "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" # " ipv6=[2a03:b0c0:0:1010::a4:b001]:9001" # " weight=43680",

+ # TODO: this method can be removed once gitweb provides a v2 formatted document + results, attr = {}, {}

for line in fallback_dir_page.splitlines(): @@ -1087,6 +1111,74 @@ class FallbackDirectory(Directory):

return results

+ @staticmethod + def _parse_v2(fallback_dir_page): + # Example of an entry... + # + # "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" + # " ipv6=[2a01:4f8:162:51e2::2]:9001" + # /* nickname=rueckgrat */ + # /* extrainfo=1 */ + + results, attr = {}, {} + + for line in fallback_dir_page.splitlines(): + addr_line_match = re.match('"([\d.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line) + nickname_match = re.match('/* nickname=(\S+) */', line) + has_extrainfo_match = re.match('/* extrainfo=([0-1]) */', line) + ipv6_line_match = re.match('" ipv6=[([\da-f:]+)]:(\d+)"', line) + + if addr_line_match: + address, dir_port, or_port, fingerprint = addr_line_match.groups() + + if not connection.is_valid_ipv4_address(address): + raise IOError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) + elif not connection.is_valid_port(or_port): + raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port)) + elif not connection.is_valid_port(dir_port): + raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) + elif not tor_tools.is_valid_fingerprint(fingerprint): + raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) + + attr = { + 'address': address, + 'or_port': int(or_port), + 'dir_port': int(dir_port), + 'fingerprint': fingerprint, + } + elif ipv6_line_match: + address, port = ipv6_line_match.groups() + + if not connection.is_valid_ipv6_address(address): + raise IOError('%s has an invalid IPv6 address: %s' % (fingerprint, address)) + elif not connection.is_valid_port(port): + raise IOError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, port)) + + attr['orport_v6'] = (address, int(port)) + elif nickname_match: + nickname = nickname_match.group(1) + + if not tor_tools.is_valid_nickname(nickname): + raise IOError('%s has an invalid nickname: %s' % (fingerprint, nickname)) + + attr['nickname'] = nickname + elif has_extrainfo_match: + attr['has_extrainfo'] = has_extrainfo_match.group(1) == '1' + + results[attr.get('fingerprint')] = FallbackDirectory( + address = attr.get('address'), + or_port = attr.get('or_port'), + dir_port = attr.get('dir_port'), + fingerprint = attr.get('fingerprint'), + nickname = attr.get('nickname'), + has_extrainfo = attr.get('has_extrainfo', False), + orport_v6 = attr.get('orport_v6'), + ) + + attr = {} + + return results + def __hash__(self): return _hash_attr(self, 'orport_v6', parent = Directory)

diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 1b818b67..69ffd8e3 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -58,7 +58,7 @@ iO3EUE0AEYah2W9gdz8t+i3Dtr0zgqLS841GC/TyDKCm+MKmN8d098qnwK0NGF9q -----END SIGNATURE----- """

-FALLBACK_DIR_CONTENT = b"""\ +FALLBACK_DIR_CONTENT_V1 = b"""\ /* Trial fallbacks for 0.2.8.1-alpha with ADDRESS_AND_PORT_STABLE_DAYS = 30 * This works around an issue where relays post a descriptor without a DirPort * when restarted. If these relays stay up, they will have been up for 120 days @@ -70,6 +70,13 @@ FALLBACK_DIR_CONTENT = b"""\ " weight=43680", """

+FALLBACK_DIR_CONTENT_V2 = b"""\ +"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" +" ipv6=[2a01:4f8:162:51e2::2]:9001" +/* nickname=rueckgrat */ +/* extrainfo=1 */ +""" +

class TestDescriptorDownloader(unittest.TestCase): @patch(URL_OPEN) @@ -178,8 +185,8 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual('5.39.92.199', fallback_directories['0BEA4A88D069753218EAAAD6D22EA87B9A1319D6'].address)

@patch(URL_OPEN) - def test_fallback_directories_from_remote(self, urlopen_mock): - urlopen_mock.return_value = io.BytesIO(FALLBACK_DIR_CONTENT) + def test_fallback_directories_from_remote_v1(self, urlopen_mock): + urlopen_mock.return_value = io.BytesIO(FALLBACK_DIR_CONTENT_V1) fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote()

expected = { @@ -199,3 +206,22 @@ class TestDescriptorDownloader(unittest.TestCase): }

self.assertEqual(expected, fallback_directories) + + @patch(URL_OPEN) + def test_fallback_directories_from_remote_v2(self, urlopen_mock): + urlopen_mock.return_value = io.BytesIO(FALLBACK_DIR_CONTENT_V2) + fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() + + expected = { + '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.descriptor.remote.FallbackDirectory( + address = '5.9.110.236', + or_port = 9001, + dir_port = 9030, + fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', + nickname = 'rueckgrat', + has_extrainfo = True, + orport_v6 = ('2a01:4f8:162:51e2::2', 9001), + ), + } + + self.assertEqual(expected, fallback_directories)