commit 7486132886310873632ffa40a38d883bdc2b5ac7 Author: Damian Johnson atagar@torproject.org Date: Fri Dec 23 10:21:05 2016 -0800

Descriptor protocol support

Adding support for the new protocol descriptor fields...

https://gitweb.torproject.org/torspec.git/commit/?id=eb4fb3c5 --- docs/change_log.rst | 4 ++ stem/descriptor/__init__.py | 75 +++++++++++++++++++++++ stem/descriptor/microdescriptor.py | 9 +++ stem/descriptor/networkstatus.py | 30 +++++++++ stem/descriptor/router_status_entry.py | 9 +++ stem/descriptor/server_descriptor.py | 9 +++ test/settings.cfg | 1 + test/unit/descriptor/microdescriptor.py | 10 +++ test/unit/descriptor/networkstatus/document_v3.py | 22 +++++++ test/unit/descriptor/protocol.py | 46 ++++++++++++++ test/unit/descriptor/router_status_entry.py | 5 ++ test/unit/descriptor/server_descriptor.py | 9 +++ 12 files changed, 229 insertions(+)

diff --git a/docs/change_log.rst b/docs/change_log.rst index d6ae108..713ef51 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -43,6 +43,10 @@ Unreleased The following are only available within Stem's `git repository <download.html>`_.

+ * **Descriptors** + + * Support for protocol descriptor fields (:spec:`eb4fb3c5`) + .. _version_1.5:

Version 1.5 (November 20th, 2016) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index fc44843..208741a 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -38,6 +38,7 @@ Package for parsing and processing descriptor data.

import base64 import codecs +import collections import copy import hashlib import os @@ -86,6 +87,72 @@ DocumentHandler = stem.util.enum.UppercaseEnum( )

+class ProtocolSupport(object): + """ + Protocols supported by a relay. + + .. versionadded:: 1.6.0 + """ + + def __init__(self, keyword, value): + # parses 'protocol' entries like: Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 + + self._entries = OrderedDict() + + for entry in value.split(): + if '=' not in entry: + raise ValueError("Protocol entires are expected to be a series of 'key=value' pairs but was: %s %s" % (keyword, value)) + + k, v = entry.split('=', 1) + + if '-' in v: + min_value, max_value = v.split('-', 1) + else: + min_value = max_value = v + + if not min_value.isdigit() or not max_value.isdigit(): + raise ValueError('Protocol values should be a number or number range, but was: %s %s' % (keyword, value)) + + self._entries[k] = Protocol(k, int(min_value), int(max_value)) + + def is_supported(self, protocol, version = None): + """ + Checks if the given protocol is supported. + + :param str protocol: protocol to check support of (DirCache, HSDir, etc) + :param int version: protocol version to check support of + + :returns: **True** if the protocol is supported, **False** otherwise + """ + + supported = self._entries.get(protocol) + + if not supported: + return False + elif version and version < supported.min_version: + return False + elif version and version > supported.max_version: + return False + else: + return True + + def __iter__(self): + for protocol in self._entries.values(): + yield protocol + + +class Protocol(collections.namedtuple('Protocol', ['name', 'min_version', 'max_version'])): + """ + Individual protocol range supported by a relay. + + .. versionadded:: 1.6.0 + + :var str name: protocol name (ex. DirCache, HSDir, etc) + :var int min_version: minimum protocol supported + :var int max_version: maximum protocol supported + """ + + def parse_file(descriptor_file, descriptor_type = None, validate = False, document_handler = DocumentHandler.ENTRIES, normalize_newlines = None, **kwargs): """ Simple function to read the descriptor contents from a file, providing an @@ -389,6 +456,14 @@ def _parse_forty_character_hex(keyword, attribute): return _parse

+def _parse_protocol_line(keyword, attribute): + def _parse(descriptor, entries): + value = _value(keyword, entries) + setattr(descriptor, attribute, ProtocolSupport(keyword, value)) + + return _parse + + def _parse_key_block(keyword, attribute, expected_block_type, value_attribute = None): def _parse(descriptor, entries): value, block_type, block_contents = entries[keyword][0] diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index d8f8cd0..e1bab9c 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -74,6 +74,7 @@ from stem.descriptor import ( _read_until_keywords, _values, _parse_simple_line, + _parse_protocol_line, _parse_key_block, )

@@ -98,6 +99,7 @@ SINGLE_FIELDS = ( 'family', 'p', 'p6', + 'pr', )

@@ -187,6 +189,7 @@ _parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC K _parse_ntor_onion_key_line = _parse_simple_line('ntor-onion-key', 'ntor_onion_key') _parse_family_line = _parse_simple_line('family', 'family', func = lambda v: v.split(' ')) _parse_p6_line = _parse_simple_line('p6', 'exit_policy_v6', func = lambda v: stem.exit_policy.MicroExitPolicy(v)) +_parse_pr_line = _parse_protocol_line('pr', 'protocols')

class Microdescriptor(Descriptor): @@ -208,6 +211,7 @@ class Microdescriptor(Descriptor): :var hash identifiers: mapping of key types (like rsa1024 or ed25519) to their base64 encoded identity, this is only used for collision prevention (:trac:`11743`) + :var stem.descriptor.ProtocolSupport protocols: supported protocols

:var str identifier: base64 encoded identity digest (**deprecated**, use identifiers instead) @@ -222,6 +226,9 @@ class Microdescriptor(Descriptor): .. versionchanged:: 1.5.0 Added the identifiers attribute, and deprecated identifier and identifier_type since the field can now appear multiple times. + + .. versionchanged:: 1.6.0 + Added the protocols attribute. """

ATTRIBUTES = { @@ -234,6 +241,7 @@ class Microdescriptor(Descriptor): 'identifier_type': (None, _parse_id_line), # deprecated in favor of identifiers 'identifier': (None, _parse_id_line), # deprecated in favor of identifiers 'identifiers': ({}, _parse_id_line), + 'protocols': (None, _parse_pr_line), 'digest': (None, _parse_digest), }

@@ -244,6 +252,7 @@ class Microdescriptor(Descriptor): 'family': _parse_family_line, 'p': _parse_p_line, 'p6': _parse_p6_line, + 'pr': _parse_pr_line, 'id': _parse_id_line, }

diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 7456b37..75ec459 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -73,6 +73,7 @@ from stem.descriptor import ( _parse_if_present, _parse_timestamp_line, _parse_forty_character_hex, + _parse_protocol_line, _parse_key_block, )

@@ -123,6 +124,10 @@ HEADER_STATUS_DOCUMENT_FIELDS = ( ('shared-rand-commit', True, False, False), ('shared-rand-previous-value', True, True, False), ('shared-rand-current-value', True, True, False), + ('recommended-client-protocols', True, True, False), + ('recommended-relay-protocols', True, True, False), + ('required-client-protocols', True, True, False), + ('required-relay-protocols', True, True, False), ('params', True, True, False), )

@@ -756,6 +761,10 @@ _parse_header_server_versions_line = _parse_versions_line('server-versions', 'se _parse_header_known_flags_line = _parse_simple_line('known-flags', 'known_flags', func = lambda v: [entry for entry in v.split(' ') if entry]) _parse_footer_bandwidth_weights_line = _parse_simple_line('bandwidth-weights', 'bandwidth_weights', func = lambda v: _parse_int_mappings('bandwidth-weights', v, True)) _parse_shared_rand_participate_line = _parse_if_present('shared-rand-participate', 'is_shared_randomness_participate') +_parse_recommended_client_protocols_line = _parse_protocol_line('recommended-client-protocols', 'recommended_client_protocols') +_parse_recommended_relay_protocols_line = _parse_protocol_line('recommended-relay-protocols', 'recommended_relay_protocols') +_parse_required_client_protocols_line = _parse_protocol_line('required-client-protocols', 'required_client_protocols') +_parse_required_relay_protocols_line = _parse_protocol_line('required-relay-protocols', 'required_relay_protocols')

class NetworkStatusDocumentV3(NetworkStatusDocument): @@ -812,6 +821,15 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): :var str shared_randomness_current_value: base64 encoded current shared random value

+ :var stem.descriptor.ProtocolSupport recommended_client_protocols: recommended + protocols for clients + :var stem.descriptor.ProtocolSupport recommended_relay_protocols: recommended + protocols for relays + :var stem.descriptor.ProtocolSupport required_client_protocols: required + protocols for clients + :var stem.descriptor.ProtocolSupport required_relay_protocols: required + protocols for relays + ***** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined

@@ -824,6 +842,10 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): shared_randomness_previous_value, shared_randomness_current_reveal_count, and shared_randomness_current_value attributes. + + .. versionchanged:: 1.6.0 + Added the recommended_client_protocols, recommended_relay_protocols, + required_client_protocols, and required_relay_protocols. """

ATTRIBUTES = { @@ -851,6 +873,10 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): 'shared_randomness_previous_value': (None, _parse_shared_rand_previous_value), 'shared_randomness_current_reveal_count': (None, _parse_shared_rand_current_value), 'shared_randomness_current_value': (None, _parse_shared_rand_current_value), + 'recommended_client_protocols': (None, _parse_recommended_client_protocols_line), + 'recommended_relay_protocols': (None, _parse_recommended_relay_protocols_line), + 'required_client_protocols': (None, _parse_required_client_protocols_line), + 'required_relay_protocols': (None, _parse_required_relay_protocols_line), 'params': ({}, _parse_header_parameters_line),

'signatures': ([], _parse_footer_directory_signature_line), @@ -876,6 +902,10 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): 'shared-rand-commit': _parsed_shared_rand_commit, 'shared-rand-previous-value': _parse_shared_rand_previous_value, 'shared-rand-current-value': _parse_shared_rand_current_value, + 'recommended-client-protocols': _parse_recommended_client_protocols_line, + 'recommended-relay-protocols': _parse_recommended_relay_protocols_line, + 'required-client-protocols': _parse_required_client_protocols_line, + 'required-relay-protocols': _parse_required_relay_protocols_line, 'params': _parse_header_parameters_line, }

diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index bf146a9..abdefa2 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -32,9 +32,12 @@ from stem.descriptor import ( _value, _values, _get_descriptor_components, + _parse_protocol_line, _read_until_keywords, )

+_parse_pr_line = _parse_protocol_line('pr', 'protocols') +

def _parse_file(document_file, validate, entry_class, entry_keyword = 'r', start_position = None, end_position = None, section_end_keywords = (), extra_args = ()): """ @@ -560,6 +563,7 @@ class RouterStatusEntryV3(RouterStatusEntry): information that isn't yet recognized

:var stem.exit_policy.MicroExitPolicy exit_policy: router's exit policy + :var stem.descriptor.ProtocolSupport protocols: supported protocols

:var list microdescriptor_hashes: ***** tuples of two values, the list of consensus methods for generating a set of digests and the 'algorithm => @@ -570,6 +574,9 @@ class RouterStatusEntryV3(RouterStatusEntry):

.. versionchanged:: 1.5.0 Added the identifier and identifier_type attributes. + + .. versionchanged:: 1.6.0 + Added the protocols attribute. """

ATTRIBUTES = dict(RouterStatusEntry.ATTRIBUTES, **{ @@ -584,6 +591,7 @@ class RouterStatusEntryV3(RouterStatusEntry): 'unrecognized_bandwidth_entries': ([], _parse_w_line),

'exit_policy': (None, _parse_p_line), + 'pr': (None, _parse_pr_line), 'microdescriptor_hashes': ([], _parse_m_line), })

@@ -591,6 +599,7 @@ class RouterStatusEntryV3(RouterStatusEntry): 'a': _parse_a_line, 'w': _parse_w_line, 'p': _parse_p_line, + 'pr': _parse_pr_line, 'id': _parse_id_line, 'm': _parse_m_line, }) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 9f7d8d1..ffd0183 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -58,6 +58,7 @@ from stem.descriptor import ( _parse_bytes_line, _parse_timestamp_line, _parse_forty_character_hex, + _parse_protocol_line, _parse_key_block, )

@@ -96,6 +97,7 @@ SINGLE_FIELDS = ( 'protocols', 'allow-single-hop-exits', 'tunnelled-dir-server', + 'proto', 'onion-key-crosscert', 'ntor-onion-key', 'ntor-onion-key-crosscert', @@ -394,6 +396,7 @@ _parse_write_history_line = functools.partial(_parse_history_line, 'write-histor _parse_ipv6_policy_line = _parse_simple_line('ipv6-policy', 'exit_policy_v6', func = lambda v: stem.exit_policy.MicroExitPolicy(v)) _parse_allow_single_hop_exits_line = _parse_if_present('allow-single-hop-exits', 'allow_single_hop_exits') _parse_tunneled_dir_server_line = _parse_if_present('tunnelled-dir-server', 'allow_tunneled_dir_requests') +_parse_proto_line = _parse_protocol_line('proto', 'protocols') _parse_caches_extra_info_line = _parse_if_present('caches-extra-info', 'extra_info_cache') _parse_family_line = _parse_simple_line('family', 'family', func = lambda v: set(v.split(' '))) _parse_eventdns_line = _parse_simple_line('eventdns', 'eventdns', func = lambda v: v == '1') @@ -447,6 +450,7 @@ class ServerDescriptor(Descriptor): :var list or_addresses: ***** alternative for our address/or_port attributes, each entry is a tuple of the form (address (**str**), port (**int**), is_ipv6 (**bool**)) + :var stem.descriptor.ProtocolSupport protocols: supported protocols

**Deprecated**, moved to extra-info descriptor...

@@ -463,6 +467,9 @@ class ServerDescriptor(Descriptor):

.. versionchanged:: 1.5.0 Added the allow_tunneled_dir_requests attribute. + + .. versionchanged:: 1.6.0 + Added the protocols attribute. """

ATTRIBUTES = { @@ -493,6 +500,7 @@ class ServerDescriptor(Descriptor): 'hibernating': (False, _parse_hibernating_line), 'allow_single_hop_exits': (False, _parse_allow_single_hop_exits_line), 'allow_tunneled_dir_requests': (False, _parse_tunneled_dir_server_line), + 'protocols': ({}, _parse_proto_line), 'extra_info_cache': (False, _parse_caches_extra_info_line), 'extra_info_digest': (None, _parse_extrainfo_digest_line), 'hidden_service_dir': (None, _parse_hidden_service_dir_line), @@ -528,6 +536,7 @@ class ServerDescriptor(Descriptor): 'ipv6-policy': _parse_ipv6_policy_line, 'allow-single-hop-exits': _parse_allow_single_hop_exits_line, 'tunnelled-dir-server': _parse_tunneled_dir_server_line, + 'proto': _parse_proto_line, 'caches-extra-info': _parse_caches_extra_info_line, 'family': _parse_family_line, 'eventdns': _parse_eventdns_line, diff --git a/test/settings.cfg b/test/settings.cfg index 4913202..627c270 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -179,6 +179,7 @@ test.unit_tests |test.unit.descriptor.export.TestExport |test.unit.descriptor.reader.TestDescriptorReader |test.unit.descriptor.remote.TestDescriptorDownloader +|test.unit.descriptor.protocol.TestProtocol |test.unit.descriptor.server_descriptor.TestServerDescriptor |test.unit.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor |test.unit.descriptor.microdescriptor.TestMicrodescriptor diff --git a/test/unit/descriptor/microdescriptor.py b/test/unit/descriptor/microdescriptor.py index 10ccd15..9d4d4fc 100644 --- a/test/unit/descriptor/microdescriptor.py +++ b/test/unit/descriptor/microdescriptor.py @@ -98,6 +98,7 @@ class TestMicrodescriptor(unittest.TestCase): self.assertEqual({}, desc.identifiers) self.assertEqual(None, desc.identifier_type) self.assertEqual(None, desc.identifier) + self.assertEqual(None, desc.protocols) self.assertEqual([], desc.get_unrecognized_lines())

def test_unrecognized_line(self): @@ -165,6 +166,15 @@ class TestMicrodescriptor(unittest.TestCase): desc = get_microdescriptor({'p': 'accept 80,110,143,443'}) self.assertEqual(stem.exit_policy.MicroExitPolicy('accept 80,110,143,443'), desc.exit_policy)

+ def test_protocols(self): + """ + Basic check for 'pr' lines. + """ + + desc = get_microdescriptor({'pr': 'Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2'}) + self.assertEqual(10, len(list(desc.protocols))) + self.assertTrue(desc.protocols.is_supported('Desc')) + def test_identifier(self): """ Basic check for 'id' lines. diff --git a/test/unit/descriptor/networkstatus/document_v3.py b/test/unit/descriptor/networkstatus/document_v3.py index aae12a6..09faf43 100644 --- a/test/unit/descriptor/networkstatus/document_v3.py +++ b/test/unit/descriptor/networkstatus/document_v3.py @@ -343,6 +343,10 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEqual(None, document.shared_randomness_previous_value) self.assertEqual(None, document.shared_randomness_current_reveal_count) self.assertEqual(None, document.shared_randomness_current_value) + self.assertEqual(None, document.recommended_client_protocols) + self.assertEqual(None, document.recommended_relay_protocols) + self.assertEqual(None, document.required_client_protocols) + self.assertEqual(None, document.required_relay_protocols) self.assertEqual(DEFAULT_PARAMS, document.params) self.assertEqual((), document.directory_authorities) self.assertEqual({}, document.bandwidth_weights) @@ -926,6 +930,24 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEqual(None, document.shared_randomness_current_reveal_count) self.assertEqual(None, document.shared_randomness_current_value)

+ def test_parameters(self): + """ + Parses the parameters attributes. + """ + + document = get_network_status_document_v3(OrderedDict([ + ('vote-status', 'vote'), + ('recommended-client-protocols', 'HSDir=1 HSIntro=3'), + ('recommended-relay-protocols', 'Cons=1 Desc=1'), + ('required-client-protocols', 'HSRend=1 Link=1-4 LinkAuth=1 Microdesc=1'), + ('required-relay-protocols', 'DirCache=1'), + ])) + + self.assertEqual(2, len(list(document.recommended_client_protocols))) + self.assertEqual(2, len(list(document.recommended_relay_protocols))) + self.assertEqual(4, len(list(document.required_client_protocols))) + self.assertEqual(1, len(list(document.required_relay_protocols))) + def test_params(self): """ General testing for the 'params' line, exercising the happy cases. diff --git a/test/unit/descriptor/protocol.py b/test/unit/descriptor/protocol.py new file mode 100644 index 0000000..a47d7c6 --- /dev/null +++ b/test/unit/descriptor/protocol.py @@ -0,0 +1,46 @@ +""" +Unit tessts for the stem.descriptor.ProtocolSupport class. +""" + +import unittest + +from stem.descriptor import Protocol, ProtocolSupport + + +class TestProtocol(unittest.TestCase): + def test_parsing(self): + expected = [ + Protocol(name = 'Desc', min_version = 1, max_version = 1), + Protocol(name = 'Link', min_version = 1, max_version = 4), + Protocol(name = 'Microdesc', min_version = 1, max_version = 1), + Protocol(name = 'Relay', min_version = 1, max_version = 2), + ] + + self.assertEqual(expected, list(ProtocolSupport('pr', 'Desc=1 Link=1-4 Microdesc=1 Relay=1-2'))) + + def test_parse_with_no_mapping(self): + try: + ProtocolSupport('pr', 'Desc Link=1-4') + self.fail('Did not raise expected exception') + except ValueError as exc: + self.assertEqual("Protocol entires are expected to be a series of 'key=value' pairs but was: pr Desc Link=1-4", str(exc)) + + def test_parse_with_non_int_version(self): + try: + ProtocolSupport('pr', 'Desc=hi Link=1-4') + self.fail('Did not raise expected exception') + except ValueError as exc: + self.assertEqual('Protocol values should be a number or number range, but was: pr Desc=hi Link=1-4', str(exc)) + + def test_is_supported(self): + protocol = ProtocolSupport('pr', 'Desc=1 Link=2-4 Microdesc=1 Relay=1-2') + self.assertFalse(protocol.is_supported('NoSuchProtocol')) + self.assertFalse(protocol.is_supported('Desc', 2)) + self.assertTrue(protocol.is_supported('Desc')) + self.assertTrue(protocol.is_supported('Desc', 1)) + + self.assertFalse(protocol.is_supported('Link', 1)) + self.assertTrue(protocol.is_supported('Link', 2)) + self.assertTrue(protocol.is_supported('Link', 3)) + self.assertTrue(protocol.is_supported('Link', 4)) + self.assertFalse(protocol.is_supported('Link', 5)) diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py index 25ba99d..b8597be 100644 --- a/test/unit/descriptor/router_status_entry.py +++ b/test/unit/descriptor/router_status_entry.py @@ -482,6 +482,11 @@ class TestRouterStatusEntry(unittest.TestCase): content = get_router_status_entry_v3({'s': s_line}, content = True) self._expect_invalid_attr(content, 'flags', expected)

+ def test_protocols(self): + desc = get_router_status_entry_v3({'pr': 'Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2'}) + self.assertEqual(10, len(list(desc.protocols))) + self.assertTrue(desc.protocols.is_supported('Desc')) + def test_versions(self): """ Handles a variety of version inputs. diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 834d1ea..d84ad48 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -657,6 +657,15 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= desc = get_relay_server_descriptor({'ipv6-policy': 'accept 22-23,53,80,110'}) self.assertEqual(expected, desc.exit_policy_v6)

+ def test_protocols(self): + """ + Checks a 'proto' line. + """ + + desc = get_relay_server_descriptor({'proto': 'Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2'}) + self.assertEqual(10, len(list(desc.protocols))) + self.assertTrue(desc.protocols.is_supported('Desc')) + def test_ntor_onion_key(self): """ Checks a 'ntor-onion-key' line.