commit bccda333d40ba6a17131725475f9fdb025c4690a Author: Damian Johnson atagar@torproject.org Date: Sat Jul 2 20:18:04 2016 -0700

Support parsing shared randomness

Adding support for the new shared randomness parameters in the consenuses and votes...

https://gitweb.torproject.org/torspec.git/commit/?id=9949f64 --- docs/change_log.rst | 1 + stem/descriptor/networkstatus.py | 102 +++++++++++++++++++++- test/mocking.py | 8 +- test/unit/descriptor/networkstatus/document_v3.py | 89 +++++++++++++++++++ 4 files changed, 197 insertions(+), 3 deletions(-)

diff --git a/docs/change_log.rst b/docs/change_log.rst index 3ef8900..1713360 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -66,6 +66,7 @@ The following are only available within Stem's `git repository * `Shorthand functions for stem.descriptor.remote <api/descriptor/remote.html#stem.descriptor.remote.get_instance>`_ * Added `fallback directory information <api/descriptor/remote.html#stem.descriptor.remote.FallbackDirectory>`_. * Support for ed25519 descriptor fields (:spec:`5a79d67`) + * Added consensus and vote's new shared randomness attributes (:spec:`9949f64`) * Added server descriptor's new allow_tunneled_dir_requests attribute (:spec:`8bc30d6`) * Server descriptor validation fails with 'extra-info-digest line had an invalid value' from additions in proposal 228 (:trac:`16227`) * :class:`~stem.descriptor.server_descriptor.BridgeDescriptor` now has 'ntor_onion_key' like its unsanitized counterparts diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index dcbfc5d..815e5f5 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -70,6 +70,7 @@ from stem.descriptor import ( _read_until_keywords, _value, _parse_simple_line, + _parse_if_present, _parse_timestamp_line, _parse_forty_character_hex, _parse_key_block, @@ -118,6 +119,10 @@ HEADER_STATUS_DOCUMENT_FIELDS = ( ('package', True, True, False), ('known-flags', True, True, True), ('flag-thresholds', True, False, False), + ('shared-rand-participate', True, False, False), + ('shared-rand-commit', True, False, False), + ('shared-rand-previous-value', True, True, False), + ('shared-rand-current-value', True, True, False), ('params', True, True, False), )

@@ -196,6 +201,7 @@ PARAM_RANGE = { 'GuardLifetime': (2592000, 157766400), # min: 30 days, max: 1826 days 'NumNTorsPerTAP': (1, 100000), 'AllowNonearlyExtend': (0, 1), + 'AuthDirNumSRVAgreements': (1, MAX_PARAM), }

@@ -210,6 +216,19 @@ class PackageVersion(collections.namedtuple('PackageVersion', ['name', 'version' """

+class SharedRandomnessCommitment(collections.namedtuple('SharedRandomnessCommitment', ['version', 'algorithm', 'identity', 'commit', 'reveal'])): + """ + Directory authority's commitment for generating the next shared random value. + + :var int version: shared randomness protocol version + :var str algorithm: hash algorithm used to make the commitment + :var str identity: authority's sha1 identity fingerprint + :var str commit: base64 encoded commitment hash to the shared random value + :var str reveal: base64 encoded commitment to the shared random value, + **None** of not provided + """ + + def _parse_file(document_file, document_type = None, validate = False, is_microdescriptor = False, document_handler = DocumentHandler.ENTRIES, **kwargs): """ Parses a network status and iterates over the RouterStatusEntry in it. The @@ -681,6 +700,54 @@ def _parse_package_line(descriptor, entries): descriptor.packages = package_versions

+def _parsed_shared_rand_commit(descriptor, entries): + # "shared-rand-commit" Version AlgName Identity Commit [Reveal] + + commitments = [] + + for value, _, _ in entries['shared-rand-commit']: + value_comp = value.split() + + if len(value_comp) < 4: + raise ValueError("'shared-rand-commit' must at least have a 'Version AlgName Identity Commit': %s" % value) + + version, algorithm, identity, commit = value_comp[:4] + reveal = value_comp[4] if len(value_comp) >= 5 else None + + if not version.isdigit(): + raise ValueError("The version on our 'shared-rand-commit' line wasn't an integer: %s" % value) + + commitments.append(SharedRandomnessCommitment(int(version), algorithm, identity, commit, reveal)) + + descriptor.shared_randomness_commitments = commitments + + +def _parse_shared_rand_previous_value(descriptor, entries): + # "shared-rand-previous-value" NumReveals Value + + value = _value('shared-rand-previous-value', entries) + value_comp = value.split(' ') + + if len(value_comp) == 2 and value_comp[0].isdigit(): + descriptor.shared_randomness_previous_reveal_count = int(value_comp[0]) + descriptor.shared_randomness_previous_value = value_comp[1] + else: + raise ValueError("A network status document's 'shared-rand-previous-value' line must be a pair of values, the first an integer but was '%s'" % value) + + +def _parse_shared_rand_current_value(descriptor, entries): + # "shared-rand-current-value" NumReveals Value + + value = _value('shared-rand-current-value', entries) + value_comp = value.split(' ') + + if len(value_comp) == 2 and value_comp[0].isdigit(): + descriptor.shared_randomness_current_reveal_count = int(value_comp[0]) + descriptor.shared_randomness_current_value = value_comp[1] + else: + raise ValueError("A network status document's 'shared-rand-current-value' line must be a pair of values, the first an integer but was '%s'" % value) + + _parse_header_valid_after_line = _parse_timestamp_line('valid-after', 'valid_after') _parse_header_fresh_until_line = _parse_timestamp_line('fresh-until', 'fresh_until') _parse_header_valid_until_line = _parse_timestamp_line('valid-until', 'valid_until') @@ -688,6 +755,7 @@ _parse_header_client_versions_line = _parse_versions_line('client-versions', 'cl _parse_header_server_versions_line = _parse_versions_line('server-versions', 'server_versions') _parse_header_known_flags_line = _parse_simple_line('known-flags', 'known_flags', func = lambda v: [entry for entry in v.split(' ') if entry]) _parse_footer_bandwidth_weights_line = _parse_simple_line('bandwidth-weights', 'bandwidth_weights', func = lambda v: _parse_int_mappings('bandwidth-weights', v, True)) +_parse_shared_rand_participate_line = _parse_if_present('shared-rand-participate', 'is_shared_randomness_participate')

class NetworkStatusDocumentV3(NetworkStatusDocument): @@ -731,11 +799,31 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): :var datetime published: time when the document was published :var dict flag_thresholds: ***** mapping of internal performance thresholds used while making the vote, values are **ints** or **floats**

+ :var bool is_shared_randomness_participate: ***** **True** if this authority + participates in establishing a shared random value, **False** otherwise + :var list shared_randomness_commitments: ***** list of + :data:`~stem.descriptor.networkstatus.SharedRandomnessCommitment` entries + :var int shared_randomness_previous_reveal_count: number of commitments + used to generate the last shared random value + :var str shared_randomness_previous_value: base64 encoded last shared random + value + :var int shared_randomness_current_reveal_count: number of commitments + used to generate the current shared random value + :var str shared_randomness_current_value: base64 encoded current shared + random value + ***** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined

.. versionchanged:: 1.4.0 Added the packages attribute. + + .. versionchanged:: 1.5.0 + Added the is_shared_randomness_participate, shared_randomness_commitments, + shared_randomness_previous_reveal_count, + shared_randomness_previous_value, + shared_randomness_current_reveal_count, and + shared_randomness_current_value attributes. """

ATTRIBUTES = { @@ -757,6 +845,12 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): 'packages': ([], _parse_package_line), 'known_flags': ([], _parse_header_known_flags_line), 'flag_thresholds': ({}, _parse_header_flag_thresholds_line), + 'is_shared_randomness_participate': (False, _parse_shared_rand_participate_line), + 'shared_randomness_commitments': ([], _parsed_shared_rand_commit), + 'shared_randomness_previous_reveal_count': (None, _parse_shared_rand_previous_value), + 'shared_randomness_previous_value': (None, _parse_shared_rand_previous_value), + 'shared_randomness_current_reveal_count': (None, _parse_shared_rand_current_value), + 'shared_randomness_current_value': (None, _parse_shared_rand_current_value), 'params': ({}, _parse_header_parameters_line),

'signatures': ([], _parse_footer_directory_signature_line), @@ -778,6 +872,10 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): 'package': _parse_package_line, 'known-flags': _parse_header_known_flags_line, 'flag-thresholds': _parse_header_flag_thresholds_line, + 'shared-rand-participate': _parse_shared_rand_participate_line, + 'shared-rand-commit': _parsed_shared_rand_commit, + 'shared-rand-previous-value': _parse_shared_rand_previous_value, + 'shared-rand-current-value': _parse_shared_rand_current_value, 'params': _parse_header_parameters_line, }

@@ -869,7 +967,7 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): # all known header fields can only appear once except

for keyword, values in list(entries.items()): - if len(values) > 1 and keyword in HEADER_FIELDS and keyword != 'package': + if len(values) > 1 and keyword in HEADER_FIELDS and keyword != 'package' and keyword != 'shared-rand-commit': raise ValueError("Network status documents can only have a single '%s' line, got %i" % (keyword, len(values)))

if self._default_params: @@ -1016,7 +1114,7 @@ def _check_for_misordered_fields(entries, expected): if actual != expected: actual_label = ', '.join(actual) expected_label = ', '.join(expected) - raise ValueError("The fields in a section of the document are misordered. It should be '%s' but was '%s'" % (actual_label, expected_label)) + raise ValueError("The fields in a section of the document are misordered. It should be '%s' but was '%s'" % (expected_label, actual_label))

def _parse_int_mappings(keyword, value, validate): diff --git a/test/mocking.py b/test/mocking.py index 165a5d4..5a86690 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -60,6 +60,12 @@ try: except ImportError: from mock import Mock, patch

+try: + # added in python 2.7 + from collections import OrderedDict +except ImportError: + from stem.util.ordereddict import OrderedDict + CRYPTO_BLOB = """ MIGJAoGBAJv5IIWQ+WDWYUdyA/0L8qbIkEVH/cwryZWoIaPAzINfrw1WfNZGtBmg skFtXhOHHqTRN4GPPrZsAIUOQGzQtGb66IQgT4tO/pj+P6QmSCCdTfhvGfgTCsC+ @@ -321,7 +327,7 @@ def _get_descriptor_content(attr = None, exclude = (), header_template = (), foo if attr is None: attr = {}

- attr = dict(attr) # shallow copy since we're destructive + attr = OrderedDict(attr) # shallow copy since we're destructive

for content, template in ((header_content, header_template), (footer_content, footer_template)): diff --git a/test/unit/descriptor/networkstatus/document_v3.py b/test/unit/descriptor/networkstatus/document_v3.py index cb4a1a0..aae12a6 100644 --- a/test/unit/descriptor/networkstatus/document_v3.py +++ b/test/unit/descriptor/networkstatus/document_v3.py @@ -39,6 +39,12 @@ from test.mocking import (

from test.unit.descriptor import get_resource

+try: + # added in python 2.7 + from collections import OrderedDict +except ImportError: + from stem.util.ordereddict import OrderedDict + BANDWIDTH_WEIGHT_ENTRIES = ( 'Wbd', 'Wbe', 'Wbg', 'Wbm', 'Wdb', @@ -331,6 +337,12 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEqual(expected_known_flags, document.known_flags) self.assertEqual([], document.packages) self.assertEqual({}, document.flag_thresholds) + self.assertEqual(False, document.is_shared_randomness_participate) + self.assertEqual([], document.shared_randomness_commitments) + self.assertEqual(None, document.shared_randomness_previous_reveal_count) + self.assertEqual(None, document.shared_randomness_previous_value) + self.assertEqual(None, document.shared_randomness_current_reveal_count) + self.assertEqual(None, document.shared_randomness_current_value) self.assertEqual(DEFAULT_PARAMS, document.params) self.assertEqual((), document.directory_authorities) self.assertEqual({}, document.bandwidth_weights) @@ -366,6 +378,12 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEqual(expected_known_flags, document.known_flags) self.assertEqual([], document.packages) self.assertEqual({}, document.flag_thresholds) + self.assertEqual(False, document.is_shared_randomness_participate) + self.assertEqual([], document.shared_randomness_commitments) + self.assertEqual(None, document.shared_randomness_previous_reveal_count) + self.assertEqual(None, document.shared_randomness_previous_value) + self.assertEqual(None, document.shared_randomness_current_reveal_count) + self.assertEqual(None, document.shared_randomness_current_value) self.assertEqual(DEFAULT_PARAMS, document.params) self.assertEqual({}, document.bandwidth_weights) self.assertEqual([DOC_SIG], document.signatures) @@ -837,6 +855,77 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= document = NetworkStatusDocumentV3(content, False) self.assertEqual({}, document.flag_thresholds)

+ def test_shared_randomness(self): + """ + Parses the shared randomness attributes. + """ + + COMMITMENT_1 = '1 sha3-256 4CAEC248004A0DC6CE86EBD5F608C9B05500C70C AAAAAFd4/kAaklgYr4ijHZjXXy/B354jQfL31BFhhE46nuOHSPITyw== AAAAAFd4/kCpZeis3yJyr//rz8hXCeeAhHa4k3lAcAiMJd1vEMTPuw==' + COMMITMENT_2 = '1 sha3-256 598536A9DD4E6C0F18B4AD4B88C7875A0A29BA31 AAAAAFd4/kC7S920awC5/HF5RfX4fKZtYqjm6qMh9G91AcjZm13DQQ==' + + document = get_network_status_document_v3(OrderedDict([ + ('vote-status', 'vote'), + ('shared-rand-participate', ''), + ('shared-rand-commit', '%s\nshared-rand-commit %s' % (COMMITMENT_1, COMMITMENT_2)), + ('shared-rand-previous-value', '8 hAQLxyt0U3gu7QR2owixRCbIltcyPrz3B0YBfUshOkE='), + ('shared-rand-current-value', '7 KEIfSB7Db+ToasQIzJhbh0CtkeSePHLEehO+ams/RTU='), + ])) + + self.assertEqual(True, document.is_shared_randomness_participate) + self.assertEqual(8, document.shared_randomness_previous_reveal_count) + self.assertEqual('hAQLxyt0U3gu7QR2owixRCbIltcyPrz3B0YBfUshOkE=', document.shared_randomness_previous_value) + self.assertEqual(7, document.shared_randomness_current_reveal_count) + self.assertEqual('KEIfSB7Db+ToasQIzJhbh0CtkeSePHLEehO+ams/RTU=', document.shared_randomness_current_value) + + self.assertEqual(2, len(document.shared_randomness_commitments)) + + first_commitment = document.shared_randomness_commitments[0] + self.assertEqual(1, first_commitment.version) + self.assertEqual('sha3-256', first_commitment.algorithm) + self.assertEqual('4CAEC248004A0DC6CE86EBD5F608C9B05500C70C', first_commitment.identity) + self.assertEqual('AAAAAFd4/kAaklgYr4ijHZjXXy/B354jQfL31BFhhE46nuOHSPITyw==', first_commitment.commit) + self.assertEqual('AAAAAFd4/kCpZeis3yJyr//rz8hXCeeAhHa4k3lAcAiMJd1vEMTPuw==', first_commitment.reveal) + + second_commitment = document.shared_randomness_commitments[1] + self.assertEqual(1, second_commitment.version) + self.assertEqual('sha3-256', second_commitment.algorithm) + self.assertEqual('598536A9DD4E6C0F18B4AD4B88C7875A0A29BA31', second_commitment.identity) + self.assertEqual('AAAAAFd4/kC7S920awC5/HF5RfX4fKZtYqjm6qMh9G91AcjZm13DQQ==', second_commitment.commit) + self.assertEqual(None, second_commitment.reveal) + + def test_shared_randomness_malformed(self): + """ + Checks shared randomness with malformed values. + """ + + test_values = [ + ({'vote-status': 'vote', 'shared-rand-commit': 'hi sha3-256 598536A9DD4E6C0F18B4AD4B88C7875A0A29BA31 AAAAAFd4/kC7S920awC5/HF5RfX4fKZtYqjm6qMh9G91AcjZm13DQQ=='}, + "The version on our 'shared-rand-commit' line wasn't an integer: hi sha3-256 598536A9DD4E6C0F18B4AD4B88C7875A0A29BA31 AAAAAFd4/kC7S920awC5/HF5RfX4fKZtYqjm6qMh9G91AcjZm13DQQ=="), + ({'vote-status': 'vote', 'shared-rand-commit': 'sha3-256 598536A9DD4E6C0F18B4AD4B88C7875A0A29BA31 AAAAAFd4/kC7S920awC5/HF5RfX4fKZtYqjm6qMh9G91AcjZm13DQQ=='}, + "'shared-rand-commit' must at least have a 'Version AlgName Identity Commit': sha3-256 598536A9DD4E6C0F18B4AD4B88C7875A0A29BA31 AAAAAFd4/kC7S920awC5/HF5RfX4fKZtYqjm6qMh9G91AcjZm13DQQ=="), + ({'vote-status': 'vote', 'shared-rand-current-value': 'hi KEIfSB7Db+ToasQIzJhbh0CtkeSePHLEehO+ams/RTU='}, + "A network status document's 'shared-rand-current-value' line must be a pair of values, the first an integer but was 'hi KEIfSB7Db+ToasQIzJhbh0CtkeSePHLEehO+ams/RTU='"), + ({'vote-status': 'vote', 'shared-rand-current-value': 'KEIfSB7Db+ToasQIzJhbh0CtkeSePHLEehO+ams/RTU='}, + "A network status document's 'shared-rand-current-value' line must be a pair of values, the first an integer but was 'KEIfSB7Db+ToasQIzJhbh0CtkeSePHLEehO+ams/RTU='"), + ] + + for attr, expected_exception in test_values: + content = get_network_status_document_v3(attr, content = True) + + try: + NetworkStatusDocumentV3(content, True) + self.fail("validation should've rejected malformed shared randomness attribute") + except ValueError as exc: + self.assertEqual(expected_exception, str(exc)) + + document = NetworkStatusDocumentV3(content, False) + + self.assertEqual([], document.shared_randomness_commitments) + self.assertEqual(None, document.shared_randomness_previous_reveal_count) + self.assertEqual(None, document.shared_randomness_previous_value) + self.assertEqual(None, document.shared_randomness_current_reveal_count) + self.assertEqual(None, document.shared_randomness_current_value) + def test_params(self): """ General testing for the 'params' line, exercising the happy cases.