commit 9ebc08da696a8607908761d2b54a622b24eb4b3f Author: Damian Johnson <atagar@torproject.org> Date: Thu Jan 22 09:32:59 2015 -0800 Merging DocumentHeader into NetworkStatusDocumentV3 --- stem/descriptor/__init__.py | 3 +- stem/descriptor/networkstatus.py | 505 ++++++++++++++++++-------------------- 2 files changed, 240 insertions(+), 268 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 13f0e56..1a7a097 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -451,7 +451,8 @@ class Descriptor(object): # set defaults for attr in self.ATTRIBUTES: - setattr(self, attr, copy.copy(self.ATTRIBUTES[attr][0])) + if not hasattr(self, attr): + setattr(self, attr, copy.copy(self.ATTRIBUTES[attr][0])) for keyword, values in list(entries.items()): try: diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index ae1ab48..2289cdf 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -432,6 +432,153 @@ class NetworkStatusDocumentV2(NetworkStatusDocument): raise ValueError("Network status document (v2) are expected to start with a 'network-status-version' line:\n%s" % str(self)) +def _parse_header_network_status_version_line(descriptor, entries): + # "network-status-version" version + + value = _value('network-status-version', entries) + + if ' ' in value: + version, flavor = value.split(' ', 1) + else: + version, flavor = value, None + + if not version.isdigit(): + raise ValueError('Network status document has a non-numeric version: network-status-version %s' % value) + + descriptor.version = int(version) + descriptor.version_flavor = flavor + descriptor.is_microdescriptor = flavor == 'microdesc' + + if descriptor.version != 3: + raise ValueError("Expected a version 3 network status document, got version '%s' instead" % descriptor.version) + + +def _parse_header_vote_status_line(descriptor, entries): + # "vote-status" type + # + # The consensus-method and consensus-methods fields are optional since + # they weren't included in version 1. Setting a default now that we + # know if we're a vote or not. + + value = _value('vote-status', entries) + + if value == 'consensus': + descriptor.is_consensus, descriptor.is_vote = True, False + elif value == 'vote': + descriptor.is_consensus, descriptor.is_vote = False, True + else: + raise ValueError("A network status document's vote-status line can only be 'consensus' or 'vote', got '%s' instead" % value) + + +def _parse_header_consensus_methods_line(descriptor, entries): + # "consensus-methods" IntegerList + + if descriptor._lazy_loading and descriptor.is_vote: + descriptor.consensus_methods = [1] + + value, consensus_methods = _value('consensus-methods', entries), [] + + for entry in value.split(' '): + if not entry.isdigit(): + raise ValueError("A network status document's consensus-methods must be a list of integer values, but was '%s'" % value) + + consensus_methods.append(int(entry)) + + descriptor.consensus_methods = consensus_methods + + +def _parse_header_consensus_method_line(descriptor, entries): + # "consensus-method" Integer + + if descriptor._lazy_loading and descriptor.is_consensus: + descriptor.consensus_method = 1 + + value = _value('consensus-method', entries) + + if not value.isdigit(): + raise ValueError("A network status document's consensus-method must be an integer, but was '%s'" % value) + + descriptor.consensus_method = int(value) + + +def _parse_header_voting_delay_line(descriptor, entries): + # "voting-delay" VoteSeconds DistSeconds + + value = _value('voting-delay', entries) + value_comp = value.split(' ') + + if len(value_comp) == 2 and value_comp[0].isdigit() and value_comp[1].isdigit(): + descriptor.vote_delay = int(value_comp[0]) + descriptor.dist_delay = int(value_comp[1]) + else: + raise ValueError("A network status document's 'voting-delay' line must be a pair of integer values, but was '%s'" % value) + + +def _parse_versions_line(keyword, attribute): + def _parse(descriptor, entries): + value, entries = _value(keyword, entries), [] + + for entry in value.split(','): + try: + entries.append(stem.version._get_version(entry)) + except ValueError: + raise ValueError("Network status document's '%s' line had '%s', which isn't a parsable tor version: %s %s" % (keyword, entry, keyword, value)) + + setattr(descriptor, attribute, entries) + + return _parse + + +def _parse_header_flag_thresholds_line(descriptor, entries): + # "flag-thresholds" SP THRESHOLDS + + value, thresholds = _value('flag-thresholds', entries).strip(), {} + + if value: + for entry in value.split(' '): + if '=' not in entry: + raise ValueError("Network status document's 'flag-thresholds' line is expected to be space separated key=value mappings, got: flag-thresholds %s" % value) + + entry_key, entry_value = entry.split('=', 1) + + try: + if entry_value.endswith('%'): + # opting for string manipulation rather than just + # 'float(entry_value) / 100' because floating point arithmetic + # will lose precision + + thresholds[entry_key] = float('0.' + entry_value[:-1].replace('.', '', 1)) + elif '.' in entry_value: + thresholds[entry_key] = float(entry_value) + else: + thresholds[entry_key] = int(entry_value) + except ValueError: + raise ValueError("Network status document's 'flag-thresholds' line is expected to have float values, got: flag-thresholds %s" % value) + + descriptor.flag_thresholds = thresholds + + +def _parse_header_parameters_line(descriptor, entries): + # "params" [Parameters] + # Parameter ::= Keyword '=' Int32 + # Int32 ::= A decimal integer between -2147483648 and 2147483647. + # Parameters ::= Parameter | Parameters SP Parameter + + if descriptor._lazy_loading and descriptor._default_params: + descriptor.params = dict(DEFAULT_PARAMS) + + value = _value('params', entries) + + # should only appear in consensus-method 7 or later + + if not descriptor.meets_consensus_method(7): + raise ValueError("A network status document's 'params' line should only appear in consensus-method 7 or later") + + if value != '': + descriptor.params = _parse_int_mappings('params', value, True) + descriptor._check_params_constraints() + + def _parse_directory_footer_line(descriptor, entries): # nothing to parse, simply checking that we don't have a value @@ -462,7 +609,13 @@ def _parse_footer_directory_signature_line(descriptor, entries): descriptor.signatures = signatures -_parse_bandwidth_weights_line = lambda descriptor, entries: setattr(descriptor, 'bandwidth_weights', _parse_int_mappings('bandwidth-weights', _value('bandwidth-weights', entries), True)) +_parse_header_valid_after_line = _parse_timestamp_line('valid-after', 'valid_after') +_parse_header_fresh_until_line = _parse_timestamp_line('fresh-until', 'fresh_until') +_parse_header_valid_until_line = _parse_timestamp_line('valid-until', 'valid_until') +_parse_header_client_versions_line = _parse_versions_line('client-versions', 'client_versions') +_parse_header_server_versions_line = _parse_versions_line('server-versions', 'server_versions') +_parse_header_known_flags_line = lambda descriptor, entries: setattr(descriptor, 'known_flags', [entry for entry in _value('known-flags', entries).split(' ') if entry]) +_parse_footer_bandwidth_weights_line = lambda descriptor, entries: setattr(descriptor, 'bandwidth_weights', _parse_int_mappings('bandwidth-weights', _value('bandwidth-weights', entries), True)) class NetworkStatusDocumentV3(NetworkStatusDocument): @@ -510,13 +663,49 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): """ ATTRIBUTES = { + 'version': (None, _parse_header_network_status_version_line), + 'version_flavor': (None, _parse_header_network_status_version_line), + 'is_consensus': (True, _parse_header_vote_status_line), + 'is_vote': (False, _parse_header_vote_status_line), + 'is_microdescriptor': (False, _parse_header_network_status_version_line), + 'consensus_methods': ([], _parse_header_consensus_methods_line), + 'published': (None, _parse_published_line), + 'consensus_method': (None, _parse_header_consensus_method_line), + 'valid_after': (None, _parse_header_valid_after_line), + 'fresh_until': (None, _parse_header_fresh_until_line), + 'valid_until': (None, _parse_header_valid_until_line), + 'vote_delay': (None, _parse_header_voting_delay_line), + 'dist_delay': (None, _parse_header_voting_delay_line), + 'client_versions': ([], _parse_header_client_versions_line), + 'server_versions': ([], _parse_header_server_versions_line), + 'known_flags': ([], _parse_header_known_flags_line), + 'flag_thresholds': ({}, _parse_header_flag_thresholds_line), + 'params': ({}, _parse_header_parameters_line), + 'signatures': ([], _parse_footer_directory_signature_line), - 'bandwidth_weights': ({}, _parse_bandwidth_weights_line), + 'bandwidth_weights': ({}, _parse_footer_bandwidth_weights_line), + } + + HEADER_PARSER_FOR_LINE = { + 'network-status-version': _parse_header_network_status_version_line, + 'vote-status': _parse_header_vote_status_line, + 'consensus-methods': _parse_header_consensus_methods_line, + 'consensus-method': _parse_header_consensus_method_line, + 'published': _parse_published_line, + 'valid-after': _parse_header_valid_after_line, + 'fresh-until': _parse_header_fresh_until_line, + 'valid-until': _parse_header_valid_until_line, + 'voting-delay': _parse_header_voting_delay_line, + 'client-versions': _parse_header_client_versions_line, + 'server-versions': _parse_header_server_versions_line, + 'known-flags': _parse_header_known_flags_line, + 'flag-thresholds': _parse_header_flag_thresholds_line, + 'params': _parse_header_parameters_line, } FOOTER_PARSER_FOR_LINE = { 'directory-footer': _parse_directory_footer_line, - 'bandwidth-weights': _parse_bandwidth_weights_line, + 'bandwidth-weights': _parse_footer_bandwidth_weights_line, 'directory-signature': _parse_footer_directory_signature_line, } @@ -535,14 +724,8 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): super(NetworkStatusDocumentV3, self).__init__(raw_content, lazy_load = not validate) document_file = io.BytesIO(raw_content) - header = _DocumentHeader(document_file, validate, default_params) - - # merge header attributes into us - for attr, value in vars(header).items(): - if attr != '_unrecognized_lines': - setattr(self, attr, value) - else: - self._unrecognized_lines += value + self._default_params = default_params + self._header(document_file, validate) self.directory_authorities = tuple(stem.descriptor.router_status_entry._parse_file( document_file, @@ -576,6 +759,7 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): def get_unrecognized_lines(self): if self._lazy_loading: + self._parse(self._header_entries, False, parser_for_line = self.HEADER_PARSER_FOR_LINE) self._parse(self._footer_entries, False, parser_for_line = self.FOOTER_PARSER_FOR_LINE) self._lazy_loading = False @@ -605,13 +789,39 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): return method(str(self).strip(), str(other).strip()) - def _footer(self, document_file, validate): - content = stem.util.str_tools._to_unicode(document_file.read()) + def _header(self, document_file, validate): + content = bytes.join(b'', _read_until_keywords((AUTH_START, ROUTERS_START, FOOTER_START), document_file)) + content = stem.util.str_tools._to_unicode(content) + entries = _get_descriptor_components(content, validate) + + if validate: + # all known header fields can only appear once except + + for keyword, values in list(entries.items()): + if len(values) > 1 and keyword in HEADER_FIELDS: + raise ValueError("Network status documents can only have a single '%s' line, got %i" % (keyword, len(values))) + + if self._default_params: + self.params = dict(DEFAULT_PARAMS) + + self._parse(entries, validate, parser_for_line = self.HEADER_PARSER_FOR_LINE) - if content: - entries = _get_descriptor_components(content, validate) + _check_for_missing_and_disallowed_fields(self, entries, HEADER_STATUS_DOCUMENT_FIELDS) + _check_for_misordered_fields(entries, HEADER_FIELDS) + + # default consensus_method and consensus_methods based on if we're a consensus or vote + + if self.is_consensus and not self.consensus_method: + self.consensus_method = 1 + elif self.is_vote and not self.consensus_methods: + self.consensus_methods = [1] else: - entries = {} + self._header_entries = entries + self._entries.update(entries) + + def _footer(self, document_file, validate): + content = stem.util.str_tools._to_unicode(document_file.read()) + entries = _get_descriptor_components(content, validate) if content else {} if validate: for keyword, values in list(entries.items()): @@ -642,257 +852,6 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): self._footer_entries = entries self._entries.update(entries) - def __hash__(self): - return hash(str(self).strip()) - - def __eq__(self, other): - return self._compare(other, lambda s, o: s == o) - - def __lt__(self, other): - return self._compare(other, lambda s, o: s < o) - - def __le__(self, other): - return self._compare(other, lambda s, o: s <= o) - - -def _parse_network_status_version_line(descriptor, entries): - # "network-status-version" version - - value = _value('network-status-version', entries) - - if ' ' in value: - version, flavor = value.split(' ', 1) - else: - version, flavor = value, None - - if not version.isdigit(): - raise ValueError('Network status document has a non-numeric version: network-status-version %s' % value) - - descriptor.version = int(version) - descriptor.version_flavor = flavor - descriptor.is_microdescriptor = flavor == 'microdesc' - - if descriptor.version != 3: - raise ValueError("Expected a version 3 network status document, got version '%s' instead" % descriptor.version) - - -def _parse_vote_status_line(descriptor, entries): - # "vote-status" type - # - # The consensus-method and consensus-methods fields are optional since - # they weren't included in version 1. Setting a default now that we - # know if we're a vote or not. - - value = _value('vote-status', entries) - - if value == 'consensus': - descriptor.is_consensus, descriptor.is_vote = True, False - elif value == 'vote': - descriptor.is_consensus, descriptor.is_vote = False, True - else: - raise ValueError("A network status document's vote-status line can only be 'consensus' or 'vote', got '%s' instead" % value) - - -def _parse_consensus_methods_line(descriptor, entries): - # "consensus-methods" IntegerList - - value, consensus_methods = _value('consensus-methods', entries), [] - - for entry in value.split(' '): - if not entry.isdigit(): - raise ValueError("A network status document's consensus-methods must be a list of integer values, but was '%s'" % value) - - consensus_methods.append(int(entry)) - - descriptor.consensus_methods = consensus_methods - - -def _parse_consensus_method_line(descriptor, entries): - # "consensus-method" Integer - - value = _value('consensus-method', entries) - - if not value.isdigit(): - raise ValueError("A network status document's consensus-method must be an integer, but was '%s'" % value) - - descriptor.consensus_method = int(value) - - -def _parse_voting_delay_line(descriptor, entries): - # "voting-delay" VoteSeconds DistSeconds - - value = _value('voting-delay', entries) - value_comp = value.split(' ') - - if len(value_comp) == 2 and value_comp[0].isdigit() and value_comp[1].isdigit(): - descriptor.vote_delay = int(value_comp[0]) - descriptor.dist_delay = int(value_comp[1]) - else: - raise ValueError("A network status document's 'voting-delay' line must be a pair of integer values, but was '%s'" % value) - - -def _parse_versions_line(keyword, attribute): - def _parse(descriptor, entries): - value, entries = _value(keyword, entries), [] - - for entry in value.split(','): - try: - entries.append(stem.version._get_version(entry)) - except ValueError: - raise ValueError("Network status document's '%s' line had '%s', which isn't a parsable tor version: %s %s" % (keyword, entry, keyword, value)) - - setattr(descriptor, attribute, entries) - - return _parse - - -def _parse_flag_thresholds_line(descriptor, entries): - # "flag-thresholds" SP THRESHOLDS - - value, thresholds = _value('flag-thresholds', entries).strip(), {} - - if value: - for entry in value.split(' '): - if '=' not in entry: - raise ValueError("Network status document's 'flag-thresholds' line is expected to be space separated key=value mappings, got: flag-thresholds %s" % value) - - entry_key, entry_value = entry.split('=', 1) - - try: - if entry_value.endswith('%'): - # opting for string manipulation rather than just - # 'float(entry_value) / 100' because floating point arithmetic - # will lose precision - - thresholds[entry_key] = float('0.' + entry_value[:-1].replace('.', '', 1)) - elif '.' in entry_value: - thresholds[entry_key] = float(entry_value) - else: - thresholds[entry_key] = int(entry_value) - except ValueError: - raise ValueError("Network status document's 'flag-thresholds' line is expected to have float values, got: flag-thresholds %s" % value) - - descriptor.flag_thresholds = thresholds - - -def _parse_parameters_line(descriptor, entries): - # "params" [Parameters] - # Parameter ::= Keyword '=' Int32 - # Int32 ::= A decimal integer between -2147483648 and 2147483647. - # Parameters ::= Parameter | Parameters SP Parameter - - value = _value('params', entries) - - # should only appear in consensus-method 7 or later - - if not descriptor.meets_consensus_method(7): - raise ValueError("A network status document's 'params' line should only appear in consensus-method 7 or later") - - # skip if this is a blank line - - params = dict(DEFAULT_PARAMS) if descriptor._default_params else {} - - if value != '': - params.update(_parse_int_mappings('params', value, True)) - descriptor.params = params - descriptor._check_params_constraints() - - -_parse_valid_after_line = _parse_timestamp_line('valid-after', 'valid_after') -_parse_fresh_until_line = _parse_timestamp_line('fresh-until', 'fresh_until') -_parse_valid_until_line = _parse_timestamp_line('valid-until', 'valid_until') -_parse_client_versions_line = _parse_versions_line('client-versions', 'client_versions') -_parse_server_versions_line = _parse_versions_line('server-versions', 'server_versions') -_parse_known_flags_line = lambda descriptor, entries: setattr(descriptor, 'known_flags', [entry for entry in _value('known-flags', entries).split(' ') if entry]) - - -class _DocumentHeader(object): - PARSER_FOR_LINE = { - 'network-status-version': _parse_network_status_version_line, - 'vote-status': _parse_vote_status_line, - 'consensus-methods': _parse_consensus_methods_line, - 'consensus-method': _parse_consensus_method_line, - 'published': _parse_published_line, - 'valid-after': _parse_valid_after_line, - 'fresh-until': _parse_fresh_until_line, - 'valid-until': _parse_valid_until_line, - 'voting-delay': _parse_voting_delay_line, - 'client-versions': _parse_client_versions_line, - 'server-versions': _parse_server_versions_line, - 'known-flags': _parse_known_flags_line, - 'flag-thresholds': _parse_flag_thresholds_line, - 'params': _parse_parameters_line, - } - - def __init__(self, document_file, validate, default_params): - self.version = None - self.version_flavor = None - self.is_consensus = True - self.is_vote = False - self.is_microdescriptor = False - self.consensus_methods = [] - self.published = None - self.consensus_method = None - self.valid_after = None - self.fresh_until = None - self.valid_until = None - self.vote_delay = None - self.dist_delay = None - self.client_versions = [] - self.server_versions = [] - self.known_flags = [] - self.flag_thresholds = {} - self.params = dict(DEFAULT_PARAMS) if default_params else {} - - self._default_params = default_params - - self._unrecognized_lines = [] - - content = bytes.join(b'', _read_until_keywords((AUTH_START, ROUTERS_START, FOOTER_START), document_file)) - content = stem.util.str_tools._to_unicode(content) - entries = _get_descriptor_components(content, validate) - self._parse(entries, validate) - - # doing this validation afterward so we know our 'is_consensus' and - # 'is_vote' attributes - - if validate: - _check_for_missing_and_disallowed_fields(self, entries, HEADER_STATUS_DOCUMENT_FIELDS) - _check_for_misordered_fields(entries, HEADER_FIELDS) - - def meets_consensus_method(self, method): - if self.consensus_method is not None: - return self.consensus_method >= method - elif self.consensus_methods is not None: - return bool([x for x in self.consensus_methods if x >= method]) - else: - return False # malformed document - - def _parse(self, entries, validate): - for keyword, values in list(entries.items()): - value, _, _ = values[0] - line = '%s %s' % (keyword, value) - - # all known header fields can only appear once except - if validate and len(values) > 1 and keyword in HEADER_FIELDS: - raise ValueError("Network status documents can only have a single '%s' line, got %i" % (keyword, len(values))) - - try: - if keyword in self.PARSER_FOR_LINE: - self.PARSER_FOR_LINE[keyword](self, entries) - else: - self._unrecognized_lines.append(line) - except ValueError as exc: - if validate: - raise exc - - # default consensus_method and consensus_methods based on if we're a consensus or vote - - if self.is_consensus and not self.consensus_method: - self.consensus_method = 1 - elif self.is_vote and not self.consensus_methods: - self.consensus_methods = [1] - def _check_params_constraints(self): """ Checks that the params we know about are within their documented ranges. @@ -956,6 +915,18 @@ class _DocumentHeader(object): if value < minimum or value > maximum: raise ValueError("'%s' value on the params line must be in the range of %i - %i, was %i" % (key, minimum, maximum, value)) + def __hash__(self): + return hash(str(self).strip()) + + def __eq__(self, other): + return self._compare(other, lambda s, o: s == o) + + def __lt__(self, other): + return self._compare(other, lambda s, o: s < o) + + def __le__(self, other): + return self._compare(other, lambda s, o: s <= o) + def _check_for_missing_and_disallowed_fields(document, entries, fields): """