commit f19d87ab0f16928be370114c993f512b3d2dfac5 Author: Damian Johnson <atagar@torproject.org> Date: Sat Jan 17 17:36:54 2015 -0800 Microdescriptor lazy loading Simplest descriptor type so pretty simple switch. --- stem/descriptor/__init__.py | 4 +- stem/descriptor/extrainfo_descriptor.py | 3 +- stem/descriptor/microdescriptor.py | 115 ++++++++++++++----------------- stem/descriptor/server_descriptor.py | 6 +- 4 files changed, 56 insertions(+), 72 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index bc1b462..ef96dd7 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -351,11 +351,11 @@ class Descriptor(object): ATTRIBUTES = {} # mapping of 'attribute' => (default_value, parsing_function) PARSER_FOR_LINE = {} # line keyword to its associated parsing function - def __init__(self, contents): + def __init__(self, contents, lazy_load = False): self._path = None self._archive_path = None self._raw_contents = contents - self._lazy_loading = False + self._lazy_loading = lazy_load self._unrecognized_lines = [] def get_path(self): diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 9e86798..4137e47 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -786,11 +786,10 @@ class ExtraInfoDescriptor(Descriptor): :raises: **ValueError** if the contents is malformed and validate is True """ - super(ExtraInfoDescriptor, self).__init__(raw_contents) + super(ExtraInfoDescriptor, self).__init__(raw_contents, lazy_load = not validate) raw_contents = stem.util.str_tools._to_unicode(raw_contents) entries = _get_descriptor_components(raw_contents, validate) - self._lazy_loading = not validate if validate: for keyword in self._required_fields(): diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index 093e6f0..0590cbb 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -73,6 +73,9 @@ from stem.descriptor import ( Descriptor, _get_descriptor_components, _read_until_keywords, + _value, + _values, + _parse_key_block, ) try: @@ -151,6 +154,29 @@ def _parse_file(descriptor_file, validate = True, **kwargs): break # done parsing descriptors +def _parse_a_line(descriptor, entries): + for value in _values('a', entries): + stem.descriptor.router_status_entry._parse_a_line(descriptor, value, True) + + +def _parse_id_line(descriptor, entries): + value = _value('id', entries) + value_comp = value.split() + + if len(value_comp) >= 2: + descriptor.identifier_type = value_comp[0] + descriptor.identifier = value_comp[1] + else: + raise ValueError("'id' lines should contain both the key type and digest: id %s" % value) + + +_parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY') +_parse_ntor_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'ntor_onion_key', _value('ntor-onion-key', entries)) +_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', _value('family', entries).split(' ')) +_parse_p_line = lambda descriptor, entries: stem.descriptor.router_status_entry._parse_p_line(descriptor, _value('p', entries), True) +_parse_p6_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('p6', entries))) + + class Microdescriptor(Descriptor): """ Microdescriptor (`descriptor specification @@ -173,33 +199,41 @@ class Microdescriptor(Descriptor): **\*** attribute is required when we're parsed with validation """ + ATTRIBUTES = { + 'onion_key': (None, _parse_onion_key_line), + 'ntor_onion_key': (None, _parse_ntor_onion_key_line), + 'or_addresses': ([], _parse_a_line), + 'family': ([], _parse_family_line), + 'exit_policy': (stem.exit_policy.MicroExitPolicy('reject 1-65535'), _parse_p_line), + 'exit_policy_v6': (None, _parse_p6_line), + 'identifier_type': (None, _parse_id_line), + 'identifier': (None, _parse_id_line), + } + + PARSER_FOR_LINE = { + 'onion-key': _parse_onion_key_line, + 'ntor-onion-key': _parse_ntor_onion_key_line, + 'a': _parse_a_line, + 'family': _parse_family_line, + 'p': _parse_p_line, + 'p6': _parse_p6_line, + 'id': _parse_id_line, + } + def __init__(self, raw_contents, validate = True, annotations = None): - super(Microdescriptor, self).__init__(raw_contents) + super(Microdescriptor, self).__init__(raw_contents, lazy_load = not validate) raw_contents = stem.util.str_tools._to_unicode(raw_contents) self.digest = hashlib.sha256(self.get_bytes()).hexdigest().upper() - - self.onion_key = None - self.ntor_onion_key = None - self.or_addresses = [] - self.family = [] - self.exit_policy = stem.exit_policy.MicroExitPolicy('reject 1-65535') - self.exit_policy_v6 = None - self.identifier_type = None - self.identifier = None - - self._unrecognized_lines = [] - self._annotation_lines = annotations if annotations else [] entries = _get_descriptor_components(raw_contents, validate) - self._parse(entries, validate) if validate: + self._parse(entries, validate) self._check_constraints(entries) - - def get_unrecognized_lines(self): - return list(self._unrecognized_lines) + else: + self._entries = entries @lru_cache() def get_annotations(self): @@ -237,53 +271,6 @@ class Microdescriptor(Descriptor): return self._annotation_lines - def _parse(self, entries, validate): - """ - Parses a series of 'keyword => (value, pgp block)' mappings and applies - them as attributes. - - :param dict entries: descriptor contents to be applied - :param bool validate: checks the validity of descriptor content if **True** - - :raises: **ValueError** if an error occurs in validation - """ - - for keyword, values in list(entries.items()): - # most just work with the first (and only) value - value, block_type, block_contents = values[0] - - line = '%s %s' % (keyword, value) # original line - - if block_contents: - line += '\n%s' % block_contents - - if keyword == 'onion-key': - if validate and (not block_contents or block_type != 'RSA PUBLIC KEY'): - raise ValueError("'onion-key' should be followed by a RSA PUBLIC KEY block: %s" % line) - - self.onion_key = block_contents - elif keyword == 'ntor-onion-key': - self.ntor_onion_key = value - elif keyword == 'a': - for entry, _, _ in values: - stem.descriptor.router_status_entry._parse_a_line(self, entry, validate) - elif keyword == 'family': - self.family = value.split(' ') - elif keyword == 'p': - stem.descriptor.router_status_entry._parse_p_line(self, value, validate) - elif keyword == 'p6': - self.exit_policy_v6 = stem.exit_policy.MicroExitPolicy(value) - elif keyword == 'id': - value_comp = value.split() - - if len(value_comp) >= 2: - self.identifier_type = value_comp[0] - self.identifier = value_comp[1] - elif validate: - raise ValueError("'id' lines should contain both the key type and digest: %s" % line) - else: - self._unrecognized_lines.append(line) - def _check_constraints(self, entries): """ Does a basic check that the entries conform to this descriptor type's diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index b6af898..a696d6a 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -533,7 +533,7 @@ class ServerDescriptor(Descriptor): :raises: **ValueError** if the contents is malformed and validate is True """ - super(ServerDescriptor, self).__init__(raw_contents) + super(ServerDescriptor, self).__init__(raw_contents, lazy_load = not validate) # Only a few things can be arbitrary bytes according to the dir-spec, so # parsing them separately. @@ -541,9 +541,6 @@ class ServerDescriptor(Descriptor): self.platform = _get_bytes_field('platform', raw_contents) self.contact = _get_bytes_field('contact', raw_contents) - raw_contents = stem.util.str_tools._to_unicode(raw_contents) - - self._lazy_loading = not validate self._annotation_lines = annotations if annotations else [] # A descriptor contains a series of 'keyword lines' which are simply a @@ -554,6 +551,7 @@ class ServerDescriptor(Descriptor): # influences the resulting exit policy, but for everything else the order # does not matter so breaking it into key / value pairs. + raw_contents = stem.util.str_tools._to_unicode(raw_contents) entries, self._unparsed_exit_policy = _get_descriptor_components(raw_contents, validate, ('accept', 'reject')) if validate: