commit 6f3a9d846d1226679bdd56dedce362d76c2a3be5 Author: Damian Johnson <atagar@torproject.org> Date: Sat Jan 17 20:33:33 2015 -0800 DirectoryAuthority lazy loading Another subsection of network status documents. --- stem/descriptor/__init__.py | 6 +- stem/descriptor/extrainfo_descriptor.py | 8 +- stem/descriptor/networkstatus.py | 180 ++++++++------------ stem/descriptor/server_descriptor.py | 17 +- .../networkstatus/directory_authority.py | 9 +- 5 files changed, 85 insertions(+), 135 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 1e1acb5..cd9dcde 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -333,12 +333,14 @@ def _parse_timestamp_line(keyword, attribute): return _parse -def _parse_sha1_digest_line(keyword, attribute): +def _parse_forty_character_hex(keyword, attribute): + # format of fingerprints, sha1 digests, etc + def _parse(descriptor, entries): value = _value(keyword, entries) if not stem.util.tor_tools.is_hex_digits(value, 40): - raise ValueError('%s line had an invalid sha1 digest: %s %s' % (keyword, keyword, value)) + raise ValueError('%s line had an invalid value (should be 40 hex characters): %s %s' % (keyword, keyword, value)) setattr(descriptor, attribute, value) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 3413711..124ce16 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -84,7 +84,7 @@ from stem.descriptor import ( _value, _values, _parse_timestamp_line, - _parse_sha1_digest_line, + _parse_forty_character_hex, _parse_key_block, ) @@ -498,8 +498,8 @@ def _parse_bridge_ip_transports_line(descriptor, entries): descriptor.ip_transports = ip_transports -_parse_geoip_db_digest_line = _parse_sha1_digest_line('geoip-db-digest', 'geoip_db_digest') -_parse_geoip6_db_digest_line = _parse_sha1_digest_line('geoip6-db-digest', 'geoip6_db_digest') +_parse_geoip_db_digest_line = _parse_forty_character_hex('geoip-db-digest', 'geoip_db_digest') +_parse_geoip6_db_digest_line = _parse_forty_character_hex('geoip6-db-digest', 'geoip6_db_digest') _parse_dirreq_v2_resp_line = functools.partial(_parse_dirreq_line, 'dirreq-v2-resp', 'dir_v2_responses', 'dir_v2_responses_unknown') _parse_dirreq_v3_resp_line = functools.partial(_parse_dirreq_line, 'dirreq-v3-resp', 'dir_v3_responses', 'dir_v3_responses_unknown') _parse_dirreq_v2_direct_dl_line = functools.partial(_parse_dirreq_line, 'dirreq-v2-direct-dl', 'dir_v2_direct_dl', 'dir_v2_direct_dl_unknown') @@ -532,7 +532,7 @@ _parse_dirreq_v3_reqs_line = functools.partial(_parse_geoip_to_count_line, 'dirr _parse_geoip_client_origins_line = functools.partial(_parse_geoip_to_count_line, 'geoip-client-origins', 'geoip_client_origins') _parse_entry_ips_line = functools.partial(_parse_geoip_to_count_line, 'entry-ips', 'entry_ips') _parse_bridge_ips_line = functools.partial(_parse_geoip_to_count_line, 'bridge-ips', 'bridge_ips') -_parse_router_digest_line = _parse_sha1_digest_line('router-digest', '_digest') +_parse_router_digest_line = _parse_forty_character_hex('router-digest', '_digest') _parse_router_signature_line = _parse_key_block('router-signature', 'signature', 'SIGNATURE') diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index a70a7cf..70f325b 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -64,6 +64,7 @@ from stem.descriptor import ( _read_until_keywords, _value, _parse_timestamp_line, + _parse_forty_character_hex, _parse_key_block, ) @@ -1027,6 +1028,43 @@ def _parse_int_mappings(keyword, value, validate): return results +def _parse_dir_source_line(descriptor, entries): + # "dir-source" nickname identity address IP dirport orport + + value = _value('dir-source', entries) + dir_source_comp = value.split(' ') + + if len(dir_source_comp) < 6: + raise ValueError("Authority entry's 'dir-source' line must have six values: dir-source %s" % value) + + if not stem.util.tor_tools.is_valid_nickname(dir_source_comp[0].rstrip('-legacy')): + raise ValueError("Authority's nickname is invalid: %s" % dir_source_comp[0]) + elif not stem.util.tor_tools.is_valid_fingerprint(dir_source_comp[1]): + raise ValueError("Authority's fingerprint is invalid: %s" % dir_source_comp[1]) + elif not dir_source_comp[2]: + # https://trac.torproject.org/7055 + raise ValueError("Authority's hostname can't be blank: dir-source %s" % value) + elif not stem.util.connection.is_valid_ipv4_address(dir_source_comp[3]): + raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[3]) + elif not stem.util.connection.is_valid_port(dir_source_comp[4], allow_zero = True): + raise ValueError("Authority's DirPort is invalid: %s" % dir_source_comp[4]) + elif not stem.util.connection.is_valid_port(dir_source_comp[5]): + raise ValueError("Authority's ORPort is invalid: %s" % dir_source_comp[5]) + + descriptor.nickname = dir_source_comp[0] + descriptor.fingerprint = dir_source_comp[1] + descriptor.hostname = dir_source_comp[2] + descriptor.address = dir_source_comp[3] + descriptor.dir_port = None if dir_source_comp[4] == '0' else int(dir_source_comp[4]) + descriptor.or_port = int(dir_source_comp[5]) + descriptor.is_legacy = descriptor.nickname.endswith('-legacy') + + +_parse_contact_line = lambda descriptor, entries: setattr(descriptor, 'contact', _value('contact', entries)) +_parse_legacy_dir_key_line = _parse_forty_character_hex('legacy-dir-key', 'legacy_dir_key') +_parse_vote_digest_line = _parse_forty_character_hex('vote-digest', 'vote_digest') + + class DirectoryAuthority(Descriptor): """ Directory authority information obtained from a v3 network status document. @@ -1059,6 +1097,26 @@ class DirectoryAuthority(Descriptor): **\*** mandatory attribute """ + ATTRIBUTES = { + 'nickname': (None, _parse_dir_source_line), + 'fingerprint': (None, _parse_dir_source_line), + 'hostname': (None, _parse_dir_source_line), + 'address': (None, _parse_dir_source_line), + 'dir_port': (None, _parse_dir_source_line), + 'or_port': (None, _parse_dir_source_line), + 'is_legacy': (False, _parse_dir_source_line), + 'contact': (None, _parse_contact_line), + 'vote_digest': (None, _parse_vote_digest_line), + 'legacy_dir_key': (None, _parse_legacy_dir_key_line), + } + + PARSER_FOR_LINE = { + 'dir-source': _parse_dir_source_line, + 'contact': _parse_contact_line, + 'legacy-dir-key': _parse_legacy_dir_key_line, + 'vote-digest': _parse_vote_digest_line, + } + def __init__(self, raw_content, validate = True, is_vote = False): """ Parse a directory authority entry in a v3 network status document. @@ -1071,47 +1129,17 @@ class DirectoryAuthority(Descriptor): :raises: ValueError if the descriptor data is invalid """ - super(DirectoryAuthority, self).__init__(raw_content) - raw_content = stem.util.str_tools._to_unicode(raw_content) - - self.nickname = None - self.fingerprint = None - self.hostname = None - self.address = None - self.dir_port = None - self.or_port = None - self.is_legacy = False - self.contact = None - - self.vote_digest = None - - self.legacy_dir_key = None - self.key_certificate = None - - self._unrecognized_lines = [] - - self._parse(raw_content, validate, is_vote) - - def _parse(self, content, validate, is_vote): - """ - Parses the given content and applies the attributes. - - :param str content: descriptor content - :param bool validate: checks validity if True - :param bool is_vote: **True** if this is for a vote, **False** if it's for - a consensus - - :raises: **ValueError** if a validity check fails - """ + super(DirectoryAuthority, self).__init__(raw_content, lazy_load = not validate) + content = stem.util.str_tools._to_unicode(raw_content) # separate the directory authority entry from its key certificate key_div = content.find('\ndir-key-certificate-version') if key_div != -1: - key_cert_content = content[key_div + 1:] + self.key_certificate = KeyCertificate(content[key_div + 1:], validate) content = content[:key_div + 1] else: - key_cert_content = None + self.key_certificate = None entries = _get_descriptor_components(content, validate) @@ -1132,12 +1160,12 @@ class DirectoryAuthority(Descriptor): required_fields += ['contact'] if is_vote: - if not key_cert_content: + if not self.key_certificate: raise ValueError('Authority votes must have a key certificate:\n%s' % content) excluded_fields += ['vote-digest'] elif not is_vote: - if key_cert_content: + if self.key_certificate: raise ValueError("Authority consensus entries shouldn't have a key certificate:\n%s" % content) if not is_legacy: @@ -1154,82 +1182,14 @@ class DirectoryAuthority(Descriptor): type_label = 'votes' if is_vote else 'consensus entries' raise ValueError("Authority %s shouldn't have a '%s' line:\n%s" % (type_label, keyword, content)) - for keyword, values in list(entries.items()): - value, _, _ = values[0] - line = '%s %s' % (keyword, value) - # all known attributes can only appear at most once - if validate and len(values) > 1 and keyword in ('dir-source', 'contact', 'legacy-dir-key', 'vote-digest'): - raise ValueError("Authority entries can only have a single '%s' line, got %i:\n%s" % (keyword, len(values), content)) - - if keyword == 'dir-source': - # "dir-source" nickname identity address IP dirport orport - - dir_source_comp = value.split(' ') + for keyword, values in list(entries.items()): + if len(values) > 1 and keyword in ('dir-source', 'contact', 'legacy-dir-key', 'vote-digest'): + raise ValueError("Authority entries can only have a single '%s' line, got %i:\n%s" % (keyword, len(values), content)) - if len(dir_source_comp) < 6: - if not validate: - continue - - raise ValueError("Authority entry's 'dir-source' line must have six values: %s" % line) - - if validate: - if not stem.util.tor_tools.is_valid_nickname(dir_source_comp[0].rstrip('-legacy')): - raise ValueError("Authority's nickname is invalid: %s" % dir_source_comp[0]) - elif not stem.util.tor_tools.is_valid_fingerprint(dir_source_comp[1]): - raise ValueError("Authority's fingerprint is invalid: %s" % dir_source_comp[1]) - elif not dir_source_comp[2]: - # https://trac.torproject.org/7055 - raise ValueError("Authority's hostname can't be blank: %s" % line) - elif not stem.util.connection.is_valid_ipv4_address(dir_source_comp[3]): - raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[3]) - elif not stem.util.connection.is_valid_port(dir_source_comp[4], allow_zero = True): - raise ValueError("Authority's DirPort is invalid: %s" % dir_source_comp[4]) - elif not stem.util.connection.is_valid_port(dir_source_comp[5]): - raise ValueError("Authority's ORPort is invalid: %s" % dir_source_comp[5]) - elif not (dir_source_comp[4].isdigit() and dir_source_comp[5].isdigit()): - continue - - self.nickname = dir_source_comp[0] - self.fingerprint = dir_source_comp[1] - self.hostname = dir_source_comp[2] - self.address = dir_source_comp[3] - self.dir_port = None if dir_source_comp[4] == '0' else int(dir_source_comp[4]) - self.or_port = int(dir_source_comp[5]) - self.is_legacy = self.nickname.endswith('-legacy') - elif keyword == 'contact': - # "contact" string - - self.contact = value - elif keyword == 'legacy-dir-key': - # "legacy-dir-key" FINGERPRINT - - if validate and not stem.util.tor_tools.is_valid_fingerprint(value): - raise ValueError('Authority has a malformed legacy directory key: %s' % line) - - self.legacy_dir_key = value - elif keyword == 'vote-digest': - # "vote-digest" digest - - # technically not a fingerprint, but has the same characteristics - if validate and not stem.util.tor_tools.is_valid_fingerprint(value): - raise ValueError('Authority has a malformed vote digest: %s' % line) - - self.vote_digest = value - else: - self._unrecognized_lines.append(line) - - if key_cert_content: - self.key_certificate = KeyCertificate(key_cert_content, validate) - - def get_unrecognized_lines(self): - """ - Returns any unrecognized lines. - - :returns: a list of unrecognized lines - """ - - return self._unrecognized_lines + self._parse(entries, validate) + else: + self._entries = entries def _compare(self, other, method): if not isinstance(other, DirectoryAuthority): diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 09dc9c3..f626050 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -57,7 +57,7 @@ from stem.descriptor import ( _value, _values, _parse_timestamp_line, - _parse_sha1_digest_line, + _parse_forty_character_hex, _parse_key_block, ) @@ -270,18 +270,6 @@ def _parse_hibernating_line(descriptor, entries): descriptor.hibernating = value == '1' -def _parse_extrainfo_digest_line(descriptor, entries): - # this is forty hex digits which just so happens to be the same a - # fingerprint - - value = _value('extra-info-digest', entries) - - if not stem.util.tor_tools.is_valid_fingerprint(value): - raise ValueError('Extra-info digests should consist of forty hex digits: %s' % value) - - descriptor.extra_info_digest = value - - def _parse_hidden_service_dir_line(descriptor, entries): value = _value('hidden-service-dir', entries) @@ -379,6 +367,7 @@ def _parse_exit_policy(descriptor, entries): _parse_published_line = _parse_timestamp_line('published', 'published') +_parse_extrainfo_digest_line = _parse_forty_character_hex('extra-info-digest', 'extra_info_digest') _parse_read_history_line = functools.partial(_parse_history_line, 'read-history', 'read_history_end', 'read_history_interval', 'read_history_values') _parse_write_history_line = functools.partial(_parse_history_line, 'write-history', 'write_history_end', 'write_history_interval', 'write_history_values') _parse_ipv6_policy_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('ipv6-policy', entries))) @@ -390,7 +379,7 @@ _parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC K _parse_signing_key_line = _parse_key_block('signing-key', 'signing_key', 'RSA PUBLIC KEY') _parse_router_signature_line = _parse_key_block('router-signature', 'signature', 'SIGNATURE') _parse_ntor_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'ntor_onion_key', _value('ntor-onion-key', entries)) -_parse_router_digest_line = _parse_sha1_digest_line('router-digest', '_digest') +_parse_router_digest_line = _parse_forty_character_hex('router-digest', '_digest') class ServerDescriptor(Descriptor): diff --git a/test/unit/descriptor/networkstatus/directory_authority.py b/test/unit/descriptor/networkstatus/directory_authority.py index 1114518..a5bc647 100644 --- a/test/unit/descriptor/networkstatus/directory_authority.py +++ b/test/unit/descriptor/networkstatus/directory_authority.py @@ -164,7 +164,7 @@ class TestDirectoryAuthority(unittest.TestCase): self.assertRaises(ValueError, DirectoryAuthority, content) authority = DirectoryAuthority(content, False) - self.assertEqual(value, authority.fingerprint) + self.assertEqual(None, authority.fingerprint) def test_malformed_address(self): """ @@ -186,7 +186,7 @@ class TestDirectoryAuthority(unittest.TestCase): self.assertRaises(ValueError, DirectoryAuthority, content) authority = DirectoryAuthority(content, False) - self.assertEqual(value, authority.address) + self.assertEqual(None, authority.address) def test_malformed_port(self): """ @@ -219,9 +219,8 @@ class TestDirectoryAuthority(unittest.TestCase): authority = DirectoryAuthority(content, False) - expected_value = 399482 if value == '399482' else None actual_value = authority.or_port if include_or_port else authority.dir_port - self.assertEqual(expected_value, actual_value) + self.assertEqual(None, actual_value) def test_legacy_dir_key(self): """ @@ -247,7 +246,7 @@ class TestDirectoryAuthority(unittest.TestCase): self.assertRaises(ValueError, DirectoryAuthority, content) authority = DirectoryAuthority(content, False) - self.assertEqual(value, authority.legacy_dir_key) + self.assertEqual(None, authority.legacy_dir_key) def test_key_certificate(self): """