commit 4f63cbca6f88772c079bb1726c72bd30f6ed8901 Author: Damian Johnson <atagar@torproject.org> Date: Sat Jan 17 19:44:03 2015 -0800 KeyCertificate lazy loading Lazy loading support for part of network status documents. --- stem/descriptor/__init__.py | 14 ++ stem/descriptor/extrainfo_descriptor.py | 16 +- stem/descriptor/networkstatus.py | 210 ++++++++------------ stem/descriptor/server_descriptor.py | 13 +- .../descriptor/networkstatus/key_certificate.py | 24 +-- 5 files changed, 116 insertions(+), 161 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index ef96dd7..1e1acb5 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -319,6 +319,20 @@ def _values(line, entries): return [entry[0] for entry in entries[line]] +def _parse_timestamp_line(keyword, attribute): + # "<keyword>" YYYY-MM-DD HH:MM:SS + + def _parse(descriptor, entries): + value = _value(keyword, entries) + + try: + setattr(descriptor, attribute, stem.util.str_tools._parse_timestamp(value)) + except ValueError: + raise ValueError("Timestamp on %s line wasn't parsable: %s %s" % (keyword, keyword, value)) + + return _parse + + def _parse_sha1_digest_line(keyword, attribute): def _parse(descriptor, entries): value = _value(keyword, entries) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 4137e47..3413711 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -83,6 +83,7 @@ from stem.descriptor import ( _get_descriptor_components, _value, _values, + _parse_timestamp_line, _parse_sha1_digest_line, _parse_key_block, ) @@ -366,17 +367,6 @@ def _parse_cell_line(keyword, attribute, descriptor, entries): raise exc -def _parse_timestamp_line(keyword, attribute, descriptor, entries): - # "<keyword>" YYYY-MM-DD HH:MM:SS - - value = _value(keyword, entries) - - try: - setattr(descriptor, attribute, stem.util.str_tools._parse_timestamp(value)) - except ValueError: - raise ValueError("Timestamp on %s line wasn't parsable: %s %s" % (keyword, keyword, value)) - - def _parse_timestamp_and_interval_line(keyword, end_attribute, interval_attribute, descriptor, entries): # "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) @@ -521,8 +511,8 @@ _parse_dirreq_v3_share_line = functools.partial(_parse_dirreq_share_line, 'dirre _parse_cell_processed_cells_line = functools.partial(_parse_cell_line, 'cell-processed-cells', 'cell_processed_cells') _parse_cell_queued_cells_line = functools.partial(_parse_cell_line, 'cell-queued-cells', 'cell_queued_cells') _parse_cell_time_in_queue_line = functools.partial(_parse_cell_line, 'cell-time-in-queue', 'cell_time_in_queue') -_parse_published_line = functools.partial(_parse_timestamp_line, 'published', 'published') -_parse_geoip_start_time_line = functools.partial(_parse_timestamp_line, 'geoip-start-time', 'geoip_start_time') +_parse_published_line = _parse_timestamp_line('published', 'published') +_parse_geoip_start_time_line = _parse_timestamp_line('geoip-start-time', 'geoip_start_time') _parse_cell_stats_end_line = functools.partial(_parse_timestamp_and_interval_line, 'cell-stats-end', 'cell_stats_end', 'cell_stats_interval') _parse_entry_stats_end_line = functools.partial(_parse_timestamp_and_interval_line, 'entry-stats-end', 'entry_stats_end', 'entry_stats_interval') _parse_exit_stats_end_line = functools.partial(_parse_timestamp_and_interval_line, 'exit-stats-end', 'exit_stats_end', 'exit_stats_interval') diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 33beb4a..a70a7cf 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -62,6 +62,9 @@ from stem.descriptor import ( DocumentHandler, _get_descriptor_components, _read_until_keywords, + _value, + _parse_timestamp_line, + _parse_key_block, ) # Version 2 network status document fields, tuples of the form... @@ -1244,6 +1247,58 @@ class DirectoryAuthority(Descriptor): return self._compare(other, lambda s, o: s <= o) +def _parse_dir_key_certificate_version_line(descriptor, entries): + # "dir-key-certificate-version" version + + value = _value('dir-key-certificate-version', entries) + + if not value.isdigit(): + raise ValueError('Key certificate has a non-integer version: dir-key-certificate-version %s' % value) + + descriptor.version = int(value) + + if descriptor.version != 3: + raise ValueError("Expected a version 3 key certificate, got version '%i' instead" % descriptor.version) + + +def _parse_dir_address_line(descriptor, entries): + # "dir-address" IPPort + + value = _value('dir-address', entries) + + if ':' not in value: + raise ValueError("Key certificate's 'dir-address' is expected to be of the form ADDRESS:PORT: dir-address %s" % value) + + address, dirport = value.split(':', 1) + + if not stem.util.connection.is_valid_ipv4_address(address): + raise ValueError("Key certificate's address isn't a valid IPv4 address: dir-address %s" % value) + elif not stem.util.connection.is_valid_port(dirport): + raise ValueError("Key certificate's dirport is invalid: dir-address %s" % value) + + descriptor.address = address + descriptor.dir_port = int(dirport) + + +def _parse_fingerprint_line(descriptor, entries): + # "fingerprint" fingerprint + + value = _value('fingerprint', entries) + + if not stem.util.tor_tools.is_valid_fingerprint(value): + raise ValueError("Key certificate's fingerprint is malformed: fingerprint %s" % value) + + descriptor.fingerprint = value + + +_parse_dir_key_published_line = _parse_timestamp_line('dir-key-published', 'published') +_parse_dir_key_expires_line = _parse_timestamp_line('dir-key-expires', 'expires') +_parse_identity_key_line = _parse_key_block('dir-identity-key', 'identity_key', 'RSA PUBLIC KEY') +_parse_signing_key_line = _parse_key_block('dir-signing-key', 'signing_key', 'RSA PUBLIC KEY') +_parse_dir_key_crosscert_line = _parse_key_block('dir-key-crosscert', 'crosscert', 'ID SIGNATURE') +_parse_dir_key_certification_line = _parse_key_block('dir-key-certification', 'certification', 'SIGNATURE') + + class KeyCertificate(Descriptor): """ Directory key certificate for a v3 network status document. @@ -1263,35 +1318,35 @@ class KeyCertificate(Descriptor): **\*** mandatory attribute """ - def __init__(self, raw_content, validate = True): - super(KeyCertificate, self).__init__(raw_content) - raw_content = stem.util.str_tools._to_unicode(raw_content) - - self.version = None - self.address = None - self.dir_port = None - self.fingerprint = None - self.identity_key = None - self.published = None - self.expires = None - self.signing_key = None - self.crosscert = None - self.certification = None - - self._unrecognized_lines = [] - - self._parse(raw_content, validate) + ATTRIBUTES = { + 'version': (None, _parse_dir_key_certificate_version_line), + 'address': (None, _parse_dir_address_line), + 'dir_port': (None, _parse_dir_address_line), + 'fingerprint': (None, _parse_fingerprint_line), + 'identity_key': (None, _parse_identity_key_line), + 'published': (None, _parse_dir_key_published_line), + 'expires': (None, _parse_dir_key_expires_line), + 'signing_key': (None, _parse_signing_key_line), + 'crosscert': (None, _parse_dir_key_crosscert_line), + 'certification': (None, _parse_dir_key_certification_line), + } + + PARSER_FOR_LINE = { + 'dir-key-certificate-version': _parse_dir_key_certificate_version_line, + 'dir-address': _parse_dir_address_line, + 'fingerprint': _parse_fingerprint_line, + 'dir-key-published': _parse_dir_key_published_line, + 'dir-key-expires': _parse_dir_key_expires_line, + 'dir-identity-key': _parse_identity_key_line, + 'dir-signing-key': _parse_signing_key_line, + 'dir-key-crosscert': _parse_dir_key_crosscert_line, + 'dir-key-certification': _parse_dir_key_certification_line, + } - def _parse(self, content, validate): - """ - Parses the given content and applies the attributes. - - :param str content: descriptor content - :param bool validate: checks validity if **True** - - :raises: **ValueError** if a validity check fails - """ + def __init__(self, raw_content, validate = True): + super(KeyCertificate, self).__init__(raw_content, lazy_load = not validate) + content = stem.util.str_tools._to_unicode(raw_content) entries = _get_descriptor_components(content, validate) if validate: @@ -1311,104 +1366,9 @@ class KeyCertificate(Descriptor): if entry_count > 1: raise ValueError("Key certificates can only have a single '%s' line, got %i:\n%s" % (keyword, entry_count, content)) - for keyword, values in list(entries.items()): - value, block_type, block_contents = values[0] - line = '%s %s' % (keyword, value) - - if keyword == 'dir-key-certificate-version': - # "dir-key-certificate-version" version - - if not value.isdigit(): - if not validate: - continue - - raise ValueError('Key certificate has a non-integer version: %s' % line) - - self.version = int(value) - - if validate and self.version != 3: - raise ValueError("Expected a version 3 key certificate, got version '%i' instead" % self.version) - elif keyword == 'dir-address': - # "dir-address" IPPort - - if ':' not in value: - if not validate: - continue - - raise ValueError("Key certificate's 'dir-address' is expected to be of the form ADDRESS:PORT: %s" % line) - - address, dirport = value.split(':', 1) - - if validate: - if not stem.util.connection.is_valid_ipv4_address(address): - raise ValueError("Key certificate's address isn't a valid IPv4 address: %s" % line) - elif not stem.util.connection.is_valid_port(dirport): - raise ValueError("Key certificate's dirport is invalid: %s" % line) - elif not dirport.isdigit(): - continue - - self.address = address - self.dir_port = int(dirport) - elif keyword == 'fingerprint': - # "fingerprint" fingerprint - - if validate and not stem.util.tor_tools.is_valid_fingerprint(value): - raise ValueError("Key certificate's fingerprint is malformed: %s" % line) - - self.fingerprint = value - elif keyword in ('dir-key-published', 'dir-key-expires'): - # "dir-key-published" YYYY-MM-DD HH:MM:SS - # "dir-key-expires" YYYY-MM-DD HH:MM:SS - - try: - date_value = stem.util.str_tools._parse_timestamp(value) - - if keyword == 'dir-key-published': - self.published = date_value - elif keyword == 'dir-key-expires': - self.expires = date_value - except ValueError: - if validate: - raise ValueError("Key certificate's '%s' time wasn't parsable: %s" % (keyword, value)) - elif keyword == 'dir-identity-key': - # "dir-identity-key" NL a public key in PEM format - - if validate and (not block_contents or block_type != 'RSA PUBLIC KEY'): - raise ValueError("'dir-identity-key' should be followed by a RSA PUBLIC KEY block: %s" % line) - - self.identity_key = block_contents - elif keyword == 'dir-signing-key': - # "dir-signing-key" NL a key in PEM format - - if validate and (not block_contents or block_type != 'RSA PUBLIC KEY'): - raise ValueError("'dir-signing-key' should be followed by a RSA PUBLIC KEY block: %s" % line) - - self.signing_key = block_contents - elif keyword == 'dir-key-crosscert': - # "dir-key-crosscert" NL CrossSignature - - if validate and (not block_contents or block_type != 'ID SIGNATURE'): - raise ValueError("'dir-key-crosscert' should be followed by a ID SIGNATURE block: %s" % line) - - self.crosscert = block_contents - elif keyword == 'dir-key-certification': - # "dir-key-certification" NL Signature - - if validate and (not block_contents or block_type != 'SIGNATURE'): - raise ValueError("'dir-key-certification' should be followed by a SIGNATURE block: %s" % line) - - self.certification = block_contents - else: - self._unrecognized_lines.append(line) - - def get_unrecognized_lines(self): - """ - Returns any unrecognized lines. - - :returns: **list** of unrecognized lines - """ - - return self._unrecognized_lines + self._parse(entries, validate) + else: + self._entries = entries def _compare(self, other, method): if not isinstance(other, KeyCertificate): diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index a696d6a..09dc9c3 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -56,6 +56,7 @@ from stem.descriptor import ( _read_until_keywords, _value, _values, + _parse_timestamp_line, _parse_sha1_digest_line, _parse_key_block, ) @@ -241,17 +242,6 @@ def _parse_platform_line(descriptor, entries): pass -def _parse_published_line(descriptor, entries): - # "published" YYYY-MM-DD HH:MM:SS - - value = _value('published', entries) - - try: - descriptor.published = stem.util.str_tools._parse_timestamp(value) - except ValueError: - raise ValueError("Published line's time wasn't parsable: published %s" % value) - - def _parse_fingerprint_line(descriptor, entries): # This is forty hex digits split into space separated groups of four. # Checking that we match this pattern. @@ -388,6 +378,7 @@ def _parse_exit_policy(descriptor, entries): del descriptor._unparsed_exit_policy +_parse_published_line = _parse_timestamp_line('published', 'published') _parse_read_history_line = functools.partial(_parse_history_line, 'read-history', 'read_history_end', 'read_history_interval', 'read_history_values') _parse_write_history_line = functools.partial(_parse_history_line, 'write-history', 'write_history_end', 'write_history_interval', 'write_history_values') _parse_ipv6_policy_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('ipv6-policy', entries))) diff --git a/test/unit/descriptor/networkstatus/key_certificate.py b/test/unit/descriptor/networkstatus/key_certificate.py index 8939c7e..12facaa 100644 --- a/test/unit/descriptor/networkstatus/key_certificate.py +++ b/test/unit/descriptor/networkstatus/key_certificate.py @@ -112,23 +112,23 @@ class TestKeyCertificate(unittest.TestCase): self.assertEqual(80, certificate.dir_port) test_values = ( - ('', None, None), - (' ', None, None), - ('127.0.0.1', None, None), - ('127.0.0.1:', None, None), - ('80', None, None), - (':80', '', 80), - ('127.0.0.1a:80', '127.0.0.1a', 80), - ('127.0.0.1:80a', None, None), + (''), + (' '), + ('127.0.0.1'), + ('127.0.0.1:'), + ('80'), + (':80'), + ('127.0.0.1a:80'), + ('127.0.0.1:80a'), ) - for test_value, expected_address, expected_port in test_values: + for test_value in test_values: content = get_key_certificate({'dir-address': test_value}, content = True) self.assertRaises(ValueError, KeyCertificate, content) certificate = KeyCertificate(content, False) - self.assertEqual(expected_address, certificate.address) - self.assertEqual(expected_port, certificate.dir_port) + self.assertEqual(None, certificate.address) + self.assertEqual(None, certificate.dir_port) def test_fingerprint(self): """ @@ -147,7 +147,7 @@ class TestKeyCertificate(unittest.TestCase): self.assertRaises(ValueError, KeyCertificate, content) certificate = KeyCertificate(content, False) - self.assertEqual(test_value.strip(), certificate.fingerprint) + self.assertEqual(None, certificate.fingerprint) def test_time_fields(self): """