[tor-commits] [stem/master] KeyCertificate lazy loading
atagar at torproject.org
atagar at torproject.org
Sun Jan 25 22:37:34 UTC 2015
commit 4f63cbca6f88772c079bb1726c72bd30f6ed8901
Author: Damian Johnson <atagar at torproject.org>
Date: Sat Jan 17 19:44:03 2015 -0800
KeyCertificate lazy loading
Lazy loading support for part of network status documents.
---
stem/descriptor/__init__.py | 14 ++
stem/descriptor/extrainfo_descriptor.py | 16 +-
stem/descriptor/networkstatus.py | 210 ++++++++------------
stem/descriptor/server_descriptor.py | 13 +-
.../descriptor/networkstatus/key_certificate.py | 24 +--
5 files changed, 116 insertions(+), 161 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index ef96dd7..1e1acb5 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -319,6 +319,20 @@ def _values(line, entries):
return [entry[0] for entry in entries[line]]
+def _parse_timestamp_line(keyword, attribute):
+ # "<keyword>" YYYY-MM-DD HH:MM:SS
+
+ def _parse(descriptor, entries):
+ value = _value(keyword, entries)
+
+ try:
+ setattr(descriptor, attribute, stem.util.str_tools._parse_timestamp(value))
+ except ValueError:
+ raise ValueError("Timestamp on %s line wasn't parsable: %s %s" % (keyword, keyword, value))
+
+ return _parse
+
+
def _parse_sha1_digest_line(keyword, attribute):
def _parse(descriptor, entries):
value = _value(keyword, entries)
diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py
index 4137e47..3413711 100644
--- a/stem/descriptor/extrainfo_descriptor.py
+++ b/stem/descriptor/extrainfo_descriptor.py
@@ -83,6 +83,7 @@ from stem.descriptor import (
_get_descriptor_components,
_value,
_values,
+ _parse_timestamp_line,
_parse_sha1_digest_line,
_parse_key_block,
)
@@ -366,17 +367,6 @@ def _parse_cell_line(keyword, attribute, descriptor, entries):
raise exc
-def _parse_timestamp_line(keyword, attribute, descriptor, entries):
- # "<keyword>" YYYY-MM-DD HH:MM:SS
-
- value = _value(keyword, entries)
-
- try:
- setattr(descriptor, attribute, stem.util.str_tools._parse_timestamp(value))
- except ValueError:
- raise ValueError("Timestamp on %s line wasn't parsable: %s %s" % (keyword, keyword, value))
-
-
def _parse_timestamp_and_interval_line(keyword, end_attribute, interval_attribute, descriptor, entries):
# "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s)
@@ -521,8 +511,8 @@ _parse_dirreq_v3_share_line = functools.partial(_parse_dirreq_share_line, 'dirre
_parse_cell_processed_cells_line = functools.partial(_parse_cell_line, 'cell-processed-cells', 'cell_processed_cells')
_parse_cell_queued_cells_line = functools.partial(_parse_cell_line, 'cell-queued-cells', 'cell_queued_cells')
_parse_cell_time_in_queue_line = functools.partial(_parse_cell_line, 'cell-time-in-queue', 'cell_time_in_queue')
-_parse_published_line = functools.partial(_parse_timestamp_line, 'published', 'published')
-_parse_geoip_start_time_line = functools.partial(_parse_timestamp_line, 'geoip-start-time', 'geoip_start_time')
+_parse_published_line = _parse_timestamp_line('published', 'published')
+_parse_geoip_start_time_line = _parse_timestamp_line('geoip-start-time', 'geoip_start_time')
_parse_cell_stats_end_line = functools.partial(_parse_timestamp_and_interval_line, 'cell-stats-end', 'cell_stats_end', 'cell_stats_interval')
_parse_entry_stats_end_line = functools.partial(_parse_timestamp_and_interval_line, 'entry-stats-end', 'entry_stats_end', 'entry_stats_interval')
_parse_exit_stats_end_line = functools.partial(_parse_timestamp_and_interval_line, 'exit-stats-end', 'exit_stats_end', 'exit_stats_interval')
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index 33beb4a..a70a7cf 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -62,6 +62,9 @@ from stem.descriptor import (
DocumentHandler,
_get_descriptor_components,
_read_until_keywords,
+ _value,
+ _parse_timestamp_line,
+ _parse_key_block,
)
# Version 2 network status document fields, tuples of the form...
@@ -1244,6 +1247,58 @@ class DirectoryAuthority(Descriptor):
return self._compare(other, lambda s, o: s <= o)
+def _parse_dir_key_certificate_version_line(descriptor, entries):
+ # "dir-key-certificate-version" version
+
+ value = _value('dir-key-certificate-version', entries)
+
+ if not value.isdigit():
+ raise ValueError('Key certificate has a non-integer version: dir-key-certificate-version %s' % value)
+
+ descriptor.version = int(value)
+
+ if descriptor.version != 3:
+ raise ValueError("Expected a version 3 key certificate, got version '%i' instead" % descriptor.version)
+
+
+def _parse_dir_address_line(descriptor, entries):
+ # "dir-address" IPPort
+
+ value = _value('dir-address', entries)
+
+ if ':' not in value:
+ raise ValueError("Key certificate's 'dir-address' is expected to be of the form ADDRESS:PORT: dir-address %s" % value)
+
+ address, dirport = value.split(':', 1)
+
+ if not stem.util.connection.is_valid_ipv4_address(address):
+ raise ValueError("Key certificate's address isn't a valid IPv4 address: dir-address %s" % value)
+ elif not stem.util.connection.is_valid_port(dirport):
+ raise ValueError("Key certificate's dirport is invalid: dir-address %s" % value)
+
+ descriptor.address = address
+ descriptor.dir_port = int(dirport)
+
+
+def _parse_fingerprint_line(descriptor, entries):
+ # "fingerprint" fingerprint
+
+ value = _value('fingerprint', entries)
+
+ if not stem.util.tor_tools.is_valid_fingerprint(value):
+ raise ValueError("Key certificate's fingerprint is malformed: fingerprint %s" % value)
+
+ descriptor.fingerprint = value
+
+
+_parse_dir_key_published_line = _parse_timestamp_line('dir-key-published', 'published')
+_parse_dir_key_expires_line = _parse_timestamp_line('dir-key-expires', 'expires')
+_parse_identity_key_line = _parse_key_block('dir-identity-key', 'identity_key', 'RSA PUBLIC KEY')
+_parse_signing_key_line = _parse_key_block('dir-signing-key', 'signing_key', 'RSA PUBLIC KEY')
+_parse_dir_key_crosscert_line = _parse_key_block('dir-key-crosscert', 'crosscert', 'ID SIGNATURE')
+_parse_dir_key_certification_line = _parse_key_block('dir-key-certification', 'certification', 'SIGNATURE')
+
+
class KeyCertificate(Descriptor):
"""
Directory key certificate for a v3 network status document.
@@ -1263,35 +1318,35 @@ class KeyCertificate(Descriptor):
**\*** mandatory attribute
"""
- def __init__(self, raw_content, validate = True):
- super(KeyCertificate, self).__init__(raw_content)
- raw_content = stem.util.str_tools._to_unicode(raw_content)
-
- self.version = None
- self.address = None
- self.dir_port = None
- self.fingerprint = None
- self.identity_key = None
- self.published = None
- self.expires = None
- self.signing_key = None
- self.crosscert = None
- self.certification = None
-
- self._unrecognized_lines = []
-
- self._parse(raw_content, validate)
+ ATTRIBUTES = {
+ 'version': (None, _parse_dir_key_certificate_version_line),
+ 'address': (None, _parse_dir_address_line),
+ 'dir_port': (None, _parse_dir_address_line),
+ 'fingerprint': (None, _parse_fingerprint_line),
+ 'identity_key': (None, _parse_identity_key_line),
+ 'published': (None, _parse_dir_key_published_line),
+ 'expires': (None, _parse_dir_key_expires_line),
+ 'signing_key': (None, _parse_signing_key_line),
+ 'crosscert': (None, _parse_dir_key_crosscert_line),
+ 'certification': (None, _parse_dir_key_certification_line),
+ }
+
+ PARSER_FOR_LINE = {
+ 'dir-key-certificate-version': _parse_dir_key_certificate_version_line,
+ 'dir-address': _parse_dir_address_line,
+ 'fingerprint': _parse_fingerprint_line,
+ 'dir-key-published': _parse_dir_key_published_line,
+ 'dir-key-expires': _parse_dir_key_expires_line,
+ 'dir-identity-key': _parse_identity_key_line,
+ 'dir-signing-key': _parse_signing_key_line,
+ 'dir-key-crosscert': _parse_dir_key_crosscert_line,
+ 'dir-key-certification': _parse_dir_key_certification_line,
+ }
- def _parse(self, content, validate):
- """
- Parses the given content and applies the attributes.
-
- :param str content: descriptor content
- :param bool validate: checks validity if **True**
-
- :raises: **ValueError** if a validity check fails
- """
+ def __init__(self, raw_content, validate = True):
+ super(KeyCertificate, self).__init__(raw_content, lazy_load = not validate)
+ content = stem.util.str_tools._to_unicode(raw_content)
entries = _get_descriptor_components(content, validate)
if validate:
@@ -1311,104 +1366,9 @@ class KeyCertificate(Descriptor):
if entry_count > 1:
raise ValueError("Key certificates can only have a single '%s' line, got %i:\n%s" % (keyword, entry_count, content))
- for keyword, values in list(entries.items()):
- value, block_type, block_contents = values[0]
- line = '%s %s' % (keyword, value)
-
- if keyword == 'dir-key-certificate-version':
- # "dir-key-certificate-version" version
-
- if not value.isdigit():
- if not validate:
- continue
-
- raise ValueError('Key certificate has a non-integer version: %s' % line)
-
- self.version = int(value)
-
- if validate and self.version != 3:
- raise ValueError("Expected a version 3 key certificate, got version '%i' instead" % self.version)
- elif keyword == 'dir-address':
- # "dir-address" IPPort
-
- if ':' not in value:
- if not validate:
- continue
-
- raise ValueError("Key certificate's 'dir-address' is expected to be of the form ADDRESS:PORT: %s" % line)
-
- address, dirport = value.split(':', 1)
-
- if validate:
- if not stem.util.connection.is_valid_ipv4_address(address):
- raise ValueError("Key certificate's address isn't a valid IPv4 address: %s" % line)
- elif not stem.util.connection.is_valid_port(dirport):
- raise ValueError("Key certificate's dirport is invalid: %s" % line)
- elif not dirport.isdigit():
- continue
-
- self.address = address
- self.dir_port = int(dirport)
- elif keyword == 'fingerprint':
- # "fingerprint" fingerprint
-
- if validate and not stem.util.tor_tools.is_valid_fingerprint(value):
- raise ValueError("Key certificate's fingerprint is malformed: %s" % line)
-
- self.fingerprint = value
- elif keyword in ('dir-key-published', 'dir-key-expires'):
- # "dir-key-published" YYYY-MM-DD HH:MM:SS
- # "dir-key-expires" YYYY-MM-DD HH:MM:SS
-
- try:
- date_value = stem.util.str_tools._parse_timestamp(value)
-
- if keyword == 'dir-key-published':
- self.published = date_value
- elif keyword == 'dir-key-expires':
- self.expires = date_value
- except ValueError:
- if validate:
- raise ValueError("Key certificate's '%s' time wasn't parsable: %s" % (keyword, value))
- elif keyword == 'dir-identity-key':
- # "dir-identity-key" NL a public key in PEM format
-
- if validate and (not block_contents or block_type != 'RSA PUBLIC KEY'):
- raise ValueError("'dir-identity-key' should be followed by a RSA PUBLIC KEY block: %s" % line)
-
- self.identity_key = block_contents
- elif keyword == 'dir-signing-key':
- # "dir-signing-key" NL a key in PEM format
-
- if validate and (not block_contents or block_type != 'RSA PUBLIC KEY'):
- raise ValueError("'dir-signing-key' should be followed by a RSA PUBLIC KEY block: %s" % line)
-
- self.signing_key = block_contents
- elif keyword == 'dir-key-crosscert':
- # "dir-key-crosscert" NL CrossSignature
-
- if validate and (not block_contents or block_type != 'ID SIGNATURE'):
- raise ValueError("'dir-key-crosscert' should be followed by a ID SIGNATURE block: %s" % line)
-
- self.crosscert = block_contents
- elif keyword == 'dir-key-certification':
- # "dir-key-certification" NL Signature
-
- if validate and (not block_contents or block_type != 'SIGNATURE'):
- raise ValueError("'dir-key-certification' should be followed by a SIGNATURE block: %s" % line)
-
- self.certification = block_contents
- else:
- self._unrecognized_lines.append(line)
-
- def get_unrecognized_lines(self):
- """
- Returns any unrecognized lines.
-
- :returns: **list** of unrecognized lines
- """
-
- return self._unrecognized_lines
+ self._parse(entries, validate)
+ else:
+ self._entries = entries
def _compare(self, other, method):
if not isinstance(other, KeyCertificate):
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index a696d6a..09dc9c3 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -56,6 +56,7 @@ from stem.descriptor import (
_read_until_keywords,
_value,
_values,
+ _parse_timestamp_line,
_parse_sha1_digest_line,
_parse_key_block,
)
@@ -241,17 +242,6 @@ def _parse_platform_line(descriptor, entries):
pass
-def _parse_published_line(descriptor, entries):
- # "published" YYYY-MM-DD HH:MM:SS
-
- value = _value('published', entries)
-
- try:
- descriptor.published = stem.util.str_tools._parse_timestamp(value)
- except ValueError:
- raise ValueError("Published line's time wasn't parsable: published %s" % value)
-
-
def _parse_fingerprint_line(descriptor, entries):
# This is forty hex digits split into space separated groups of four.
# Checking that we match this pattern.
@@ -388,6 +378,7 @@ def _parse_exit_policy(descriptor, entries):
del descriptor._unparsed_exit_policy
+_parse_published_line = _parse_timestamp_line('published', 'published')
_parse_read_history_line = functools.partial(_parse_history_line, 'read-history', 'read_history_end', 'read_history_interval', 'read_history_values')
_parse_write_history_line = functools.partial(_parse_history_line, 'write-history', 'write_history_end', 'write_history_interval', 'write_history_values')
_parse_ipv6_policy_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('ipv6-policy', entries)))
diff --git a/test/unit/descriptor/networkstatus/key_certificate.py b/test/unit/descriptor/networkstatus/key_certificate.py
index 8939c7e..12facaa 100644
--- a/test/unit/descriptor/networkstatus/key_certificate.py
+++ b/test/unit/descriptor/networkstatus/key_certificate.py
@@ -112,23 +112,23 @@ class TestKeyCertificate(unittest.TestCase):
self.assertEqual(80, certificate.dir_port)
test_values = (
- ('', None, None),
- (' ', None, None),
- ('127.0.0.1', None, None),
- ('127.0.0.1:', None, None),
- ('80', None, None),
- (':80', '', 80),
- ('127.0.0.1a:80', '127.0.0.1a', 80),
- ('127.0.0.1:80a', None, None),
+ (''),
+ (' '),
+ ('127.0.0.1'),
+ ('127.0.0.1:'),
+ ('80'),
+ (':80'),
+ ('127.0.0.1a:80'),
+ ('127.0.0.1:80a'),
)
- for test_value, expected_address, expected_port in test_values:
+ for test_value in test_values:
content = get_key_certificate({'dir-address': test_value}, content = True)
self.assertRaises(ValueError, KeyCertificate, content)
certificate = KeyCertificate(content, False)
- self.assertEqual(expected_address, certificate.address)
- self.assertEqual(expected_port, certificate.dir_port)
+ self.assertEqual(None, certificate.address)
+ self.assertEqual(None, certificate.dir_port)
def test_fingerprint(self):
"""
@@ -147,7 +147,7 @@ class TestKeyCertificate(unittest.TestCase):
self.assertRaises(ValueError, KeyCertificate, content)
certificate = KeyCertificate(content, False)
- self.assertEqual(test_value.strip(), certificate.fingerprint)
+ self.assertEqual(None, certificate.fingerprint)
def test_time_fields(self):
"""
More information about the tor-commits
mailing list