[tor-commits] [stem/master] DirectoryAuthority lazy loading
atagar at torproject.org
atagar at torproject.org
Sun Jan 25 22:37:34 UTC 2015
commit 6f3a9d846d1226679bdd56dedce362d76c2a3be5
Author: Damian Johnson <atagar at torproject.org>
Date: Sat Jan 17 20:33:33 2015 -0800
DirectoryAuthority lazy loading
Another subsection of network status documents.
---
stem/descriptor/__init__.py | 6 +-
stem/descriptor/extrainfo_descriptor.py | 8 +-
stem/descriptor/networkstatus.py | 180 ++++++++------------
stem/descriptor/server_descriptor.py | 17 +-
.../networkstatus/directory_authority.py | 9 +-
5 files changed, 85 insertions(+), 135 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index 1e1acb5..cd9dcde 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -333,12 +333,14 @@ def _parse_timestamp_line(keyword, attribute):
return _parse
-def _parse_sha1_digest_line(keyword, attribute):
+def _parse_forty_character_hex(keyword, attribute):
+ # format of fingerprints, sha1 digests, etc
+
def _parse(descriptor, entries):
value = _value(keyword, entries)
if not stem.util.tor_tools.is_hex_digits(value, 40):
- raise ValueError('%s line had an invalid sha1 digest: %s %s' % (keyword, keyword, value))
+ raise ValueError('%s line had an invalid value (should be 40 hex characters): %s %s' % (keyword, keyword, value))
setattr(descriptor, attribute, value)
diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py
index 3413711..124ce16 100644
--- a/stem/descriptor/extrainfo_descriptor.py
+++ b/stem/descriptor/extrainfo_descriptor.py
@@ -84,7 +84,7 @@ from stem.descriptor import (
_value,
_values,
_parse_timestamp_line,
- _parse_sha1_digest_line,
+ _parse_forty_character_hex,
_parse_key_block,
)
@@ -498,8 +498,8 @@ def _parse_bridge_ip_transports_line(descriptor, entries):
descriptor.ip_transports = ip_transports
-_parse_geoip_db_digest_line = _parse_sha1_digest_line('geoip-db-digest', 'geoip_db_digest')
-_parse_geoip6_db_digest_line = _parse_sha1_digest_line('geoip6-db-digest', 'geoip6_db_digest')
+_parse_geoip_db_digest_line = _parse_forty_character_hex('geoip-db-digest', 'geoip_db_digest')
+_parse_geoip6_db_digest_line = _parse_forty_character_hex('geoip6-db-digest', 'geoip6_db_digest')
_parse_dirreq_v2_resp_line = functools.partial(_parse_dirreq_line, 'dirreq-v2-resp', 'dir_v2_responses', 'dir_v2_responses_unknown')
_parse_dirreq_v3_resp_line = functools.partial(_parse_dirreq_line, 'dirreq-v3-resp', 'dir_v3_responses', 'dir_v3_responses_unknown')
_parse_dirreq_v2_direct_dl_line = functools.partial(_parse_dirreq_line, 'dirreq-v2-direct-dl', 'dir_v2_direct_dl', 'dir_v2_direct_dl_unknown')
@@ -532,7 +532,7 @@ _parse_dirreq_v3_reqs_line = functools.partial(_parse_geoip_to_count_line, 'dirr
_parse_geoip_client_origins_line = functools.partial(_parse_geoip_to_count_line, 'geoip-client-origins', 'geoip_client_origins')
_parse_entry_ips_line = functools.partial(_parse_geoip_to_count_line, 'entry-ips', 'entry_ips')
_parse_bridge_ips_line = functools.partial(_parse_geoip_to_count_line, 'bridge-ips', 'bridge_ips')
-_parse_router_digest_line = _parse_sha1_digest_line('router-digest', '_digest')
+_parse_router_digest_line = _parse_forty_character_hex('router-digest', '_digest')
_parse_router_signature_line = _parse_key_block('router-signature', 'signature', 'SIGNATURE')
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index a70a7cf..70f325b 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -64,6 +64,7 @@ from stem.descriptor import (
_read_until_keywords,
_value,
_parse_timestamp_line,
+ _parse_forty_character_hex,
_parse_key_block,
)
@@ -1027,6 +1028,43 @@ def _parse_int_mappings(keyword, value, validate):
return results
+def _parse_dir_source_line(descriptor, entries):
+ # "dir-source" nickname identity address IP dirport orport
+
+ value = _value('dir-source', entries)
+ dir_source_comp = value.split(' ')
+
+ if len(dir_source_comp) < 6:
+ raise ValueError("Authority entry's 'dir-source' line must have six values: dir-source %s" % value)
+
+ if not stem.util.tor_tools.is_valid_nickname(dir_source_comp[0].rstrip('-legacy')):
+ raise ValueError("Authority's nickname is invalid: %s" % dir_source_comp[0])
+ elif not stem.util.tor_tools.is_valid_fingerprint(dir_source_comp[1]):
+ raise ValueError("Authority's fingerprint is invalid: %s" % dir_source_comp[1])
+ elif not dir_source_comp[2]:
+ # https://trac.torproject.org/7055
+ raise ValueError("Authority's hostname can't be blank: dir-source %s" % value)
+ elif not stem.util.connection.is_valid_ipv4_address(dir_source_comp[3]):
+ raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[3])
+ elif not stem.util.connection.is_valid_port(dir_source_comp[4], allow_zero = True):
+ raise ValueError("Authority's DirPort is invalid: %s" % dir_source_comp[4])
+ elif not stem.util.connection.is_valid_port(dir_source_comp[5]):
+ raise ValueError("Authority's ORPort is invalid: %s" % dir_source_comp[5])
+
+ descriptor.nickname = dir_source_comp[0]
+ descriptor.fingerprint = dir_source_comp[1]
+ descriptor.hostname = dir_source_comp[2]
+ descriptor.address = dir_source_comp[3]
+ descriptor.dir_port = None if dir_source_comp[4] == '0' else int(dir_source_comp[4])
+ descriptor.or_port = int(dir_source_comp[5])
+ descriptor.is_legacy = descriptor.nickname.endswith('-legacy')
+
+
+_parse_contact_line = lambda descriptor, entries: setattr(descriptor, 'contact', _value('contact', entries))
+_parse_legacy_dir_key_line = _parse_forty_character_hex('legacy-dir-key', 'legacy_dir_key')
+_parse_vote_digest_line = _parse_forty_character_hex('vote-digest', 'vote_digest')
+
+
class DirectoryAuthority(Descriptor):
"""
Directory authority information obtained from a v3 network status document.
@@ -1059,6 +1097,26 @@ class DirectoryAuthority(Descriptor):
**\*** mandatory attribute
"""
+ ATTRIBUTES = {
+ 'nickname': (None, _parse_dir_source_line),
+ 'fingerprint': (None, _parse_dir_source_line),
+ 'hostname': (None, _parse_dir_source_line),
+ 'address': (None, _parse_dir_source_line),
+ 'dir_port': (None, _parse_dir_source_line),
+ 'or_port': (None, _parse_dir_source_line),
+ 'is_legacy': (False, _parse_dir_source_line),
+ 'contact': (None, _parse_contact_line),
+ 'vote_digest': (None, _parse_vote_digest_line),
+ 'legacy_dir_key': (None, _parse_legacy_dir_key_line),
+ }
+
+ PARSER_FOR_LINE = {
+ 'dir-source': _parse_dir_source_line,
+ 'contact': _parse_contact_line,
+ 'legacy-dir-key': _parse_legacy_dir_key_line,
+ 'vote-digest': _parse_vote_digest_line,
+ }
+
def __init__(self, raw_content, validate = True, is_vote = False):
"""
Parse a directory authority entry in a v3 network status document.
@@ -1071,47 +1129,17 @@ class DirectoryAuthority(Descriptor):
:raises: ValueError if the descriptor data is invalid
"""
- super(DirectoryAuthority, self).__init__(raw_content)
- raw_content = stem.util.str_tools._to_unicode(raw_content)
-
- self.nickname = None
- self.fingerprint = None
- self.hostname = None
- self.address = None
- self.dir_port = None
- self.or_port = None
- self.is_legacy = False
- self.contact = None
-
- self.vote_digest = None
-
- self.legacy_dir_key = None
- self.key_certificate = None
-
- self._unrecognized_lines = []
-
- self._parse(raw_content, validate, is_vote)
-
- def _parse(self, content, validate, is_vote):
- """
- Parses the given content and applies the attributes.
-
- :param str content: descriptor content
- :param bool validate: checks validity if True
- :param bool is_vote: **True** if this is for a vote, **False** if it's for
- a consensus
-
- :raises: **ValueError** if a validity check fails
- """
+ super(DirectoryAuthority, self).__init__(raw_content, lazy_load = not validate)
+ content = stem.util.str_tools._to_unicode(raw_content)
# separate the directory authority entry from its key certificate
key_div = content.find('\ndir-key-certificate-version')
if key_div != -1:
- key_cert_content = content[key_div + 1:]
+ self.key_certificate = KeyCertificate(content[key_div + 1:], validate)
content = content[:key_div + 1]
else:
- key_cert_content = None
+ self.key_certificate = None
entries = _get_descriptor_components(content, validate)
@@ -1132,12 +1160,12 @@ class DirectoryAuthority(Descriptor):
required_fields += ['contact']
if is_vote:
- if not key_cert_content:
+ if not self.key_certificate:
raise ValueError('Authority votes must have a key certificate:\n%s' % content)
excluded_fields += ['vote-digest']
elif not is_vote:
- if key_cert_content:
+ if self.key_certificate:
raise ValueError("Authority consensus entries shouldn't have a key certificate:\n%s" % content)
if not is_legacy:
@@ -1154,82 +1182,14 @@ class DirectoryAuthority(Descriptor):
type_label = 'votes' if is_vote else 'consensus entries'
raise ValueError("Authority %s shouldn't have a '%s' line:\n%s" % (type_label, keyword, content))
- for keyword, values in list(entries.items()):
- value, _, _ = values[0]
- line = '%s %s' % (keyword, value)
-
# all known attributes can only appear at most once
- if validate and len(values) > 1 and keyword in ('dir-source', 'contact', 'legacy-dir-key', 'vote-digest'):
- raise ValueError("Authority entries can only have a single '%s' line, got %i:\n%s" % (keyword, len(values), content))
-
- if keyword == 'dir-source':
- # "dir-source" nickname identity address IP dirport orport
-
- dir_source_comp = value.split(' ')
+ for keyword, values in list(entries.items()):
+ if len(values) > 1 and keyword in ('dir-source', 'contact', 'legacy-dir-key', 'vote-digest'):
+ raise ValueError("Authority entries can only have a single '%s' line, got %i:\n%s" % (keyword, len(values), content))
- if len(dir_source_comp) < 6:
- if not validate:
- continue
-
- raise ValueError("Authority entry's 'dir-source' line must have six values: %s" % line)
-
- if validate:
- if not stem.util.tor_tools.is_valid_nickname(dir_source_comp[0].rstrip('-legacy')):
- raise ValueError("Authority's nickname is invalid: %s" % dir_source_comp[0])
- elif not stem.util.tor_tools.is_valid_fingerprint(dir_source_comp[1]):
- raise ValueError("Authority's fingerprint is invalid: %s" % dir_source_comp[1])
- elif not dir_source_comp[2]:
- # https://trac.torproject.org/7055
- raise ValueError("Authority's hostname can't be blank: %s" % line)
- elif not stem.util.connection.is_valid_ipv4_address(dir_source_comp[3]):
- raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[3])
- elif not stem.util.connection.is_valid_port(dir_source_comp[4], allow_zero = True):
- raise ValueError("Authority's DirPort is invalid: %s" % dir_source_comp[4])
- elif not stem.util.connection.is_valid_port(dir_source_comp[5]):
- raise ValueError("Authority's ORPort is invalid: %s" % dir_source_comp[5])
- elif not (dir_source_comp[4].isdigit() and dir_source_comp[5].isdigit()):
- continue
-
- self.nickname = dir_source_comp[0]
- self.fingerprint = dir_source_comp[1]
- self.hostname = dir_source_comp[2]
- self.address = dir_source_comp[3]
- self.dir_port = None if dir_source_comp[4] == '0' else int(dir_source_comp[4])
- self.or_port = int(dir_source_comp[5])
- self.is_legacy = self.nickname.endswith('-legacy')
- elif keyword == 'contact':
- # "contact" string
-
- self.contact = value
- elif keyword == 'legacy-dir-key':
- # "legacy-dir-key" FINGERPRINT
-
- if validate and not stem.util.tor_tools.is_valid_fingerprint(value):
- raise ValueError('Authority has a malformed legacy directory key: %s' % line)
-
- self.legacy_dir_key = value
- elif keyword == 'vote-digest':
- # "vote-digest" digest
-
- # technically not a fingerprint, but has the same characteristics
- if validate and not stem.util.tor_tools.is_valid_fingerprint(value):
- raise ValueError('Authority has a malformed vote digest: %s' % line)
-
- self.vote_digest = value
- else:
- self._unrecognized_lines.append(line)
-
- if key_cert_content:
- self.key_certificate = KeyCertificate(key_cert_content, validate)
-
- def get_unrecognized_lines(self):
- """
- Returns any unrecognized lines.
-
- :returns: a list of unrecognized lines
- """
-
- return self._unrecognized_lines
+ self._parse(entries, validate)
+ else:
+ self._entries = entries
def _compare(self, other, method):
if not isinstance(other, DirectoryAuthority):
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index 09dc9c3..f626050 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -57,7 +57,7 @@ from stem.descriptor import (
_value,
_values,
_parse_timestamp_line,
- _parse_sha1_digest_line,
+ _parse_forty_character_hex,
_parse_key_block,
)
@@ -270,18 +270,6 @@ def _parse_hibernating_line(descriptor, entries):
descriptor.hibernating = value == '1'
-def _parse_extrainfo_digest_line(descriptor, entries):
- # this is forty hex digits which just so happens to be the same a
- # fingerprint
-
- value = _value('extra-info-digest', entries)
-
- if not stem.util.tor_tools.is_valid_fingerprint(value):
- raise ValueError('Extra-info digests should consist of forty hex digits: %s' % value)
-
- descriptor.extra_info_digest = value
-
-
def _parse_hidden_service_dir_line(descriptor, entries):
value = _value('hidden-service-dir', entries)
@@ -379,6 +367,7 @@ def _parse_exit_policy(descriptor, entries):
_parse_published_line = _parse_timestamp_line('published', 'published')
+_parse_extrainfo_digest_line = _parse_forty_character_hex('extra-info-digest', 'extra_info_digest')
_parse_read_history_line = functools.partial(_parse_history_line, 'read-history', 'read_history_end', 'read_history_interval', 'read_history_values')
_parse_write_history_line = functools.partial(_parse_history_line, 'write-history', 'write_history_end', 'write_history_interval', 'write_history_values')
_parse_ipv6_policy_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('ipv6-policy', entries)))
@@ -390,7 +379,7 @@ _parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC K
_parse_signing_key_line = _parse_key_block('signing-key', 'signing_key', 'RSA PUBLIC KEY')
_parse_router_signature_line = _parse_key_block('router-signature', 'signature', 'SIGNATURE')
_parse_ntor_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'ntor_onion_key', _value('ntor-onion-key', entries))
-_parse_router_digest_line = _parse_sha1_digest_line('router-digest', '_digest')
+_parse_router_digest_line = _parse_forty_character_hex('router-digest', '_digest')
class ServerDescriptor(Descriptor):
diff --git a/test/unit/descriptor/networkstatus/directory_authority.py b/test/unit/descriptor/networkstatus/directory_authority.py
index 1114518..a5bc647 100644
--- a/test/unit/descriptor/networkstatus/directory_authority.py
+++ b/test/unit/descriptor/networkstatus/directory_authority.py
@@ -164,7 +164,7 @@ class TestDirectoryAuthority(unittest.TestCase):
self.assertRaises(ValueError, DirectoryAuthority, content)
authority = DirectoryAuthority(content, False)
- self.assertEqual(value, authority.fingerprint)
+ self.assertEqual(None, authority.fingerprint)
def test_malformed_address(self):
"""
@@ -186,7 +186,7 @@ class TestDirectoryAuthority(unittest.TestCase):
self.assertRaises(ValueError, DirectoryAuthority, content)
authority = DirectoryAuthority(content, False)
- self.assertEqual(value, authority.address)
+ self.assertEqual(None, authority.address)
def test_malformed_port(self):
"""
@@ -219,9 +219,8 @@ class TestDirectoryAuthority(unittest.TestCase):
authority = DirectoryAuthority(content, False)
- expected_value = 399482 if value == '399482' else None
actual_value = authority.or_port if include_or_port else authority.dir_port
- self.assertEqual(expected_value, actual_value)
+ self.assertEqual(None, actual_value)
def test_legacy_dir_key(self):
"""
@@ -247,7 +246,7 @@ class TestDirectoryAuthority(unittest.TestCase):
self.assertRaises(ValueError, DirectoryAuthority, content)
authority = DirectoryAuthority(content, False)
- self.assertEqual(value, authority.legacy_dir_key)
+ self.assertEqual(None, authority.legacy_dir_key)
def test_key_certificate(self):
"""
More information about the tor-commits
mailing list