[tor-commits] [stem/master] Lazy parsing for ServerDescriptor subclasses
atagar at torproject.org
atagar at torproject.org
Sun Jan 25 22:37:34 UTC 2015
commit eb69babe993d2b8c7a155428765aca251f569ba2
Author: Damian Johnson <atagar at torproject.org>
Date: Sun Jan 11 15:44:59 2015 -0800
Lazy parsing for ServerDescriptor subclasses
Including ServerDescriptor's RelayDescriptor and BridgeDescriptor subclasses.
This is actually a tad nicer than what we orgininally had in that we no longer
need a _parse() function for subclasses. Rather, they simply specify the
additional fields they include.
---
stem/descriptor/server_descriptor.py | 261 ++++++++++++++++++----------------
1 file changed, 135 insertions(+), 126 deletions(-)
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index 7ecc9c3..944c6c0 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -396,73 +396,31 @@ def _parse_history_line(descriptor, entries, keyword):
return timestamp, interval, history_values
+def _parse_router_digest_line(descriptor, entries):
+ descriptor._digest = _value('router-digest', entries)
+
+ if not stem.util.tor_tools.is_hex_digits(descriptor._digest, 40):
+ raise ValueError('Router digest line had an invalid sha1 digest: router-digest %s' % descriptor._digest)
+
+
+def _key_block(entries, keyword, expected_block_type):
+ value, block_type, block_contents = entries[keyword][0]
+
+ if not block_contents or block_type != expected_block_type:
+ raise ValueError("'%s' should be followed by a %s block" % (keyword, expected_block_type))
+
+ return block_contents
+
+
_parse_ipv6_policy_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('ipv6-policy', entries)))
_parse_allow_single_hop_exits_line = lambda descriptor, entries: setattr(descriptor, 'allow_single_hop_exits', True)
_parse_caches_extra_info_line = lambda descriptor, entries: setattr(descriptor, 'extra_info_cache', True)
_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', set(_value('family', entries).split(' ')))
_parse_eventdns_line = lambda descriptor, entries: setattr(descriptor, 'eventdns', _value('eventdns', entries) == '1')
-
-
-SERVER_DESCRIPTOR_ATTRIBUTES = {
- 'nickname': (None, _parse_router_line),
- 'fingerprint': (None, _parse_fingerprint_line),
- 'published': (None, _parse_published_line),
-
- 'address': (None, _parse_router_line),
- 'or_port': (None, _parse_router_line),
- 'socks_port': (None, _parse_router_line),
- 'dir_port': (None, _parse_router_line),
-
- 'tor_version': (None, _parse_platform_line),
- 'operating_system': (None, _parse_platform_line),
- 'uptime': (None, _parse_uptime_line),
- 'exit_policy_v6': (DEFAULT_IPV6_EXIT_POLICY, _parse_ipv6_policy_line),
- 'family': (set(), _parse_family_line),
-
- 'average_bandwidth': (None, _parse_bandwidth_line),
- 'burst_bandwidth': (None, _parse_bandwidth_line),
- 'observed_bandwidth': (None, _parse_bandwidth_line),
-
- 'link_protocols': (None, _parse_protocols_line),
- 'circuit_protocols': (None, _parse_protocols_line),
- 'hibernating': (False, _parse_hibernating_line),
- 'allow_single_hop_exits': (False, _parse_allow_single_hop_exits_line),
- 'extra_info_cache': (False, _parse_caches_extra_info_line),
- 'extra_info_digest': (None, _parse_extrainfo_digest_line),
- 'hidden_service_dir': (None, _parse_hidden_service_dir_line),
- 'eventdns': (None, _parse_eventdns_line),
- 'or_addresses': ([], _parse_or_address_line),
-
- 'read_history_end': (None, _parse_read_history_line),
- 'read_history_interval': (None, _parse_read_history_line),
- 'read_history_values': (None, _parse_read_history_line),
-
- 'write_history_end': (None, _parse_write_history_line),
- 'write_history_interval': (None, _parse_write_history_line),
- 'write_history_values': (None, _parse_write_history_line),
-}
-
-
-PARSER_FOR_LINE = {
- 'router': _parse_router_line,
- 'bandwidth': _parse_bandwidth_line,
- 'platform': _parse_platform_line,
- 'published': _parse_published_line,
- 'fingerprint': _parse_fingerprint_line,
- 'hibernating': _parse_hibernating_line,
- 'extra-info-digest': _parse_extrainfo_digest_line,
- 'hidden-service-dir': _parse_hidden_service_dir_line,
- 'uptime': _parse_uptime_line,
- 'protocols': _parse_protocols_line,
- 'or-address': _parse_or_address_line,
- 'read-history': _parse_read_history_line,
- 'write-history': _parse_write_history_line,
- 'ipv6-policy': _parse_ipv6_policy_line,
- 'allow-single-hop-exits': _parse_allow_single_hop_exits_line,
- 'caches-extra-info': _parse_caches_extra_info_line,
- 'family': _parse_family_line,
- 'eventdns': _parse_eventdns_line,
-}
+_parse_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'onion_key', _key_block(entries, 'onion-key', 'RSA PUBLIC KEY'))
+_parse_signing_key_line = lambda descriptor, entries: setattr(descriptor, 'signing_key', _key_block(entries, 'signing-key', 'RSA PUBLIC KEY'))
+_parse_router_signature_line = lambda descriptor, entries: setattr(descriptor, 'signature', _key_block(entries, 'router-signature', 'SIGNATURE'))
+_parse_ntor_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'ntor_onion_key', _value('ntor-onion-key', entries))
class ServerDescriptor(Descriptor):
@@ -638,13 +596,13 @@ class ServerDescriptor(Descriptor):
# set defaults
- for attr in SERVER_DESCRIPTOR_ATTRIBUTES:
- setattr(self, attr, SERVER_DESCRIPTOR_ATTRIBUTES[attr][0])
+ for attr in self._attributes():
+ setattr(self, attr, self._attributes()[attr][0])
for keyword, values in list(entries.items()):
try:
- if keyword in PARSER_FOR_LINE:
- PARSER_FOR_LINE[keyword](self, entries)
+ if keyword in self._parser_for_line():
+ self._parser_for_line()[keyword](self, entries)
elif keyword == 'contact':
pass # parsed as a bytes field earlier
else:
@@ -710,11 +668,85 @@ class ServerDescriptor(Descriptor):
def _last_keyword(self):
return 'router-signature'
+ @lru_cache()
+ def _attributes(self):
+ """
+ Provides a mapping of attributes we should have...
+
+ attrubute => (default_value, parsing_function)
+ """
+
+ return {
+ 'nickname': (None, _parse_router_line),
+ 'fingerprint': (None, _parse_fingerprint_line),
+ 'published': (None, _parse_published_line),
+
+ 'address': (None, _parse_router_line),
+ 'or_port': (None, _parse_router_line),
+ 'socks_port': (None, _parse_router_line),
+ 'dir_port': (None, _parse_router_line),
+
+ 'tor_version': (None, _parse_platform_line),
+ 'operating_system': (None, _parse_platform_line),
+ 'uptime': (None, _parse_uptime_line),
+ 'exit_policy_v6': (DEFAULT_IPV6_EXIT_POLICY, _parse_ipv6_policy_line),
+ 'family': (set(), _parse_family_line),
+
+ 'average_bandwidth': (None, _parse_bandwidth_line),
+ 'burst_bandwidth': (None, _parse_bandwidth_line),
+ 'observed_bandwidth': (None, _parse_bandwidth_line),
+
+ 'link_protocols': (None, _parse_protocols_line),
+ 'circuit_protocols': (None, _parse_protocols_line),
+ 'hibernating': (False, _parse_hibernating_line),
+ 'allow_single_hop_exits': (False, _parse_allow_single_hop_exits_line),
+ 'extra_info_cache': (False, _parse_caches_extra_info_line),
+ 'extra_info_digest': (None, _parse_extrainfo_digest_line),
+ 'hidden_service_dir': (None, _parse_hidden_service_dir_line),
+ 'eventdns': (None, _parse_eventdns_line),
+ 'or_addresses': ([], _parse_or_address_line),
+
+ 'read_history_end': (None, _parse_read_history_line),
+ 'read_history_interval': (None, _parse_read_history_line),
+ 'read_history_values': (None, _parse_read_history_line),
+
+ 'write_history_end': (None, _parse_write_history_line),
+ 'write_history_interval': (None, _parse_write_history_line),
+ 'write_history_values': (None, _parse_write_history_line),
+ }
+
+ @lru_cache()
+ def _parser_for_line(self):
+ """
+ Provides the parsing function for the line with a given keyword.
+ """
+
+ return {
+ 'router': _parse_router_line,
+ 'bandwidth': _parse_bandwidth_line,
+ 'platform': _parse_platform_line,
+ 'published': _parse_published_line,
+ 'fingerprint': _parse_fingerprint_line,
+ 'hibernating': _parse_hibernating_line,
+ 'extra-info-digest': _parse_extrainfo_digest_line,
+ 'hidden-service-dir': _parse_hidden_service_dir_line,
+ 'uptime': _parse_uptime_line,
+ 'protocols': _parse_protocols_line,
+ 'or-address': _parse_or_address_line,
+ 'read-history': _parse_read_history_line,
+ 'write-history': _parse_write_history_line,
+ 'ipv6-policy': _parse_ipv6_policy_line,
+ 'allow-single-hop-exits': _parse_allow_single_hop_exits_line,
+ 'caches-extra-info': _parse_caches_extra_info_line,
+ 'family': _parse_family_line,
+ 'eventdns': _parse_eventdns_line,
+ }
+
def __getattr__(self, name):
# If attribute isn't already present we might be lazy loading it...
- if self._lazy_loading and name in SERVER_DESCRIPTOR_ATTRIBUTES:
- default, parsing_function = SERVER_DESCRIPTOR_ATTRIBUTES[name]
+ if self._lazy_loading and name in self._attributes():
+ default, parsing_function = self._attributes()[name]
try:
if parsing_function:
@@ -727,7 +759,11 @@ class ServerDescriptor(Descriptor):
del self._exit_policy_list
except (ValueError, KeyError):
- setattr(self, name, default)
+ try:
+ # despite having a validation failure check to see if we set something
+ return super(ServerDescriptor, self).__getattribute__(name)
+ except AttributeError:
+ setattr(self, name, default)
return super(ServerDescriptor, self).__getattribute__(name)
@@ -746,11 +782,6 @@ class RelayDescriptor(ServerDescriptor):
"""
def __init__(self, raw_contents, validate = True, annotations = None):
- self.onion_key = None
- self.ntor_onion_key = None
- self.signing_key = None
- self.signature = None
-
super(RelayDescriptor, self).__init__(raw_contents, validate, annotations)
# validate the descriptor if required
@@ -874,45 +905,30 @@ class RelayDescriptor(ServerDescriptor):
if digest != local_digest:
raise ValueError('Decrypted digest does not match local digest (calculated: %s, local: %s)' % (digest, local_digest))
- def _parse(self, entries, validate):
- entries = dict(entries) # shallow copy since we're destructive
-
- # handles fields only in server descriptors
-
- for keyword, values in list(entries.items()):
- value, block_type, block_contents = values[0]
- line = '%s %s' % (keyword, value)
-
- if keyword == 'onion-key':
- if validate and (not block_contents or block_type != 'RSA PUBLIC KEY'):
- raise ValueError("'onion-key' should be followed by a RSA PUBLIC KEY block: %s" % line)
-
- self.onion_key = block_contents
- del entries['onion-key']
- elif keyword == 'ntor-onion-key':
- self.ntor_onion_key = value
- del entries['ntor-onion-key']
- elif keyword == 'signing-key':
- if validate and (not block_contents or block_type != 'RSA PUBLIC KEY'):
- raise ValueError("'signing-key' should be followed by a RSA PUBLIC KEY block: %s" % line)
-
- self.signing_key = block_contents
- del entries['signing-key']
- elif keyword == 'router-signature':
- if validate and (not block_contents or block_type != 'SIGNATURE'):
- raise ValueError("'router-signature' should be followed by a SIGNATURE block: %s" % line)
-
- self.signature = block_contents
- del entries['router-signature']
-
- ServerDescriptor._parse(self, entries, validate)
-
def _compare(self, other, method):
if not isinstance(other, RelayDescriptor):
return False
return method(str(self).strip(), str(other).strip())
+ @lru_cache()
+ def _attributes(self):
+ return dict(super(RelayDescriptor, self)._attributes(), **{
+ 'onion_key': (None, _parse_onion_key_line),
+ 'ntor_onion_key': (None, _parse_ntor_onion_key_line),
+ 'signing_key': (None, _parse_signing_key_line),
+ 'signature': (None, _parse_router_signature_line),
+ })
+
+ @lru_cache()
+ def _parser_for_line(self):
+ return dict(super(RelayDescriptor, self)._parser_for_line(), **{
+ 'onion-key': _parse_onion_key_line,
+ 'ntor-onion-key': _parse_ntor_onion_key_line,
+ 'signing-key': _parse_signing_key_line,
+ 'router-signature': _parse_router_signature_line,
+ })
+
def __hash__(self):
return hash(str(self).strip())
@@ -947,30 +963,11 @@ class BridgeDescriptor(ServerDescriptor):
"""
def __init__(self, raw_contents, validate = True, annotations = None):
- self._digest = None
-
super(BridgeDescriptor, self).__init__(raw_contents, validate, annotations)
def digest(self):
return self._digest
- def _parse(self, entries, validate):
- entries = dict(entries)
-
- # handles fields only in bridge descriptors
- for keyword, values in list(entries.items()):
- value, block_type, block_contents = values[0]
- line = '%s %s' % (keyword, value)
-
- if keyword == 'router-digest':
- if validate and not stem.util.tor_tools.is_hex_digits(value, 40):
- raise ValueError('Router digest line had an invalid sha1 digest: %s' % line)
-
- self._digest = stem.util.str_tools._to_unicode(value)
- del entries['router-digest']
-
- ServerDescriptor._parse(self, entries, validate)
-
def is_scrubbed(self):
"""
Checks if we've been properly scrubbed in accordance with the `bridge
@@ -1040,6 +1037,18 @@ class BridgeDescriptor(ServerDescriptor):
def _last_keyword(self):
return None
+ @lru_cache()
+ def _attributes(self):
+ return dict(super(BridgeDescriptor, self)._attributes(), **{
+ '_digest': (None, _parse_router_digest_line),
+ })
+
+ @lru_cache()
+ def _parser_for_line(self):
+ return dict(super(BridgeDescriptor, self)._parser_for_line(), **{
+ 'router-digest': _parse_router_digest_line,
+ })
+
def _compare(self, other, method):
if not isinstance(other, BridgeDescriptor):
return False
More information about the tor-commits
mailing list