[tor-commits] [stem/master] Moving common parsing helpers to descriptor module
atagar at torproject.org
atagar at torproject.org
Sun Jan 25 22:37:34 UTC 2015
commit 79eee85c1b4cac47d91d6a2dfa29867822f1bf31
Author: Damian Johnson <atagar at torproject.org>
Date: Sat Jan 17 15:31:15 2015 -0800
Moving common parsing helpers to descriptor module
Moving a couple common helpers to the common descriptor __init__.py module.
---
stem/descriptor/__init__.py | 32 +++++++++++++++
stem/descriptor/extrainfo_descriptor.py | 60 +++++------------------------
stem/descriptor/server_descriptor.py | 60 ++++++++---------------------
test/unit/descriptor/server_descriptor.py | 2 +-
4 files changed, 58 insertions(+), 96 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index 0baacdb..bc1b462 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -311,6 +311,38 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto
raise TypeError("Unrecognized metrics descriptor format. type: '%s', version: '%i.%i'" % (descriptor_type, major_version, minor_version))
+def _value(line, entries):
+ return entries[line][0][0]
+
+
+def _values(line, entries):
+ return [entry[0] for entry in entries[line]]
+
+
+def _parse_sha1_digest_line(keyword, attribute):
+ def _parse(descriptor, entries):
+ value = _value(keyword, entries)
+
+ if not stem.util.tor_tools.is_hex_digits(value, 40):
+ raise ValueError('%s line had an invalid sha1 digest: %s %s' % (keyword, keyword, value))
+
+ setattr(descriptor, attribute, value)
+
+ return _parse
+
+
+def _parse_key_block(keyword, attribute, expected_block_type):
+ def _parse(descriptor, entries):
+ value, block_type, block_contents = entries[keyword][0]
+
+ if not block_contents or block_type != expected_block_type:
+ raise ValueError("'%s' should be followed by a %s block" % (keyword, expected_block_type))
+
+ setattr(descriptor, attribute, block_contents)
+
+ return _parse
+
+
class Descriptor(object):
"""
Common parent for all types of descriptors.
diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py
index 97623e7..9e86798 100644
--- a/stem/descriptor/extrainfo_descriptor.py
+++ b/stem/descriptor/extrainfo_descriptor.py
@@ -81,6 +81,10 @@ from stem.descriptor import (
Descriptor,
_read_until_keywords,
_get_descriptor_components,
+ _value,
+ _values,
+ _parse_sha1_digest_line,
+ _parse_key_block,
)
try:
@@ -225,14 +229,6 @@ def _parse_timestamp_and_interval(keyword, content):
raise ValueError("%s line's timestamp wasn't parsable: %s" % (keyword, line))
-def _value(line, entries):
- return entries[line][0][0]
-
-
-def _values(line, entries):
- return [entry[0] for entry in entries[line]]
-
-
def _parse_extra_info_line(descriptor, entries):
# "extra-info" Nickname Fingerprint
@@ -250,28 +246,6 @@ def _parse_extra_info_line(descriptor, entries):
descriptor.fingerprint = extra_info_comp[1]
-def _parse_geoip_db_digest_line(descriptor, entries):
- # "geoip-db-digest" Digest
-
- value = _value('geoip-db-digest', entries)
-
- if not stem.util.tor_tools.is_hex_digits(value, 40):
- raise ValueError('Geoip digest line had an invalid sha1 digest: geoip-db-digest %s' % value)
-
- descriptor.geoip_db_digest = value
-
-
-def _parse_geoip6_db_digest_line(descriptor, entries):
- # "geoip6-db-digest" Digest
-
- value = _value('geoip6-db-digest', entries)
-
- if not stem.util.tor_tools.is_hex_digits(value, 40):
- raise ValueError('Geoip v6 digest line had an invalid sha1 digest: geoip6-db-digest %s' % value)
-
- descriptor.geoip6_db_digest = value
-
-
def _parse_transport_line(descriptor, entries):
# "transport" transportname address:port [arglist]
# Everything after the transportname is scrubbed in published bridge
@@ -534,24 +508,8 @@ def _parse_bridge_ip_transports_line(descriptor, entries):
descriptor.ip_transports = ip_transports
-def _parse_router_signature_line(descriptor, entries):
- value, block_type, block_contents = entries['router-signature'][0]
-
- if not block_contents or block_type != 'SIGNATURE':
- raise ValueError("'router-signature' should be followed by a SIGNATURE block rather than a '%s'" % block_type)
-
- descriptor.signature = block_contents
-
-
-def _parse_router_digest(descriptor, entries):
- value = _value('router-digest', entries)
-
- if not stem.util.tor_tools.is_hex_digits(value, 40):
- raise ValueError('Router digest line had an invalid sha1 digest: router-digest %s' % value)
-
- descriptor._digest = value
-
-
+_parse_geoip_db_digest_line = _parse_sha1_digest_line('geoip-db-digest', 'geoip_db_digest')
+_parse_geoip6_db_digest_line = _parse_sha1_digest_line('geoip6-db-digest', 'geoip6_db_digest')
_parse_dirreq_v2_resp_line = functools.partial(_parse_dirreq_line, 'dirreq-v2-resp', 'dir_v2_responses', 'dir_v2_responses_unknown')
_parse_dirreq_v3_resp_line = functools.partial(_parse_dirreq_line, 'dirreq-v3-resp', 'dir_v3_responses', 'dir_v3_responses_unknown')
_parse_dirreq_v2_direct_dl_line = functools.partial(_parse_dirreq_line, 'dirreq-v2-direct-dl', 'dir_v2_direct_dl', 'dir_v2_direct_dl_unknown')
@@ -584,6 +542,8 @@ _parse_dirreq_v3_reqs_line = functools.partial(_parse_geoip_to_count_line, 'dirr
_parse_geoip_client_origins_line = functools.partial(_parse_geoip_to_count_line, 'geoip-client-origins', 'geoip_client_origins')
_parse_entry_ips_line = functools.partial(_parse_geoip_to_count_line, 'entry-ips', 'entry_ips')
_parse_bridge_ips_line = functools.partial(_parse_geoip_to_count_line, 'bridge-ips', 'bridge_ips')
+_parse_router_digest_line = _parse_sha1_digest_line('router-digest', '_digest')
+_parse_router_signature_line = _parse_key_block('router-signature', 'signature', 'SIGNATURE')
class ExtraInfoDescriptor(Descriptor):
@@ -908,11 +868,11 @@ class BridgeExtraInfoDescriptor(ExtraInfoDescriptor):
"""
ATTRIBUTES = dict(ExtraInfoDescriptor.ATTRIBUTES, **{
- '_digest': (None, _parse_router_digest),
+ '_digest': (None, _parse_router_digest_line),
})
PARSER_FOR_LINE = dict(ExtraInfoDescriptor.PARSER_FOR_LINE, **{
- 'router-digest': _parse_router_digest,
+ 'router-digest': _parse_router_digest_line,
})
def digest(self):
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index 5a2ca1e..b6af898 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -33,6 +33,7 @@ etc). This information is provided from a few sources...
import base64
import codecs
+import functools
import hashlib
import re
@@ -53,6 +54,10 @@ from stem.descriptor import (
_get_bytes_field,
_get_descriptor_components,
_read_until_keywords,
+ _value,
+ _values,
+ _parse_sha1_digest_line,
+ _parse_key_block,
)
try:
@@ -166,14 +171,6 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True, **kwargs):
break # done parsing descriptors
-def _value(line, entries):
- return entries[line][0][0]
-
-
-def _values(line, entries):
- return [entry[0] for entry in entries[line]]
-
-
def _parse_router_line(descriptor, entries):
# "router" nickname address ORPort SocksPort DirPort
@@ -364,23 +361,7 @@ def _parse_or_address_line(descriptor, entries):
descriptor.or_addresses = or_addresses
-def _parse_read_history_line(descriptor, entries):
- timestamp, interval, history_values = _parse_history_line(descriptor, entries, 'read-history')
-
- descriptor.read_history_end = timestamp
- descriptor.read_history_interval = interval
- descriptor.read_history_values = history_values
-
-
-def _parse_write_history_line(descriptor, entries):
- timestamp, interval, history_values = _parse_history_line(descriptor, entries, 'write-history')
-
- descriptor.write_history_end = timestamp
- descriptor.write_history_interval = interval
- descriptor.write_history_values = history_values
-
-
-def _parse_history_line(descriptor, entries, keyword):
+def _parse_history_line(keyword, history_end_attribute, history_interval_attribute, history_values_attribute, descriptor, entries):
value = _value(keyword, entries)
timestamp, interval, remainder = stem.descriptor.extrainfo_descriptor._parse_timestamp_and_interval(keyword, value)
@@ -392,23 +373,9 @@ def _parse_history_line(descriptor, entries, keyword):
except ValueError:
raise ValueError('%s line has non-numeric values: %s %s' % (keyword, keyword, value))
- return timestamp, interval, history_values
-
-
-def _parse_router_digest_line(descriptor, entries):
- descriptor._digest = _value('router-digest', entries)
-
- if not stem.util.tor_tools.is_hex_digits(descriptor._digest, 40):
- raise ValueError('Router digest line had an invalid sha1 digest: router-digest %s' % descriptor._digest)
-
-
-def _key_block(entries, keyword, expected_block_type):
- value, block_type, block_contents = entries[keyword][0]
-
- if not block_contents or block_type != expected_block_type:
- raise ValueError("'%s' should be followed by a %s block" % (keyword, expected_block_type))
-
- return block_contents
+ setattr(descriptor, history_end_attribute, timestamp)
+ setattr(descriptor, history_interval_attribute, interval)
+ setattr(descriptor, history_values_attribute, history_values)
def _parse_exit_policy(descriptor, entries):
@@ -421,15 +388,18 @@ def _parse_exit_policy(descriptor, entries):
del descriptor._unparsed_exit_policy
+_parse_read_history_line = functools.partial(_parse_history_line, 'read-history', 'read_history_end', 'read_history_interval', 'read_history_values')
+_parse_write_history_line = functools.partial(_parse_history_line, 'write-history', 'write_history_end', 'write_history_interval', 'write_history_values')
_parse_ipv6_policy_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('ipv6-policy', entries)))
_parse_allow_single_hop_exits_line = lambda descriptor, entries: setattr(descriptor, 'allow_single_hop_exits', True)
_parse_caches_extra_info_line = lambda descriptor, entries: setattr(descriptor, 'extra_info_cache', True)
_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', set(_value('family', entries).split(' ')))
_parse_eventdns_line = lambda descriptor, entries: setattr(descriptor, 'eventdns', _value('eventdns', entries) == '1')
-_parse_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'onion_key', _key_block(entries, 'onion-key', 'RSA PUBLIC KEY'))
-_parse_signing_key_line = lambda descriptor, entries: setattr(descriptor, 'signing_key', _key_block(entries, 'signing-key', 'RSA PUBLIC KEY'))
-_parse_router_signature_line = lambda descriptor, entries: setattr(descriptor, 'signature', _key_block(entries, 'router-signature', 'SIGNATURE'))
+_parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY')
+_parse_signing_key_line = _parse_key_block('signing-key', 'signing_key', 'RSA PUBLIC KEY')
+_parse_router_signature_line = _parse_key_block('router-signature', 'signature', 'SIGNATURE')
_parse_ntor_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'ntor_onion_key', _value('ntor-onion-key', entries))
+_parse_router_digest_line = _parse_sha1_digest_line('router-digest', '_digest')
class ServerDescriptor(Descriptor):
diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py
index 61654a7..f261798 100644
--- a/test/unit/descriptor/server_descriptor.py
+++ b/test/unit/descriptor/server_descriptor.py
@@ -672,7 +672,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4=
self.assertRaises(ValueError, BridgeDescriptor, desc_text)
desc = BridgeDescriptor(desc_text, validate = False)
- self.assertEqual(value, desc.digest())
+ self.assertEqual(None, desc.digest())
def test_or_address_v4(self):
"""
More information about the tor-commits
mailing list