commit 79eee85c1b4cac47d91d6a2dfa29867822f1bf31 Author: Damian Johnson <atagar@torproject.org> Date: Sat Jan 17 15:31:15 2015 -0800 Moving common parsing helpers to descriptor module Moving a couple common helpers to the common descriptor __init__.py module. --- stem/descriptor/__init__.py | 32 +++++++++++++++ stem/descriptor/extrainfo_descriptor.py | 60 +++++------------------------ stem/descriptor/server_descriptor.py | 60 ++++++++--------------------- test/unit/descriptor/server_descriptor.py | 2 +- 4 files changed, 58 insertions(+), 96 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 0baacdb..bc1b462 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -311,6 +311,38 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto raise TypeError("Unrecognized metrics descriptor format. type: '%s', version: '%i.%i'" % (descriptor_type, major_version, minor_version)) +def _value(line, entries): + return entries[line][0][0] + + +def _values(line, entries): + return [entry[0] for entry in entries[line]] + + +def _parse_sha1_digest_line(keyword, attribute): + def _parse(descriptor, entries): + value = _value(keyword, entries) + + if not stem.util.tor_tools.is_hex_digits(value, 40): + raise ValueError('%s line had an invalid sha1 digest: %s %s' % (keyword, keyword, value)) + + setattr(descriptor, attribute, value) + + return _parse + + +def _parse_key_block(keyword, attribute, expected_block_type): + def _parse(descriptor, entries): + value, block_type, block_contents = entries[keyword][0] + + if not block_contents or block_type != expected_block_type: + raise ValueError("'%s' should be followed by a %s block" % (keyword, expected_block_type)) + + setattr(descriptor, attribute, block_contents) + + return _parse + + class Descriptor(object): """ Common parent for all types of descriptors. diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 97623e7..9e86798 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -81,6 +81,10 @@ from stem.descriptor import ( Descriptor, _read_until_keywords, _get_descriptor_components, + _value, + _values, + _parse_sha1_digest_line, + _parse_key_block, ) try: @@ -225,14 +229,6 @@ def _parse_timestamp_and_interval(keyword, content): raise ValueError("%s line's timestamp wasn't parsable: %s" % (keyword, line)) -def _value(line, entries): - return entries[line][0][0] - - -def _values(line, entries): - return [entry[0] for entry in entries[line]] - - def _parse_extra_info_line(descriptor, entries): # "extra-info" Nickname Fingerprint @@ -250,28 +246,6 @@ def _parse_extra_info_line(descriptor, entries): descriptor.fingerprint = extra_info_comp[1] -def _parse_geoip_db_digest_line(descriptor, entries): - # "geoip-db-digest" Digest - - value = _value('geoip-db-digest', entries) - - if not stem.util.tor_tools.is_hex_digits(value, 40): - raise ValueError('Geoip digest line had an invalid sha1 digest: geoip-db-digest %s' % value) - - descriptor.geoip_db_digest = value - - -def _parse_geoip6_db_digest_line(descriptor, entries): - # "geoip6-db-digest" Digest - - value = _value('geoip6-db-digest', entries) - - if not stem.util.tor_tools.is_hex_digits(value, 40): - raise ValueError('Geoip v6 digest line had an invalid sha1 digest: geoip6-db-digest %s' % value) - - descriptor.geoip6_db_digest = value - - def _parse_transport_line(descriptor, entries): # "transport" transportname address:port [arglist] # Everything after the transportname is scrubbed in published bridge @@ -534,24 +508,8 @@ def _parse_bridge_ip_transports_line(descriptor, entries): descriptor.ip_transports = ip_transports -def _parse_router_signature_line(descriptor, entries): - value, block_type, block_contents = entries['router-signature'][0] - - if not block_contents or block_type != 'SIGNATURE': - raise ValueError("'router-signature' should be followed by a SIGNATURE block rather than a '%s'" % block_type) - - descriptor.signature = block_contents - - -def _parse_router_digest(descriptor, entries): - value = _value('router-digest', entries) - - if not stem.util.tor_tools.is_hex_digits(value, 40): - raise ValueError('Router digest line had an invalid sha1 digest: router-digest %s' % value) - - descriptor._digest = value - - +_parse_geoip_db_digest_line = _parse_sha1_digest_line('geoip-db-digest', 'geoip_db_digest') +_parse_geoip6_db_digest_line = _parse_sha1_digest_line('geoip6-db-digest', 'geoip6_db_digest') _parse_dirreq_v2_resp_line = functools.partial(_parse_dirreq_line, 'dirreq-v2-resp', 'dir_v2_responses', 'dir_v2_responses_unknown') _parse_dirreq_v3_resp_line = functools.partial(_parse_dirreq_line, 'dirreq-v3-resp', 'dir_v3_responses', 'dir_v3_responses_unknown') _parse_dirreq_v2_direct_dl_line = functools.partial(_parse_dirreq_line, 'dirreq-v2-direct-dl', 'dir_v2_direct_dl', 'dir_v2_direct_dl_unknown') @@ -584,6 +542,8 @@ _parse_dirreq_v3_reqs_line = functools.partial(_parse_geoip_to_count_line, 'dirr _parse_geoip_client_origins_line = functools.partial(_parse_geoip_to_count_line, 'geoip-client-origins', 'geoip_client_origins') _parse_entry_ips_line = functools.partial(_parse_geoip_to_count_line, 'entry-ips', 'entry_ips') _parse_bridge_ips_line = functools.partial(_parse_geoip_to_count_line, 'bridge-ips', 'bridge_ips') +_parse_router_digest_line = _parse_sha1_digest_line('router-digest', '_digest') +_parse_router_signature_line = _parse_key_block('router-signature', 'signature', 'SIGNATURE') class ExtraInfoDescriptor(Descriptor): @@ -908,11 +868,11 @@ class BridgeExtraInfoDescriptor(ExtraInfoDescriptor): """ ATTRIBUTES = dict(ExtraInfoDescriptor.ATTRIBUTES, **{ - '_digest': (None, _parse_router_digest), + '_digest': (None, _parse_router_digest_line), }) PARSER_FOR_LINE = dict(ExtraInfoDescriptor.PARSER_FOR_LINE, **{ - 'router-digest': _parse_router_digest, + 'router-digest': _parse_router_digest_line, }) def digest(self): diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 5a2ca1e..b6af898 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -33,6 +33,7 @@ etc). This information is provided from a few sources... import base64 import codecs +import functools import hashlib import re @@ -53,6 +54,10 @@ from stem.descriptor import ( _get_bytes_field, _get_descriptor_components, _read_until_keywords, + _value, + _values, + _parse_sha1_digest_line, + _parse_key_block, ) try: @@ -166,14 +171,6 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True, **kwargs): break # done parsing descriptors -def _value(line, entries): - return entries[line][0][0] - - -def _values(line, entries): - return [entry[0] for entry in entries[line]] - - def _parse_router_line(descriptor, entries): # "router" nickname address ORPort SocksPort DirPort @@ -364,23 +361,7 @@ def _parse_or_address_line(descriptor, entries): descriptor.or_addresses = or_addresses -def _parse_read_history_line(descriptor, entries): - timestamp, interval, history_values = _parse_history_line(descriptor, entries, 'read-history') - - descriptor.read_history_end = timestamp - descriptor.read_history_interval = interval - descriptor.read_history_values = history_values - - -def _parse_write_history_line(descriptor, entries): - timestamp, interval, history_values = _parse_history_line(descriptor, entries, 'write-history') - - descriptor.write_history_end = timestamp - descriptor.write_history_interval = interval - descriptor.write_history_values = history_values - - -def _parse_history_line(descriptor, entries, keyword): +def _parse_history_line(keyword, history_end_attribute, history_interval_attribute, history_values_attribute, descriptor, entries): value = _value(keyword, entries) timestamp, interval, remainder = stem.descriptor.extrainfo_descriptor._parse_timestamp_and_interval(keyword, value) @@ -392,23 +373,9 @@ def _parse_history_line(descriptor, entries, keyword): except ValueError: raise ValueError('%s line has non-numeric values: %s %s' % (keyword, keyword, value)) - return timestamp, interval, history_values - - -def _parse_router_digest_line(descriptor, entries): - descriptor._digest = _value('router-digest', entries) - - if not stem.util.tor_tools.is_hex_digits(descriptor._digest, 40): - raise ValueError('Router digest line had an invalid sha1 digest: router-digest %s' % descriptor._digest) - - -def _key_block(entries, keyword, expected_block_type): - value, block_type, block_contents = entries[keyword][0] - - if not block_contents or block_type != expected_block_type: - raise ValueError("'%s' should be followed by a %s block" % (keyword, expected_block_type)) - - return block_contents + setattr(descriptor, history_end_attribute, timestamp) + setattr(descriptor, history_interval_attribute, interval) + setattr(descriptor, history_values_attribute, history_values) def _parse_exit_policy(descriptor, entries): @@ -421,15 +388,18 @@ def _parse_exit_policy(descriptor, entries): del descriptor._unparsed_exit_policy +_parse_read_history_line = functools.partial(_parse_history_line, 'read-history', 'read_history_end', 'read_history_interval', 'read_history_values') +_parse_write_history_line = functools.partial(_parse_history_line, 'write-history', 'write_history_end', 'write_history_interval', 'write_history_values') _parse_ipv6_policy_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('ipv6-policy', entries))) _parse_allow_single_hop_exits_line = lambda descriptor, entries: setattr(descriptor, 'allow_single_hop_exits', True) _parse_caches_extra_info_line = lambda descriptor, entries: setattr(descriptor, 'extra_info_cache', True) _parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', set(_value('family', entries).split(' '))) _parse_eventdns_line = lambda descriptor, entries: setattr(descriptor, 'eventdns', _value('eventdns', entries) == '1') -_parse_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'onion_key', _key_block(entries, 'onion-key', 'RSA PUBLIC KEY')) -_parse_signing_key_line = lambda descriptor, entries: setattr(descriptor, 'signing_key', _key_block(entries, 'signing-key', 'RSA PUBLIC KEY')) -_parse_router_signature_line = lambda descriptor, entries: setattr(descriptor, 'signature', _key_block(entries, 'router-signature', 'SIGNATURE')) +_parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY') +_parse_signing_key_line = _parse_key_block('signing-key', 'signing_key', 'RSA PUBLIC KEY') +_parse_router_signature_line = _parse_key_block('router-signature', 'signature', 'SIGNATURE') _parse_ntor_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'ntor_onion_key', _value('ntor-onion-key', entries)) +_parse_router_digest_line = _parse_sha1_digest_line('router-digest', '_digest') class ServerDescriptor(Descriptor): diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 61654a7..f261798 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -672,7 +672,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertRaises(ValueError, BridgeDescriptor, desc_text) desc = BridgeDescriptor(desc_text, validate = False) - self.assertEqual(value, desc.digest()) + self.assertEqual(None, desc.digest()) def test_or_address_v4(self): """