commit a03731a26e218ee6e02cf00b99ef3a38e105de8c Author: Damian Johnson atagar@torproject.org Date: Wed Dec 30 10:28:32 2015 -0800

Don't assign lambdas to variables

Stylistic complaint E731 of the new version of pep8...

line 363 - E731 do not assign a lambda expression, use a def | _parse_client_versions_line = lambda descriptor, entries: setattr(descriptor, 'client_versions', _value('client-versions', entries).split(','))

Reason for this is described at...

http://stackoverflow.com/a/25010243/1067192

In the process of doing this realized we were mis-parsing a couple infrequently used server descriptor fields. --- docs/change_log.rst | 1 + stem/descriptor/__init__.py | 106 +++++++++++++-------------- stem/descriptor/microdescriptor.py | 10 ++- stem/descriptor/networkstatus.py | 10 +-- stem/descriptor/remote.py | 11 +-- stem/descriptor/server_descriptor.py | 11 +-- stem/version.py | 13 ++-- test/runner.py | 5 +- test/unit/descriptor/router_status_entry.py | 5 +- test/unit/descriptor/server_descriptor.py | 2 +- 10 files changed, 86 insertions(+), 88 deletions(-)

diff --git a/docs/change_log.rst b/docs/change_log.rst index fb0a71c..80938da 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -64,6 +64,7 @@ The following are only available within Stem's `git repository * Unable to read descriptors from data directories on Windows due to their CRLF newlines (:trac:`17051`) * TypeError under python3 when using 'use_mirrors = True' (:trac:`17083`) * Deprecated hidden service descriptor's *introduction_points_auth* field, which was never implemented in tor (:trac:`15190`, :spec:`9c218f9`) + * Fixed parsing of server descriptor's *allow-single-hop-exits* and *caches-extra-info* lines

* **Utilities**

diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 678325a..af96bde 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -188,53 +188,48 @@ def parse_file(descriptor_file, descriptor_type = None, validate = False, docume

descriptor_path = getattr(descriptor_file, 'name', None) filename = '<undefined>' if descriptor_path is None else os.path.basename(descriptor_file.name) - file_parser = None

- if descriptor_type is not None: - descriptor_type_match = re.match('^(\S+) (\d+).(\d+)$', descriptor_type) + def parse(descriptor_file): + if normalize_newlines: + descriptor_file = NewlineNormalizer(descriptor_file)

- if descriptor_type_match: - desc_type, major_version, minor_version = descriptor_type_match.groups() - file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler, **kwargs) - else: - raise ValueError("The descriptor_type must be of the form '<type> <major_version>.<minor_version>'") - elif metrics_header_match: - # Metrics descriptor handling - - desc_type, major_version, minor_version = metrics_header_match.groups() - file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler, **kwargs) - else: - # Cached descriptor handling. These contain multiple descriptors per file. - - if normalize_newlines is None and stem.util.system.is_windows(): - normalize_newlines = True - - if filename == 'cached-descriptors' or filename == 'cached-descriptors.new': - file_parser = lambda f: stem.descriptor.server_descriptor._parse_file(f, validate = validate, **kwargs) - elif filename == 'cached-extrainfo' or filename == 'cached-extrainfo.new': - file_parser = lambda f: stem.descriptor.extrainfo_descriptor._parse_file(f, validate = validate, **kwargs) - elif filename == 'cached-microdescs' or filename == 'cached-microdescs.new': - file_parser = lambda f: stem.descriptor.microdescriptor._parse_file(f, validate = validate, **kwargs) - elif filename == 'cached-consensus': - file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, validate = validate, document_handler = document_handler, **kwargs) - elif filename == 'cached-microdesc-consensus': - file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs) - - if normalize_newlines: - descriptor_file = NewlineNormalizer(descriptor_file) - - if file_parser: - for desc in file_parser(descriptor_file): - if descriptor_path is not None: - desc._set_path(os.path.abspath(descriptor_path)) + if descriptor_type is not None: + descriptor_type_match = re.match('^(\S+) (\d+).(\d+)$', descriptor_type)

- yield desc + if descriptor_type_match: + desc_type, major_version, minor_version = descriptor_type_match.groups() + return _parse_metrics_file(desc_type, int(major_version), int(minor_version), descriptor_file, validate, document_handler, **kwargs) + else: + raise ValueError("The descriptor_type must be of the form '<type> <major_version>.<minor_version>'") + elif metrics_header_match: + # Metrics descriptor handling

- return + desc_type, major_version, minor_version = metrics_header_match.groups() + return _parse_metrics_file(desc_type, int(major_version), int(minor_version), descriptor_file, validate, document_handler, **kwargs) + else: + # Cached descriptor handling. These contain multiple descriptors per file. + + if normalize_newlines is None and stem.util.system.is_windows(): + descriptor_file = NewlineNormalizer(descriptor_file) + + if filename == 'cached-descriptors' or filename == 'cached-descriptors.new': + return stem.descriptor.server_descriptor._parse_file(descriptor_file, validate = validate, **kwargs) + elif filename == 'cached-extrainfo' or filename == 'cached-extrainfo.new': + return stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, validate = validate, **kwargs) + elif filename == 'cached-microdescs' or filename == 'cached-microdescs.new': + return stem.descriptor.microdescriptor._parse_file(descriptor_file, validate = validate, **kwargs) + elif filename == 'cached-consensus': + return stem.descriptor.networkstatus._parse_file(descriptor_file, validate = validate, document_handler = document_handler, **kwargs) + elif filename == 'cached-microdesc-consensus': + return stem.descriptor.networkstatus._parse_file(descriptor_file, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs) + else: + raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line))

- # Not recognized as a descriptor file. + for desc in parse(descriptor_file): + if descriptor_path is not None: + desc._set_path(os.path.abspath(descriptor_path))

- raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line)) + yield desc

def _parse_file_for_path(descriptor_file, *args, **kwargs): @@ -336,13 +331,18 @@ def _values(line, entries): return [entry[0] for entry in entries[line]]

-def _parse_simple_line(keyword, attribute): +def _parse_simple_line(keyword, attribute, func = None): def _parse(descriptor, entries): - setattr(descriptor, attribute, _value(keyword, entries)) + value = _value(keyword, entries) + setattr(descriptor, attribute, func(value) if func else value)

return _parse

+def _parse_if_present(keyword, attribute): + return lambda descriptor, entries: setattr(descriptor, attribute, keyword in entries) + + def _parse_bytes_line(keyword, attribute): def _parse(descriptor, entries): line_match = re.search(stem.util.str_tools._to_bytes('^(opt )?%s(?:[%s]+(.*))?$' % (keyword, WHITESPACE)), descriptor.get_bytes(), re.MULTILINE) @@ -675,13 +675,7 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi **True** """

- if skip: - content = None - content_append = lambda x: None - else: - content = [] - content_append = content.append - + content = None if skip else [] ending_keyword = None

if isinstance(keywords, (bytes, str_type)): @@ -690,8 +684,8 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi if ignore_first: first_line = descriptor_file.readline()

- if first_line: - content_append(first_line) + if first_line and content is not None: + content.append(first_line)

keyword_match = re.compile(SPECIFIC_KEYWORD_LINE % '|'.join(keywords))

@@ -713,12 +707,12 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi

if not inclusive: descriptor_file.seek(last_position) - else: - content_append(line) + elif content is not None: + content.append(line)

break - else: - content_append(line) + elif content is not None: + content.append(line)

if include_ending_keyword: return (content, ending_keyword) diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index b190635..3f74f70 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -72,7 +72,6 @@ from stem.descriptor import ( Descriptor, _get_descriptor_components, _read_until_keywords, - _value, _values, _parse_simple_line, _parse_key_block, @@ -180,11 +179,14 @@ def _parse_id_line(descriptor, entries): descriptor.identifiers = identities

-_parse_digest = lambda descriptor, entries: setattr(descriptor, 'digest', hashlib.sha256(descriptor.get_bytes()).hexdigest().upper()) +def _parse_digest(descriptor, entries): + setattr(descriptor, 'digest', hashlib.sha256(descriptor.get_bytes()).hexdigest().upper()) + + _parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY') _parse_ntor_onion_key_line = _parse_simple_line('ntor-onion-key', 'ntor_onion_key') -_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', _value('family', entries).split(' ')) -_parse_p6_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('p6', entries))) +_parse_family_line = _parse_simple_line('family', 'family', func = lambda v: v.split(' ')) +_parse_p6_line = _parse_simple_line('p6', 'exit_policy_v6', func = lambda v: stem.exit_policy.MicroExitPolicy(v))

class Microdescriptor(Descriptor): diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 103f72c..b471c1f 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -360,10 +360,10 @@ _parse_network_status_version_line = _parse_version_line('network-status-version _parse_fingerprint_line = _parse_forty_character_hex('fingerprint', 'fingerprint') _parse_contact_line = _parse_simple_line('contact', 'contact') _parse_dir_signing_key_line = _parse_key_block('dir-signing-key', 'signing_key', 'RSA PUBLIC KEY') -_parse_client_versions_line = lambda descriptor, entries: setattr(descriptor, 'client_versions', _value('client-versions', entries).split(',')) -_parse_server_versions_line = lambda descriptor, entries: setattr(descriptor, 'server_versions', _value('server-versions', entries).split(',')) +_parse_client_versions_line = _parse_simple_line('client-versions', 'client_versions', func = lambda v: v.split(',')) +_parse_server_versions_line = _parse_simple_line('server-versions', 'server_versions', func = lambda v: v.split(',')) _parse_published_line = _parse_timestamp_line('published', 'published') -_parse_dir_options_line = lambda descriptor, entries: setattr(descriptor, 'options', _value('dir-options', entries).split()) +_parse_dir_options_line = _parse_simple_line('dir-options', 'options', func = lambda v: v.split()) _parse_directory_signature_line = _parse_key_block('directory-signature', 'signature', 'SIGNATURE', value_attribute = 'signing_authority')

@@ -685,8 +685,8 @@ _parse_header_fresh_until_line = _parse_timestamp_line('fresh-until', 'fresh_unt _parse_header_valid_until_line = _parse_timestamp_line('valid-until', 'valid_until') _parse_header_client_versions_line = _parse_versions_line('client-versions', 'client_versions') _parse_header_server_versions_line = _parse_versions_line('server-versions', 'server_versions') -_parse_header_known_flags_line = lambda descriptor, entries: setattr(descriptor, 'known_flags', [entry for entry in _value('known-flags', entries).split(' ') if entry]) -_parse_footer_bandwidth_weights_line = lambda descriptor, entries: setattr(descriptor, 'bandwidth_weights', _parse_int_mappings('bandwidth-weights', _value('bandwidth-weights', entries), True)) +_parse_header_known_flags_line = _parse_simple_line('known-flags', 'known_flags', func = lambda v: [entry for entry in v.split(' ') if entry]) +_parse_footer_bandwidth_weights_line = _parse_simple_line('bandwidth-weights', 'bandwidth_weights', func = lambda v: _parse_int_mappings('bandwidth-weights', v, True))

class NetworkStatusDocumentV3(NetworkStatusDocument): diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index b2b5b8e..4115550 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -96,11 +96,6 @@ from stem.util import log MAX_FINGERPRINTS = 96 MAX_MICRODESCRIPTOR_HASHES = 92

-# We commonly only want authorities that vote in the consensus, and hence have -# a v3ident. - -HAS_V3IDENT = lambda auth: auth.v3ident is not None -

def _guess_descriptor_type(resource): # Attempts to determine the descriptor type based on the resource url. This @@ -345,7 +340,7 @@ class Query(object): """

if use_authority or not self.endpoints: - authority = random.choice(list(filter(HAS_V3IDENT, get_authorities().values()))) + authority = random.choice(list(filter(lambda auth: auth.v3ident is not None, get_authorities().values()))) address, dirport = authority.address, authority.dir_port else: address, dirport = random.choice(self.endpoints) @@ -395,7 +390,7 @@ class DescriptorDownloader(object): def __init__(self, use_mirrors = False, **default_args): self._default_args = default_args

- authorities = filter(HAS_V3IDENT, get_authorities().values()) + authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values()) self._endpoints = [(auth.address, auth.dir_port) for auth in authorities]

if use_mirrors: @@ -417,7 +412,7 @@ class DescriptorDownloader(object): :raises: **Exception** if unable to determine the directory mirrors """

- authorities = filter(HAS_V3IDENT, get_authorities().values()) + authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values()) new_endpoints = set([(auth.address, auth.dir_port) for auth in authorities])

consensus = list(self.get_consensus(document_handler = stem.descriptor.DocumentHandler.DOCUMENT).run())[0] diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index a555e12..c2a358f 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -54,6 +54,7 @@ from stem.descriptor import ( _value, _values, _parse_simple_line, + _parse_if_present, _parse_bytes_line, _parse_timestamp_line, _parse_forty_character_hex, @@ -394,11 +395,11 @@ _parse_contact_line = _parse_bytes_line('contact', 'contact') _parse_published_line = _parse_timestamp_line('published', 'published') _parse_read_history_line = functools.partial(_parse_history_line, 'read-history', 'read_history_end', 'read_history_interval', 'read_history_values') _parse_write_history_line = functools.partial(_parse_history_line, 'write-history', 'write_history_end', 'write_history_interval', 'write_history_values') -_parse_ipv6_policy_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('ipv6-policy', entries))) -_parse_allow_single_hop_exits_line = lambda descriptor, entries: setattr(descriptor, 'allow_single_hop_exits', 'allow_single_hop_exits' in entries) -_parse_caches_extra_info_line = lambda descriptor, entries: setattr(descriptor, 'extra_info_cache', 'extra_info_cache' in entries) -_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', set(_value('family', entries).split(' '))) -_parse_eventdns_line = lambda descriptor, entries: setattr(descriptor, 'eventdns', _value('eventdns', entries) == '1') +_parse_ipv6_policy_line = _parse_simple_line('ipv6-policy', 'exit_policy_v6', func = lambda v: stem.exit_policy.MicroExitPolicy(v)) +_parse_allow_single_hop_exits_line = _parse_if_present('allow-single-hop-exits', 'allow_single_hop_exits') +_parse_caches_extra_info_line = _parse_if_present('caches-extra-info', 'extra_info_cache') +_parse_family_line = _parse_simple_line('family', 'family', func = lambda v: set(v.split(' '))) +_parse_eventdns_line = _parse_simple_line('eventdns', 'eventdns', func = lambda v: v == '1') _parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY') _parse_onion_key_crosscert_line = _parse_key_block('onion-key-crosscert', 'onion_key_crosscert', 'CROSSCERT') _parse_signing_key_line = _parse_key_block('signing-key', 'signing_key', 'RSA PUBLIC KEY') diff --git a/stem/version.py b/stem/version.py index 68ee40f..349d9bb 100644 --- a/stem/version.py +++ b/stem/version.py @@ -328,12 +328,13 @@ class _VersionRequirements(object): :param bool to_inclusive: if comparison is inclusive with the ending version """

- if from_inclusive and to_inclusive: - new_rule = lambda v: from_version <= v <= to_version - elif from_inclusive: - new_rule = lambda v: from_version <= v < to_version - else: - new_rule = lambda v: from_version < v < to_version + def new_rule(v): + if from_inclusive and to_inclusive: + return from_version <= v <= to_version + elif from_inclusive: + return from_version <= v < to_version + else: + return from_version < v < to_version

self.rules.append(new_rule)

diff --git a/test/runner.py b/test/runner.py index 0370abd..80f390e 100644 --- a/test/runner.py +++ b/test/runner.py @@ -713,10 +713,11 @@ class Runner(object): try: # wait to fully complete if we're running tests with network activity, # otherwise finish after local bootstraping + complete_percent = 100 if Target.ONLINE in self.attribute_targets else 5

- # prints output from tor's stdout while it starts up - print_init_line = lambda line: println(' %s' % line, SUBSTATUS) + def print_init_line(line): + println(' %s' % line, SUBSTATUS)

torrc_dst = os.path.join(self._test_dir, 'torrc') self._tor_process = stem.process.launch_tor(tor_cmd, None, torrc_dst, complete_percent, print_init_line, take_ownership = True) diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py index 73cda28..25ba99d 100644 --- a/test/unit/descriptor/router_status_entry.py +++ b/test/unit/descriptor/router_status_entry.py @@ -49,7 +49,10 @@ m 21 sha256=AVp41YVxKEJCaoEf0+77Cdvyw5YgpyDXdob0+LSv/pE

def vote_document(): - mock_document = lambda x: x # just need anything with a __dict__ + class Stub(object): + pass + + mock_document = Stub() # just need anything with a __dict__ setattr(mock_document, 'is_vote', True) setattr(mock_document, 'is_consensus', False) return mock_document diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index d295884..cb587d0 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -376,7 +376,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEqual(['1'], desc.circuit_protocols) self.assertEqual(False, desc.hibernating) self.assertEqual(False, desc.allow_single_hop_exits) - self.assertEqual(False, desc.extra_info_cache) + self.assertEqual(True, desc.extra_info_cache) self.assertEqual('BB1F13AA431421BEA29B840A2E33BB1C31C2990B', desc.extra_info_digest) self.assertEqual(None, desc.hidden_service_dir) self.assertEqual(set(), desc.family)