[tor-commits] [stem/master] Don't assign lambdas to variables
atagar at torproject.org
atagar at torproject.org
Thu Dec 31 17:05:05 UTC 2015
commit a03731a26e218ee6e02cf00b99ef3a38e105de8c
Author: Damian Johnson <atagar at torproject.org>
Date: Wed Dec 30 10:28:32 2015 -0800
Don't assign lambdas to variables
Stylistic complaint E731 of the new version of pep8...
line 363 - E731 do not assign a lambda expression, use a def |
_parse_client_versions_line = lambda descriptor, entries: setattr(descriptor,
'client_versions', _value('client-versions', entries).split(','))
Reason for this is described at...
http://stackoverflow.com/a/25010243/1067192
In the process of doing this realized we were mis-parsing a couple infrequently
used server descriptor fields.
---
docs/change_log.rst | 1 +
stem/descriptor/__init__.py | 106 +++++++++++++--------------
stem/descriptor/microdescriptor.py | 10 ++-
stem/descriptor/networkstatus.py | 10 +--
stem/descriptor/remote.py | 11 +--
stem/descriptor/server_descriptor.py | 11 +--
stem/version.py | 13 ++--
test/runner.py | 5 +-
test/unit/descriptor/router_status_entry.py | 5 +-
test/unit/descriptor/server_descriptor.py | 2 +-
10 files changed, 86 insertions(+), 88 deletions(-)
diff --git a/docs/change_log.rst b/docs/change_log.rst
index fb0a71c..80938da 100644
--- a/docs/change_log.rst
+++ b/docs/change_log.rst
@@ -64,6 +64,7 @@ The following are only available within Stem's `git repository
* Unable to read descriptors from data directories on Windows due to their CRLF newlines (:trac:`17051`)
* TypeError under python3 when using 'use_mirrors = True' (:trac:`17083`)
* Deprecated hidden service descriptor's *introduction_points_auth* field, which was never implemented in tor (:trac:`15190`, :spec:`9c218f9`)
+ * Fixed parsing of server descriptor's *allow-single-hop-exits* and *caches-extra-info* lines
* **Utilities**
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index 678325a..af96bde 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -188,53 +188,48 @@ def parse_file(descriptor_file, descriptor_type = None, validate = False, docume
descriptor_path = getattr(descriptor_file, 'name', None)
filename = '<undefined>' if descriptor_path is None else os.path.basename(descriptor_file.name)
- file_parser = None
- if descriptor_type is not None:
- descriptor_type_match = re.match('^(\S+) (\d+).(\d+)$', descriptor_type)
+ def parse(descriptor_file):
+ if normalize_newlines:
+ descriptor_file = NewlineNormalizer(descriptor_file)
- if descriptor_type_match:
- desc_type, major_version, minor_version = descriptor_type_match.groups()
- file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler, **kwargs)
- else:
- raise ValueError("The descriptor_type must be of the form '<type> <major_version>.<minor_version>'")
- elif metrics_header_match:
- # Metrics descriptor handling
-
- desc_type, major_version, minor_version = metrics_header_match.groups()
- file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler, **kwargs)
- else:
- # Cached descriptor handling. These contain multiple descriptors per file.
-
- if normalize_newlines is None and stem.util.system.is_windows():
- normalize_newlines = True
-
- if filename == 'cached-descriptors' or filename == 'cached-descriptors.new':
- file_parser = lambda f: stem.descriptor.server_descriptor._parse_file(f, validate = validate, **kwargs)
- elif filename == 'cached-extrainfo' or filename == 'cached-extrainfo.new':
- file_parser = lambda f: stem.descriptor.extrainfo_descriptor._parse_file(f, validate = validate, **kwargs)
- elif filename == 'cached-microdescs' or filename == 'cached-microdescs.new':
- file_parser = lambda f: stem.descriptor.microdescriptor._parse_file(f, validate = validate, **kwargs)
- elif filename == 'cached-consensus':
- file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, validate = validate, document_handler = document_handler, **kwargs)
- elif filename == 'cached-microdesc-consensus':
- file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs)
-
- if normalize_newlines:
- descriptor_file = NewlineNormalizer(descriptor_file)
-
- if file_parser:
- for desc in file_parser(descriptor_file):
- if descriptor_path is not None:
- desc._set_path(os.path.abspath(descriptor_path))
+ if descriptor_type is not None:
+ descriptor_type_match = re.match('^(\S+) (\d+).(\d+)$', descriptor_type)
- yield desc
+ if descriptor_type_match:
+ desc_type, major_version, minor_version = descriptor_type_match.groups()
+ return _parse_metrics_file(desc_type, int(major_version), int(minor_version), descriptor_file, validate, document_handler, **kwargs)
+ else:
+ raise ValueError("The descriptor_type must be of the form '<type> <major_version>.<minor_version>'")
+ elif metrics_header_match:
+ # Metrics descriptor handling
- return
+ desc_type, major_version, minor_version = metrics_header_match.groups()
+ return _parse_metrics_file(desc_type, int(major_version), int(minor_version), descriptor_file, validate, document_handler, **kwargs)
+ else:
+ # Cached descriptor handling. These contain multiple descriptors per file.
+
+ if normalize_newlines is None and stem.util.system.is_windows():
+ descriptor_file = NewlineNormalizer(descriptor_file)
+
+ if filename == 'cached-descriptors' or filename == 'cached-descriptors.new':
+ return stem.descriptor.server_descriptor._parse_file(descriptor_file, validate = validate, **kwargs)
+ elif filename == 'cached-extrainfo' or filename == 'cached-extrainfo.new':
+ return stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, validate = validate, **kwargs)
+ elif filename == 'cached-microdescs' or filename == 'cached-microdescs.new':
+ return stem.descriptor.microdescriptor._parse_file(descriptor_file, validate = validate, **kwargs)
+ elif filename == 'cached-consensus':
+ return stem.descriptor.networkstatus._parse_file(descriptor_file, validate = validate, document_handler = document_handler, **kwargs)
+ elif filename == 'cached-microdesc-consensus':
+ return stem.descriptor.networkstatus._parse_file(descriptor_file, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs)
+ else:
+ raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line))
- # Not recognized as a descriptor file.
+ for desc in parse(descriptor_file):
+ if descriptor_path is not None:
+ desc._set_path(os.path.abspath(descriptor_path))
- raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line))
+ yield desc
def _parse_file_for_path(descriptor_file, *args, **kwargs):
@@ -336,13 +331,18 @@ def _values(line, entries):
return [entry[0] for entry in entries[line]]
-def _parse_simple_line(keyword, attribute):
+def _parse_simple_line(keyword, attribute, func = None):
def _parse(descriptor, entries):
- setattr(descriptor, attribute, _value(keyword, entries))
+ value = _value(keyword, entries)
+ setattr(descriptor, attribute, func(value) if func else value)
return _parse
+def _parse_if_present(keyword, attribute):
+ return lambda descriptor, entries: setattr(descriptor, attribute, keyword in entries)
+
+
def _parse_bytes_line(keyword, attribute):
def _parse(descriptor, entries):
line_match = re.search(stem.util.str_tools._to_bytes('^(opt )?%s(?:[%s]+(.*))?$' % (keyword, WHITESPACE)), descriptor.get_bytes(), re.MULTILINE)
@@ -675,13 +675,7 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi
**True**
"""
- if skip:
- content = None
- content_append = lambda x: None
- else:
- content = []
- content_append = content.append
-
+ content = None if skip else []
ending_keyword = None
if isinstance(keywords, (bytes, str_type)):
@@ -690,8 +684,8 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi
if ignore_first:
first_line = descriptor_file.readline()
- if first_line:
- content_append(first_line)
+ if first_line and content is not None:
+ content.append(first_line)
keyword_match = re.compile(SPECIFIC_KEYWORD_LINE % '|'.join(keywords))
@@ -713,12 +707,12 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi
if not inclusive:
descriptor_file.seek(last_position)
- else:
- content_append(line)
+ elif content is not None:
+ content.append(line)
break
- else:
- content_append(line)
+ elif content is not None:
+ content.append(line)
if include_ending_keyword:
return (content, ending_keyword)
diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py
index b190635..3f74f70 100644
--- a/stem/descriptor/microdescriptor.py
+++ b/stem/descriptor/microdescriptor.py
@@ -72,7 +72,6 @@ from stem.descriptor import (
Descriptor,
_get_descriptor_components,
_read_until_keywords,
- _value,
_values,
_parse_simple_line,
_parse_key_block,
@@ -180,11 +179,14 @@ def _parse_id_line(descriptor, entries):
descriptor.identifiers = identities
-_parse_digest = lambda descriptor, entries: setattr(descriptor, 'digest', hashlib.sha256(descriptor.get_bytes()).hexdigest().upper())
+def _parse_digest(descriptor, entries):
+ setattr(descriptor, 'digest', hashlib.sha256(descriptor.get_bytes()).hexdigest().upper())
+
+
_parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY')
_parse_ntor_onion_key_line = _parse_simple_line('ntor-onion-key', 'ntor_onion_key')
-_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', _value('family', entries).split(' '))
-_parse_p6_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('p6', entries)))
+_parse_family_line = _parse_simple_line('family', 'family', func = lambda v: v.split(' '))
+_parse_p6_line = _parse_simple_line('p6', 'exit_policy_v6', func = lambda v: stem.exit_policy.MicroExitPolicy(v))
class Microdescriptor(Descriptor):
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index 103f72c..b471c1f 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -360,10 +360,10 @@ _parse_network_status_version_line = _parse_version_line('network-status-version
_parse_fingerprint_line = _parse_forty_character_hex('fingerprint', 'fingerprint')
_parse_contact_line = _parse_simple_line('contact', 'contact')
_parse_dir_signing_key_line = _parse_key_block('dir-signing-key', 'signing_key', 'RSA PUBLIC KEY')
-_parse_client_versions_line = lambda descriptor, entries: setattr(descriptor, 'client_versions', _value('client-versions', entries).split(','))
-_parse_server_versions_line = lambda descriptor, entries: setattr(descriptor, 'server_versions', _value('server-versions', entries).split(','))
+_parse_client_versions_line = _parse_simple_line('client-versions', 'client_versions', func = lambda v: v.split(','))
+_parse_server_versions_line = _parse_simple_line('server-versions', 'server_versions', func = lambda v: v.split(','))
_parse_published_line = _parse_timestamp_line('published', 'published')
-_parse_dir_options_line = lambda descriptor, entries: setattr(descriptor, 'options', _value('dir-options', entries).split())
+_parse_dir_options_line = _parse_simple_line('dir-options', 'options', func = lambda v: v.split())
_parse_directory_signature_line = _parse_key_block('directory-signature', 'signature', 'SIGNATURE', value_attribute = 'signing_authority')
@@ -685,8 +685,8 @@ _parse_header_fresh_until_line = _parse_timestamp_line('fresh-until', 'fresh_unt
_parse_header_valid_until_line = _parse_timestamp_line('valid-until', 'valid_until')
_parse_header_client_versions_line = _parse_versions_line('client-versions', 'client_versions')
_parse_header_server_versions_line = _parse_versions_line('server-versions', 'server_versions')
-_parse_header_known_flags_line = lambda descriptor, entries: setattr(descriptor, 'known_flags', [entry for entry in _value('known-flags', entries).split(' ') if entry])
-_parse_footer_bandwidth_weights_line = lambda descriptor, entries: setattr(descriptor, 'bandwidth_weights', _parse_int_mappings('bandwidth-weights', _value('bandwidth-weights', entries), True))
+_parse_header_known_flags_line = _parse_simple_line('known-flags', 'known_flags', func = lambda v: [entry for entry in v.split(' ') if entry])
+_parse_footer_bandwidth_weights_line = _parse_simple_line('bandwidth-weights', 'bandwidth_weights', func = lambda v: _parse_int_mappings('bandwidth-weights', v, True))
class NetworkStatusDocumentV3(NetworkStatusDocument):
diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py
index b2b5b8e..4115550 100644
--- a/stem/descriptor/remote.py
+++ b/stem/descriptor/remote.py
@@ -96,11 +96,6 @@ from stem.util import log
MAX_FINGERPRINTS = 96
MAX_MICRODESCRIPTOR_HASHES = 92
-# We commonly only want authorities that vote in the consensus, and hence have
-# a v3ident.
-
-HAS_V3IDENT = lambda auth: auth.v3ident is not None
-
def _guess_descriptor_type(resource):
# Attempts to determine the descriptor type based on the resource url. This
@@ -345,7 +340,7 @@ class Query(object):
"""
if use_authority or not self.endpoints:
- authority = random.choice(list(filter(HAS_V3IDENT, get_authorities().values())))
+ authority = random.choice(list(filter(lambda auth: auth.v3ident is not None, get_authorities().values())))
address, dirport = authority.address, authority.dir_port
else:
address, dirport = random.choice(self.endpoints)
@@ -395,7 +390,7 @@ class DescriptorDownloader(object):
def __init__(self, use_mirrors = False, **default_args):
self._default_args = default_args
- authorities = filter(HAS_V3IDENT, get_authorities().values())
+ authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values())
self._endpoints = [(auth.address, auth.dir_port) for auth in authorities]
if use_mirrors:
@@ -417,7 +412,7 @@ class DescriptorDownloader(object):
:raises: **Exception** if unable to determine the directory mirrors
"""
- authorities = filter(HAS_V3IDENT, get_authorities().values())
+ authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values())
new_endpoints = set([(auth.address, auth.dir_port) for auth in authorities])
consensus = list(self.get_consensus(document_handler = stem.descriptor.DocumentHandler.DOCUMENT).run())[0]
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index a555e12..c2a358f 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -54,6 +54,7 @@ from stem.descriptor import (
_value,
_values,
_parse_simple_line,
+ _parse_if_present,
_parse_bytes_line,
_parse_timestamp_line,
_parse_forty_character_hex,
@@ -394,11 +395,11 @@ _parse_contact_line = _parse_bytes_line('contact', 'contact')
_parse_published_line = _parse_timestamp_line('published', 'published')
_parse_read_history_line = functools.partial(_parse_history_line, 'read-history', 'read_history_end', 'read_history_interval', 'read_history_values')
_parse_write_history_line = functools.partial(_parse_history_line, 'write-history', 'write_history_end', 'write_history_interval', 'write_history_values')
-_parse_ipv6_policy_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('ipv6-policy', entries)))
-_parse_allow_single_hop_exits_line = lambda descriptor, entries: setattr(descriptor, 'allow_single_hop_exits', 'allow_single_hop_exits' in entries)
-_parse_caches_extra_info_line = lambda descriptor, entries: setattr(descriptor, 'extra_info_cache', 'extra_info_cache' in entries)
-_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', set(_value('family', entries).split(' ')))
-_parse_eventdns_line = lambda descriptor, entries: setattr(descriptor, 'eventdns', _value('eventdns', entries) == '1')
+_parse_ipv6_policy_line = _parse_simple_line('ipv6-policy', 'exit_policy_v6', func = lambda v: stem.exit_policy.MicroExitPolicy(v))
+_parse_allow_single_hop_exits_line = _parse_if_present('allow-single-hop-exits', 'allow_single_hop_exits')
+_parse_caches_extra_info_line = _parse_if_present('caches-extra-info', 'extra_info_cache')
+_parse_family_line = _parse_simple_line('family', 'family', func = lambda v: set(v.split(' ')))
+_parse_eventdns_line = _parse_simple_line('eventdns', 'eventdns', func = lambda v: v == '1')
_parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY')
_parse_onion_key_crosscert_line = _parse_key_block('onion-key-crosscert', 'onion_key_crosscert', 'CROSSCERT')
_parse_signing_key_line = _parse_key_block('signing-key', 'signing_key', 'RSA PUBLIC KEY')
diff --git a/stem/version.py b/stem/version.py
index 68ee40f..349d9bb 100644
--- a/stem/version.py
+++ b/stem/version.py
@@ -328,12 +328,13 @@ class _VersionRequirements(object):
:param bool to_inclusive: if comparison is inclusive with the ending version
"""
- if from_inclusive and to_inclusive:
- new_rule = lambda v: from_version <= v <= to_version
- elif from_inclusive:
- new_rule = lambda v: from_version <= v < to_version
- else:
- new_rule = lambda v: from_version < v < to_version
+ def new_rule(v):
+ if from_inclusive and to_inclusive:
+ return from_version <= v <= to_version
+ elif from_inclusive:
+ return from_version <= v < to_version
+ else:
+ return from_version < v < to_version
self.rules.append(new_rule)
diff --git a/test/runner.py b/test/runner.py
index 0370abd..80f390e 100644
--- a/test/runner.py
+++ b/test/runner.py
@@ -713,10 +713,11 @@ class Runner(object):
try:
# wait to fully complete if we're running tests with network activity,
# otherwise finish after local bootstraping
+
complete_percent = 100 if Target.ONLINE in self.attribute_targets else 5
- # prints output from tor's stdout while it starts up
- print_init_line = lambda line: println(' %s' % line, SUBSTATUS)
+ def print_init_line(line):
+ println(' %s' % line, SUBSTATUS)
torrc_dst = os.path.join(self._test_dir, 'torrc')
self._tor_process = stem.process.launch_tor(tor_cmd, None, torrc_dst, complete_percent, print_init_line, take_ownership = True)
diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py
index 73cda28..25ba99d 100644
--- a/test/unit/descriptor/router_status_entry.py
+++ b/test/unit/descriptor/router_status_entry.py
@@ -49,7 +49,10 @@ m 21 sha256=AVp41YVxKEJCaoEf0+77Cdvyw5YgpyDXdob0+LSv/pE
def vote_document():
- mock_document = lambda x: x # just need anything with a __dict__
+ class Stub(object):
+ pass
+
+ mock_document = Stub() # just need anything with a __dict__
setattr(mock_document, 'is_vote', True)
setattr(mock_document, 'is_consensus', False)
return mock_document
diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py
index d295884..cb587d0 100644
--- a/test/unit/descriptor/server_descriptor.py
+++ b/test/unit/descriptor/server_descriptor.py
@@ -376,7 +376,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4=
self.assertEqual(['1'], desc.circuit_protocols)
self.assertEqual(False, desc.hibernating)
self.assertEqual(False, desc.allow_single_hop_exits)
- self.assertEqual(False, desc.extra_info_cache)
+ self.assertEqual(True, desc.extra_info_cache)
self.assertEqual('BB1F13AA431421BEA29B840A2E33BB1C31C2990B', desc.extra_info_digest)
self.assertEqual(None, desc.hidden_service_dir)
self.assertEqual(set(), desc.family)
More information about the tor-commits
mailing list