[tor-commits] [stem/master] Propagating additional descriptor keyword arguments
atagar at torproject.org
atagar at torproject.org
Fri Aug 16 16:06:46 UTC 2013
commit 5be9904711cc7f15f438cad9456dfcc31309c73d
Author: Damian Johnson <atagar at torproject.org>
Date: Fri Aug 16 08:41:45 2013 -0700
Propagating additional descriptor keyword arguments
Adding a 'kwargs' argument to all methods for fetching descriptors. This
argument is passed along to the descriptor contructor. I just ran into a need
for this because the NetworkStatusDocumentV3 class has a 'default_params'
argument that I couldn't access through parse_file(). We'll likely be adding
more constructor args in the future so this will help to make those... well,
useful.
---
stem/descriptor/__init__.py | 41 ++++++++++++++++---------------
stem/descriptor/extrainfo_descriptor.py | 7 +++---
stem/descriptor/microdescriptor.py | 5 ++--
stem/descriptor/networkstatus.py | 8 +++---
stem/descriptor/reader.py | 8 +++---
stem/descriptor/remote.py | 5 +++-
stem/descriptor/server_descriptor.py | 7 +++---
7 files changed, 46 insertions(+), 35 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index 14b29d1..cd1f42a 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -76,7 +76,7 @@ DocumentHandler = stem.util.enum.UppercaseEnum(
)
-def parse_file(descriptor_file, descriptor_type = None, validate = True, document_handler = DocumentHandler.ENTRIES):
+def parse_file(descriptor_file, descriptor_type = None, validate = True, document_handler = DocumentHandler.ENTRIES, **kwargs):
"""
Simple function to read the descriptor contents from a file, providing an
iterator for its :class:`~stem.descriptor.__init__.Descriptor` contents.
@@ -133,6 +133,7 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen
**True**, skips these checks otherwise
:param stem.descriptor.__init__.DocumentHandler document_handler: method in
which to parse the :class:`~stem.descriptor.networkstatus.NetworkStatusDocument`
+ :param dict kwargs: additional arguments for the descriptor constructor
:returns: iterator for :class:`~stem.descriptor.__init__.Descriptor` instances in the file
@@ -146,7 +147,7 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen
if isinstance(descriptor_file, (bytes, unicode)):
with open(descriptor_file) as desc_file:
- for desc in parse_file(desc_file, descriptor_type, validate, document_handler):
+ for desc in parse_file(desc_file, descriptor_type, validate, document_handler, **kwargs):
yield desc
return
@@ -173,27 +174,27 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen
if descriptor_type_match:
desc_type, major_version, minor_version = descriptor_type_match.groups()
- file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler)
+ file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler, **kwargs)
else:
raise ValueError("The descriptor_type must be of the form '<type> <major_version>.<minor_version>'")
elif metrics_header_match:
# Metrics descriptor handling
desc_type, major_version, minor_version = metrics_header_match.groups()
- file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler)
+ file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler, **kwargs)
else:
# Cached descriptor handling. These contain multiple descriptors per file.
if filename == "cached-descriptors":
- file_parser = lambda f: stem.descriptor.server_descriptor._parse_file(f, validate = validate)
+ file_parser = lambda f: stem.descriptor.server_descriptor._parse_file(f, validate = validate, **kwargs)
elif filename == "cached-extrainfo":
- file_parser = lambda f: stem.descriptor.extrainfo_descriptor._parse_file(f, validate = validate)
+ file_parser = lambda f: stem.descriptor.extrainfo_descriptor._parse_file(f, validate = validate, **kwargs)
elif filename == "cached-microdescs":
- file_parser = lambda f: stem.descriptor.microdescriptor._parse_file(f, validate = validate)
+ file_parser = lambda f: stem.descriptor.microdescriptor._parse_file(f, validate = validate, **kwargs)
elif filename == "cached-consensus":
- file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, validate = validate, document_handler = document_handler)
+ file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, validate = validate, document_handler = document_handler, **kwargs)
elif filename == "cached-microdesc-consensus":
- file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, is_microdescriptor = True, validate = validate, document_handler = document_handler)
+ file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs)
if file_parser:
for desc in file_parser(descriptor_file):
@@ -209,50 +210,50 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen
raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line))
-def _parse_metrics_file(descriptor_type, major_version, minor_version, descriptor_file, validate, document_handler):
+def _parse_metrics_file(descriptor_type, major_version, minor_version, descriptor_file, validate, document_handler, **kwargs):
# Parses descriptor files from metrics, yielding individual descriptors. This
# throws a TypeError if the descriptor_type or version isn't recognized.
if descriptor_type == "server-descriptor" and major_version == 1:
- for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = False, validate = validate):
+ for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = False, validate = validate, **kwargs):
yield desc
elif descriptor_type == "bridge-server-descriptor" and major_version == 1:
- for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = True, validate = validate):
+ for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = True, validate = validate, **kwargs):
yield desc
elif descriptor_type == "extra-info" and major_version == 1:
- for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = False, validate = validate):
+ for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = False, validate = validate, **kwargs):
yield desc
elif descriptor_type == "microdescriptor" and major_version == 1:
- for desc in stem.descriptor.microdescriptor._parse_file(descriptor_file, validate = validate):
+ for desc in stem.descriptor.microdescriptor._parse_file(descriptor_file, validate = validate, **kwargs):
yield desc
elif descriptor_type == "bridge-extra-info" and major_version == 1:
# version 1.1 introduced a 'transport' field...
# https://trac.torproject.org/6257
- for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = True, validate = validate):
+ for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = True, validate = validate, **kwargs):
yield desc
elif descriptor_type == "network-status-2" and major_version == 1:
document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV2
- for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler):
+ for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler, **kwargs):
yield desc
elif descriptor_type == "dir-key-certificate-3" and major_version == 1:
- for desc in stem.descriptor.networkstatus._parse_file_key_certs(descriptor_file, validate = validate):
+ for desc in stem.descriptor.networkstatus._parse_file_key_certs(descriptor_file, validate = validate, **kwargs):
yield desc
elif descriptor_type in ("network-status-consensus-3", "network-status-vote-3") and major_version == 1:
document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV3
- for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler):
+ for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler, **kwargs):
yield desc
elif descriptor_type == "network-status-microdesc-consensus-3" and major_version == 1:
document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV3
- for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, is_microdescriptor = True, validate = validate, document_handler = document_handler):
+ for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs):
yield desc
elif descriptor_type == "bridge-network-status" and major_version == 1:
document_type = stem.descriptor.networkstatus.BridgeNetworkStatusDocument
- for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler):
+ for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler, **kwargs):
yield desc
else:
raise TypeError("Unrecognized metrics descriptor format. type: '%s', version: '%i.%i'" % (descriptor_type, major_version, minor_version))
diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py
index fac0991..1441375 100644
--- a/stem/descriptor/extrainfo_descriptor.py
+++ b/stem/descriptor/extrainfo_descriptor.py
@@ -144,7 +144,7 @@ SINGLE_FIELDS = (
)
-def _parse_file(descriptor_file, is_bridge = False, validate = True):
+def _parse_file(descriptor_file, is_bridge = False, validate = True, **kwargs):
"""
Iterates over the extra-info descriptors in a file.
@@ -152,6 +152,7 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True):
:param bool is_bridge: parses the file as being a bridge descriptor
:param bool validate: checks the validity of the descriptor's content if
**True**, skips these checks otherwise
+ :param dict kwargs: additional arguments for the descriptor constructor
:returns: iterator for :class:`~stem.descriptor.extrainfo_descriptor.ExtraInfoDescriptor`
instances in the file
@@ -170,9 +171,9 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True):
if extrainfo_content:
if is_bridge:
- yield BridgeExtraInfoDescriptor(bytes.join(b"", extrainfo_content), validate)
+ yield BridgeExtraInfoDescriptor(bytes.join(b"", extrainfo_content), validate, **kwargs)
else:
- yield RelayExtraInfoDescriptor(bytes.join(b"", extrainfo_content), validate)
+ yield RelayExtraInfoDescriptor(bytes.join(b"", extrainfo_content), validate, **kwargs)
else:
break # done parsing file
diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py
index 5834e18..f31e369 100644
--- a/stem/descriptor/microdescriptor.py
+++ b/stem/descriptor/microdescriptor.py
@@ -88,13 +88,14 @@ SINGLE_FIELDS = (
)
-def _parse_file(descriptor_file, validate = True):
+def _parse_file(descriptor_file, validate = True, **kwargs):
"""
Iterates over the microdescriptors in a file.
:param file descriptor_file: file with descriptor content
:param bool validate: checks the validity of the descriptor's content if
**True**, skips these checks otherwise
+ :param dict kwargs: additional arguments for the descriptor constructor
:returns: iterator for Microdescriptor instances in the file
@@ -136,7 +137,7 @@ def _parse_file(descriptor_file, validate = True):
descriptor_text = bytes.join(b"", descriptor_lines)
- yield Microdescriptor(descriptor_text, validate, annotations)
+ yield Microdescriptor(descriptor_text, validate, annotations, **kwargs)
else:
break # done parsing descriptors
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index ec21304..6258fdc 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -145,7 +145,7 @@ KEY_CERTIFICATE_PARAMS = (
)
-def _parse_file(document_file, document_type = None, validate = True, is_microdescriptor = False, document_handler = DocumentHandler.ENTRIES):
+def _parse_file(document_file, document_type = None, validate = True, is_microdescriptor = False, document_handler = DocumentHandler.ENTRIES, **kwargs):
"""
Parses a network status and iterates over the RouterStatusEntry in it. The
document that these instances reference have an empty 'routers' attribute to
@@ -159,6 +159,7 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde
consensus, **False** otherwise
:param stem.descriptor.__init__.DocumentHandler document_handler: method in
which to parse :class:`~stem.descriptor.networkstatus.NetworkStatusDocument`
+ :param dict kwargs: additional arguments for the descriptor constructor
:returns: :class:`stem.descriptor.networkstatus.NetworkStatusDocument` object
@@ -188,7 +189,7 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde
raise ValueError("Document type %i isn't recognized (only able to parse v2, v3, and bridge)" % document_type)
if document_handler == DocumentHandler.DOCUMENT:
- yield document_type(document_file.read(), validate)
+ yield document_type(document_file.read(), validate, **kwargs)
return
# getting the document without the routers section
@@ -203,7 +204,7 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde
document_content = bytes.join(b"", header + footer)
if document_handler == DocumentHandler.BARE_DOCUMENT:
- yield document_type(document_content, validate)
+ yield document_type(document_content, validate, **kwargs)
elif document_handler == DocumentHandler.ENTRIES:
desc_iterator = stem.descriptor.router_status_entry._parse_file(
document_file,
@@ -213,6 +214,7 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde
start_position = routers_start,
end_position = routers_end,
extra_args = (document_type(document_content, validate),),
+ **kwargs
)
for desc in desc_iterator:
diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py
index 1723929..45a003c 100644
--- a/stem/descriptor/reader.py
+++ b/stem/descriptor/reader.py
@@ -253,9 +253,10 @@ class DescriptorReader(object):
listings from this path, errors are ignored
:param stem.descriptor.__init__.DocumentHandler document_handler: method in
which to parse :class:`~stem.descriptor.networkstatus.NetworkStatusDocument`
+ :param dict kwargs: additional arguments for the descriptor constructor
"""
- def __init__(self, target, validate = True, follow_links = False, buffer_size = 100, persistence_path = None, document_handler = stem.descriptor.DocumentHandler.ENTRIES):
+ def __init__(self, target, validate = True, follow_links = False, buffer_size = 100, persistence_path = None, document_handler = stem.descriptor.DocumentHandler.ENTRIES, **kwargs):
if isinstance(target, (bytes, unicode)):
self._targets = [target]
else:
@@ -269,6 +270,7 @@ class DescriptorReader(object):
self._follow_links = follow_links
self._persistence_path = persistence_path
self._document_handler = document_handler
+ self._kwargs = kwargs
self._read_listeners = []
self._skip_listeners = []
self._processed_files = {}
@@ -513,7 +515,7 @@ class DescriptorReader(object):
self._notify_read_listeners(target)
with open(target, 'rb') as target_file:
- for desc in stem.descriptor.parse_file(target_file, validate = self._validate, document_handler = self._document_handler):
+ for desc in stem.descriptor.parse_file(target_file, validate = self._validate, document_handler = self._document_handler, **self._kwargs):
if self._is_stopped.is_set():
return
@@ -542,7 +544,7 @@ class DescriptorReader(object):
entry = tar_file.extractfile(tar_entry)
try:
- for desc in stem.descriptor.parse_file(entry, validate = self._validate, document_handler = self._document_handler):
+ for desc in stem.descriptor.parse_file(entry, validate = self._validate, document_handler = self._document_handler, **self._kwargs):
if self._is_stopped.is_set():
return
diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py
index 40d9afe..601775e 100644
--- a/stem/descriptor/remote.py
+++ b/stem/descriptor/remote.py
@@ -215,13 +215,14 @@ class Query(object):
**True**, skips these checks otherwise
:var stem.descriptor.__init__.DocumentHandler document_handler: method in
which to parse a :class:`~stem.descriptor.networkstatus.NetworkStatusDocument`
+ :var dict kwargs: additional arguments for the descriptor constructor
:param bool start: start making the request when constructed (default is **True**)
:param bool block: only return after the request has been completed, this is
the same as running **query.run(True)** (default is **False**)
"""
- def __init__(self, resource, descriptor_type = None, endpoints = None, retries = 2, fall_back_to_authority = False, timeout = None, start = True, block = False, validate = True, document_handler = stem.descriptor.DocumentHandler.ENTRIES):
+ def __init__(self, resource, descriptor_type = None, endpoints = None, retries = 2, fall_back_to_authority = False, timeout = None, start = True, block = False, validate = True, document_handler = stem.descriptor.DocumentHandler.ENTRIES, **kwargs):
if not resource.startswith('/'):
raise ValueError("Resources should start with a '/': %s" % resource)
@@ -247,6 +248,7 @@ class Query(object):
self.validate = validate
self.document_handler = document_handler
+ self.kwargs = kwargs
self._downloader_thread = None
self._downloader_thread_lock = threading.RLock()
@@ -319,6 +321,7 @@ class Query(object):
self.descriptor_type,
validate = self.validate,
document_handler = self.document_handler,
+ **self.kwargs
)
for desc in results:
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index 39d4645..a824613 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -82,7 +82,7 @@ SINGLE_FIELDS = (
)
-def _parse_file(descriptor_file, is_bridge = False, validate = True):
+def _parse_file(descriptor_file, is_bridge = False, validate = True, **kwargs):
"""
Iterates over the server descriptors in a file.
@@ -90,6 +90,7 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True):
:param bool is_bridge: parses the file as being a bridge descriptor
:param bool validate: checks the validity of the descriptor's content if
**True**, skips these checks otherwise
+ :param dict kwargs: additional arguments for the descriptor constructor
:returns: iterator for ServerDescriptor instances in the file
@@ -139,9 +140,9 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True):
descriptor_text = bytes.join(b"", descriptor_content)
if is_bridge:
- yield BridgeDescriptor(descriptor_text, validate, annotations)
+ yield BridgeDescriptor(descriptor_text, validate, annotations, **kwargs)
else:
- yield RelayDescriptor(descriptor_text, validate, annotations)
+ yield RelayDescriptor(descriptor_text, validate, annotations, **kwargs)
else:
if validate and annotations:
raise ValueError('Content conform to being a server descriptor:\n%s' % '\n'.join(annotations))
More information about the tor-commits
mailing list