commit 5be9904711cc7f15f438cad9456dfcc31309c73d Author: Damian Johnson atagar@torproject.org Date: Fri Aug 16 08:41:45 2013 -0700

Propagating additional descriptor keyword arguments

Adding a 'kwargs' argument to all methods for fetching descriptors. This argument is passed along to the descriptor contructor. I just ran into a need for this because the NetworkStatusDocumentV3 class has a 'default_params' argument that I couldn't access through parse_file(). We'll likely be adding more constructor args in the future so this will help to make those... well, useful. --- stem/descriptor/__init__.py | 41 ++++++++++++++++--------------- stem/descriptor/extrainfo_descriptor.py | 7 +++--- stem/descriptor/microdescriptor.py | 5 ++-- stem/descriptor/networkstatus.py | 8 +++--- stem/descriptor/reader.py | 8 +++--- stem/descriptor/remote.py | 5 +++- stem/descriptor/server_descriptor.py | 7 +++--- 7 files changed, 46 insertions(+), 35 deletions(-)

diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 14b29d1..cd1f42a 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -76,7 +76,7 @@ DocumentHandler = stem.util.enum.UppercaseEnum( )

-def parse_file(descriptor_file, descriptor_type = None, validate = True, document_handler = DocumentHandler.ENTRIES): +def parse_file(descriptor_file, descriptor_type = None, validate = True, document_handler = DocumentHandler.ENTRIES, **kwargs): """ Simple function to read the descriptor contents from a file, providing an iterator for its :class:`~stem.descriptor.__init__.Descriptor` contents. @@ -133,6 +133,7 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen **True**, skips these checks otherwise :param stem.descriptor.__init__.DocumentHandler document_handler: method in which to parse the :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` + :param dict kwargs: additional arguments for the descriptor constructor

:returns: iterator for :class:`~stem.descriptor.__init__.Descriptor` instances in the file

@@ -146,7 +147,7 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen

if isinstance(descriptor_file, (bytes, unicode)): with open(descriptor_file) as desc_file: - for desc in parse_file(desc_file, descriptor_type, validate, document_handler): + for desc in parse_file(desc_file, descriptor_type, validate, document_handler, **kwargs): yield desc

return @@ -173,27 +174,27 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen

if descriptor_type_match: desc_type, major_version, minor_version = descriptor_type_match.groups() - file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler) + file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler, **kwargs) else: raise ValueError("The descriptor_type must be of the form '<type> <major_version>.<minor_version>'") elif metrics_header_match: # Metrics descriptor handling

desc_type, major_version, minor_version = metrics_header_match.groups() - file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler) + file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler, **kwargs) else: # Cached descriptor handling. These contain multiple descriptors per file.

if filename == "cached-descriptors": - file_parser = lambda f: stem.descriptor.server_descriptor._parse_file(f, validate = validate) + file_parser = lambda f: stem.descriptor.server_descriptor._parse_file(f, validate = validate, **kwargs) elif filename == "cached-extrainfo": - file_parser = lambda f: stem.descriptor.extrainfo_descriptor._parse_file(f, validate = validate) + file_parser = lambda f: stem.descriptor.extrainfo_descriptor._parse_file(f, validate = validate, **kwargs) elif filename == "cached-microdescs": - file_parser = lambda f: stem.descriptor.microdescriptor._parse_file(f, validate = validate) + file_parser = lambda f: stem.descriptor.microdescriptor._parse_file(f, validate = validate, **kwargs) elif filename == "cached-consensus": - file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, validate = validate, document_handler = document_handler) + file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, validate = validate, document_handler = document_handler, **kwargs) elif filename == "cached-microdesc-consensus": - file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, is_microdescriptor = True, validate = validate, document_handler = document_handler) + file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs)

if file_parser: for desc in file_parser(descriptor_file): @@ -209,50 +210,50 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line))

-def _parse_metrics_file(descriptor_type, major_version, minor_version, descriptor_file, validate, document_handler): +def _parse_metrics_file(descriptor_type, major_version, minor_version, descriptor_file, validate, document_handler, **kwargs): # Parses descriptor files from metrics, yielding individual descriptors. This # throws a TypeError if the descriptor_type or version isn't recognized.

if descriptor_type == "server-descriptor" and major_version == 1: - for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = False, validate = validate): + for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = False, validate = validate, **kwargs): yield desc elif descriptor_type == "bridge-server-descriptor" and major_version == 1: - for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = True, validate = validate): + for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = True, validate = validate, **kwargs): yield desc elif descriptor_type == "extra-info" and major_version == 1: - for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = False, validate = validate): + for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = False, validate = validate, **kwargs): yield desc elif descriptor_type == "microdescriptor" and major_version == 1: - for desc in stem.descriptor.microdescriptor._parse_file(descriptor_file, validate = validate): + for desc in stem.descriptor.microdescriptor._parse_file(descriptor_file, validate = validate, **kwargs): yield desc elif descriptor_type == "bridge-extra-info" and major_version == 1: # version 1.1 introduced a 'transport' field... # https://trac.torproject.org/6257

- for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = True, validate = validate): + for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = True, validate = validate, **kwargs): yield desc elif descriptor_type == "network-status-2" and major_version == 1: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV2

- for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler): + for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler, **kwargs): yield desc elif descriptor_type == "dir-key-certificate-3" and major_version == 1: - for desc in stem.descriptor.networkstatus._parse_file_key_certs(descriptor_file, validate = validate): + for desc in stem.descriptor.networkstatus._parse_file_key_certs(descriptor_file, validate = validate, **kwargs): yield desc elif descriptor_type in ("network-status-consensus-3", "network-status-vote-3") and major_version == 1: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV3

- for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler): + for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler, **kwargs): yield desc elif descriptor_type == "network-status-microdesc-consensus-3" and major_version == 1: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV3

- for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, is_microdescriptor = True, validate = validate, document_handler = document_handler): + for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs): yield desc elif descriptor_type == "bridge-network-status" and major_version == 1: document_type = stem.descriptor.networkstatus.BridgeNetworkStatusDocument

- for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler): + for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler, **kwargs): yield desc else: raise TypeError("Unrecognized metrics descriptor format. type: '%s', version: '%i.%i'" % (descriptor_type, major_version, minor_version)) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index fac0991..1441375 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -144,7 +144,7 @@ SINGLE_FIELDS = ( )

-def _parse_file(descriptor_file, is_bridge = False, validate = True): +def _parse_file(descriptor_file, is_bridge = False, validate = True, **kwargs): """ Iterates over the extra-info descriptors in a file.

@@ -152,6 +152,7 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True): :param bool is_bridge: parses the file as being a bridge descriptor :param bool validate: checks the validity of the descriptor's content if **True**, skips these checks otherwise + :param dict kwargs: additional arguments for the descriptor constructor

:returns: iterator for :class:`~stem.descriptor.extrainfo_descriptor.ExtraInfoDescriptor` instances in the file @@ -170,9 +171,9 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True):

if extrainfo_content: if is_bridge: - yield BridgeExtraInfoDescriptor(bytes.join(b"", extrainfo_content), validate) + yield BridgeExtraInfoDescriptor(bytes.join(b"", extrainfo_content), validate, **kwargs) else: - yield RelayExtraInfoDescriptor(bytes.join(b"", extrainfo_content), validate) + yield RelayExtraInfoDescriptor(bytes.join(b"", extrainfo_content), validate, **kwargs) else: break # done parsing file

diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index 5834e18..f31e369 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -88,13 +88,14 @@ SINGLE_FIELDS = ( )

-def _parse_file(descriptor_file, validate = True): +def _parse_file(descriptor_file, validate = True, **kwargs): """ Iterates over the microdescriptors in a file.

:param file descriptor_file: file with descriptor content :param bool validate: checks the validity of the descriptor's content if **True**, skips these checks otherwise + :param dict kwargs: additional arguments for the descriptor constructor

:returns: iterator for Microdescriptor instances in the file

@@ -136,7 +137,7 @@ def _parse_file(descriptor_file, validate = True):

descriptor_text = bytes.join(b"", descriptor_lines)

- yield Microdescriptor(descriptor_text, validate, annotations) + yield Microdescriptor(descriptor_text, validate, annotations, **kwargs) else: break # done parsing descriptors

diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index ec21304..6258fdc 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -145,7 +145,7 @@ KEY_CERTIFICATE_PARAMS = ( )

-def _parse_file(document_file, document_type = None, validate = True, is_microdescriptor = False, document_handler = DocumentHandler.ENTRIES): +def _parse_file(document_file, document_type = None, validate = True, is_microdescriptor = False, document_handler = DocumentHandler.ENTRIES, **kwargs): """ Parses a network status and iterates over the RouterStatusEntry in it. The document that these instances reference have an empty 'routers' attribute to @@ -159,6 +159,7 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde consensus, **False** otherwise :param stem.descriptor.__init__.DocumentHandler document_handler: method in which to parse :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` + :param dict kwargs: additional arguments for the descriptor constructor

:returns: :class:`stem.descriptor.networkstatus.NetworkStatusDocument` object

@@ -188,7 +189,7 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde raise ValueError("Document type %i isn't recognized (only able to parse v2, v3, and bridge)" % document_type)

if document_handler == DocumentHandler.DOCUMENT: - yield document_type(document_file.read(), validate) + yield document_type(document_file.read(), validate, **kwargs) return

# getting the document without the routers section @@ -203,7 +204,7 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde document_content = bytes.join(b"", header + footer)

if document_handler == DocumentHandler.BARE_DOCUMENT: - yield document_type(document_content, validate) + yield document_type(document_content, validate, **kwargs) elif document_handler == DocumentHandler.ENTRIES: desc_iterator = stem.descriptor.router_status_entry._parse_file( document_file, @@ -213,6 +214,7 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde start_position = routers_start, end_position = routers_end, extra_args = (document_type(document_content, validate),), + **kwargs )

for desc in desc_iterator: diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 1723929..45a003c 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -253,9 +253,10 @@ class DescriptorReader(object): listings from this path, errors are ignored :param stem.descriptor.__init__.DocumentHandler document_handler: method in which to parse :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` + :param dict kwargs: additional arguments for the descriptor constructor """

- def __init__(self, target, validate = True, follow_links = False, buffer_size = 100, persistence_path = None, document_handler = stem.descriptor.DocumentHandler.ENTRIES): + def __init__(self, target, validate = True, follow_links = False, buffer_size = 100, persistence_path = None, document_handler = stem.descriptor.DocumentHandler.ENTRIES, **kwargs): if isinstance(target, (bytes, unicode)): self._targets = [target] else: @@ -269,6 +270,7 @@ class DescriptorReader(object): self._follow_links = follow_links self._persistence_path = persistence_path self._document_handler = document_handler + self._kwargs = kwargs self._read_listeners = [] self._skip_listeners = [] self._processed_files = {} @@ -513,7 +515,7 @@ class DescriptorReader(object): self._notify_read_listeners(target)

with open(target, 'rb') as target_file: - for desc in stem.descriptor.parse_file(target_file, validate = self._validate, document_handler = self._document_handler): + for desc in stem.descriptor.parse_file(target_file, validate = self._validate, document_handler = self._document_handler, **self._kwargs): if self._is_stopped.is_set(): return

@@ -542,7 +544,7 @@ class DescriptorReader(object): entry = tar_file.extractfile(tar_entry)

try: - for desc in stem.descriptor.parse_file(entry, validate = self._validate, document_handler = self._document_handler): + for desc in stem.descriptor.parse_file(entry, validate = self._validate, document_handler = self._document_handler, **self._kwargs): if self._is_stopped.is_set(): return

diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 40d9afe..601775e 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -215,13 +215,14 @@ class Query(object): **True**, skips these checks otherwise :var stem.descriptor.__init__.DocumentHandler document_handler: method in which to parse a :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` + :var dict kwargs: additional arguments for the descriptor constructor

:param bool start: start making the request when constructed (default is **True**) :param bool block: only return after the request has been completed, this is the same as running **query.run(True)** (default is **False**) """

- def __init__(self, resource, descriptor_type = None, endpoints = None, retries = 2, fall_back_to_authority = False, timeout = None, start = True, block = False, validate = True, document_handler = stem.descriptor.DocumentHandler.ENTRIES): + def __init__(self, resource, descriptor_type = None, endpoints = None, retries = 2, fall_back_to_authority = False, timeout = None, start = True, block = False, validate = True, document_handler = stem.descriptor.DocumentHandler.ENTRIES, **kwargs): if not resource.startswith('/'): raise ValueError("Resources should start with a '/': %s" % resource)

@@ -247,6 +248,7 @@ class Query(object):

self.validate = validate self.document_handler = document_handler + self.kwargs = kwargs

self._downloader_thread = None self._downloader_thread_lock = threading.RLock() @@ -319,6 +321,7 @@ class Query(object): self.descriptor_type, validate = self.validate, document_handler = self.document_handler, + **self.kwargs )

for desc in results: diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 39d4645..a824613 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -82,7 +82,7 @@ SINGLE_FIELDS = ( )

-def _parse_file(descriptor_file, is_bridge = False, validate = True): +def _parse_file(descriptor_file, is_bridge = False, validate = True, **kwargs): """ Iterates over the server descriptors in a file.

@@ -90,6 +90,7 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True): :param bool is_bridge: parses the file as being a bridge descriptor :param bool validate: checks the validity of the descriptor's content if **True**, skips these checks otherwise + :param dict kwargs: additional arguments for the descriptor constructor

:returns: iterator for ServerDescriptor instances in the file

@@ -139,9 +140,9 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True): descriptor_text = bytes.join(b"", descriptor_content)

if is_bridge: - yield BridgeDescriptor(descriptor_text, validate, annotations) + yield BridgeDescriptor(descriptor_text, validate, annotations, **kwargs) else: - yield RelayDescriptor(descriptor_text, validate, annotations) + yield RelayDescriptor(descriptor_text, validate, annotations, **kwargs) else: if validate and annotations: raise ValueError('Content conform to being a server descriptor:\n%s' % '\n'.join(annotations))