- tor-commits - lists.torproject.org

[stem/master] Handling the server descriptor's contact and family lines
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 29fad9389f92557e349ffebb32d9cef58b2796c0 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Mar 19 09:14:11 2012 -0700 Handling the server descriptor's contact and family lines --- stem/descriptor/server_descriptor.py | 23 ++++++++++++++++++++--- 1 files changed, 20 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 8ed9afa..b4030d7 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -119,16 +119,19 @@ class ServerDescriptorV2(Descriptor): signing_key_type (str) - block type of the signing_key, probably "RSA PUBLIC KEY" (*) router_sig (str) - signature for this descriptor (*) router_sig_type (str) - block type of the router_sig, probably "SIGNATURE" (*) + contact (str) - relay's contact information + family (list) - nicknames or fingerprints of relays it has a declared family with (*) * required fields, others are left as None if undefined """ nickname = address = or_port = socks_port = dir_port = None average_bandwidth = burst_bandwidth = observed_bandwidth = None - platform = tor_version = published = fingerprint = None - uptime = onion_key = signing_key = None + platform = tor_version = published = fingerprint = uptime = None + onion_key = onion_key_type = signing_key = signing_key_type = None + router_sig = router_sig_type = contact = None hibernating = False - unrecognized_entries = [] + family = unrecognized_entries = [] def __init__(self, contents): Descriptor.__init__(self, contents) @@ -294,6 +297,20 @@ class ServerDescriptorV2(Descriptor): self.router_sig_type = block_type self.router_sig = block_contents + elif keyword == "contact": + self.contact = value + elif keyword == "family": + self.family = value.split(" ") else: unrecognized_entries.append(line) + + def is_valid(self): + """ + Validates that our content matches our signature. + + Returns: + True if our signature matches our content, False otherwise + """ + + raise NotImplementedError # TODO: implement

1 0

[stem/master] Making server descriptor validation optional
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 7e64119a665ecbfb3ee06d3ddcfe066f212f63f6 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Mar 20 08:53:08 2012 -0700 Making server descriptor validation optional Adding constructor pydocs and an argument to skip validation. This should have performance advantages and allow for best-effort parsing of malformed descriptor content. --- stem/descriptor/server_descriptor.py | 109 +++++++++++++++++++++------------ 1 files changed, 69 insertions(+), 40 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index b4030d7..8e3f37b 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -133,7 +133,25 @@ class ServerDescriptorV2(Descriptor): hibernating = False family = unrecognized_entries = [] - def __init__(self, contents): + def __init__(self, contents, validate = True): + """ + Version 2 server descriptor constructor, created from an individual relay's + descriptor content (as provided by "GETINFO desc/*", cached descriptors, + and metrics). + + By default this validates the descriptor's content as it's parsed. This + validation can be disables to either improve performance or be accepting of + malformed data. + + Arguments: + contents (str) - descriptor content provided by the relay + validate (bool) - checks the validity of the descriptor's content if True, + skips these checks otherwise + + Raises: + ValueError if the contents is malformed and validate is True + """ + Descriptor.__init__(self, contents) # A descriptor contains a series of 'keyword lines' which are simply a @@ -160,6 +178,7 @@ class ServerDescriptorV2(Descriptor): line_match = KEYWORD_LINE.match(line) if not line_match: + if not validate: continue raise ValueError("Line contains invalid characters: %s" % line) keyword, value = line_match.groups() @@ -174,13 +193,14 @@ class ServerDescriptorV2(Descriptor): # validates restrictions about the entries - for keyword in REQUIRED_FIELDS: - if not keyword in entries: - raise ValueError("Descriptor must have a '%s' entry" % keyword - - for keyword in SINGLE_FIELDS + REQUIRED_FIELDS: - if keyword in entries and len(entries[keyword]) > 1: - raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) + if validate: + for keyword in REQUIRED_FIELDS: + if not keyword in entries: + raise ValueError("Descriptor must have a '%s' entry" % keyword + + for keyword in SINGLE_FIELDS + REQUIRED_FIELDS: + if keyword in entries and len(entries[keyword]) > 1: + raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) # parse all the entries into our attributes @@ -196,17 +216,20 @@ class ServerDescriptorV2(Descriptor): router_comp = value.split() if len(router_comp) != 5: + if not validate: continue raise ValueError("Router line must have five values: %s" % line - elif not stem.util.tor_tools.is_valid_nickname(router_comp[0]): - raise TypeError("Router line entry isn't a valid nickname: %s" % router_comp[0]) - elif not stem.util.connection.is_valid_ip_address(router_comp[1]): - raise TypeError("Router line entry isn't a valid IPv4 address: %s" % router_comp[1]) - elif not stem.util.connection.is_valid_port(router_comp[2], allow_zero = True): - raise TypeError("Router line's ORPort is invalid: %s" % router_comp[2]) - elif router_comp[3] != "0": - raise TypeError("Router line's SocksPort should be zero: %s" % router_comp[3]) - elif not stem.util.connection.is_valid_port(router_comp[4], allow_zero = True): - raise TypeError("Router line's DirPort is invalid: %s" % router_comp[4]) + + if validate: + if not stem.util.tor_tools.is_valid_nickname(router_comp[0]): + raise ValueError("Router line entry isn't a valid nickname: %s" % router_comp[0]) + elif not stem.util.connection.is_valid_ip_address(router_comp[1]): + raise ValueError("Router line entry isn't a valid IPv4 address: %s" % router_comp[1]) + elif not stem.util.connection.is_valid_port(router_comp[2], allow_zero = True): + raise ValueError("Router line's ORPort is invalid: %s" % router_comp[2]) + elif router_comp[3] != "0": + raise ValueError("Router line's SocksPort should be zero: %s" % router_comp[3]) + elif not stem.util.connection.is_valid_port(router_comp[4], allow_zero = True): + raise ValueError("Router line's DirPort is invalid: %s" % router_comp[4]) self.nickname = router_comp[0] self.address = router_comp[1] @@ -218,13 +241,16 @@ class ServerDescriptorV2(Descriptor): bandwidth_comp = value.split() if len(bandwidth_comp) != 3: + if not validate: continue raise ValueError("Bandwidth line must have three values: %s" % line - elif not bandwidth_comp[0].isdigit()): - raise TypeError("Bandwidth line's average rate isn't numeric: %s" % bandwidth_comp[0]) - elif not bandwidth_comp[1].isdigit()): - raise TypeError("Bandwidth line's burst rate isn't numeric: %s" % bandwidth_comp[1]) - elif not bandwidth_comp[2].isdigit()): - raise TypeError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2]) + + if validate: + if not bandwidth_comp[0].isdigit()): + raise ValueError("Bandwidth line's average rate isn't numeric: %s" % bandwidth_comp[0]) + elif not bandwidth_comp[1].isdigit()): + raise ValueError("Bandwidth line's burst rate isn't numeric: %s" % bandwidth_comp[1]) + elif not bandwidth_comp[2].isdigit()): + raise ValueError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2]) average_bandwidth = int(router_comp[0]) burst_bandwidth = int(router_comp[1]) @@ -252,48 +278,51 @@ class ServerDescriptorV2(Descriptor): try: self.published = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") except ValueError: - raise TypeError("Published line's time wasn't parseable: %s" % line) + if validate: + raise ValueError("Published line's time wasn't parseable: %s" % line) elif keyword == "fingerprint": # This is fourty hex digits split into space separated groups of four. # Checking that we match this pattern. fingerprint = value.replace(" ", "") - for grouping in value.split(" "): - if len(grouping) != 4: - raise TypeError("Fingerprint line should have groupings of four hex digits: %s" % value) - - if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): - raise TypeError("Tor relay fingerprints consist of fourty hex digits: %s" % value) + if validate: + for grouping in value.split(" "): + if len(grouping) != 4: + raise ValueError("Fingerprint line should have groupings of four hex digits: %s" % value) + + if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): + raise ValueError("Tor relay fingerprints consist of fourty hex digits: %s" % value) self.fingerprint = fingerprint elif keyword == "hibernating": # "hibernating" 0|1 (in practice only set if one) - if not value in ("0", "1"): - raise TypeError("Hibernating line had an invalid value, must be zero or one: %s" % value) + if validate and not value in ("0", "1"): + raise ValueError("Hibernating line had an invalid value, must be zero or one: %s" % value) self.hibernating = value == "1" elif keyword == "uptime": if not value.isdigit(): - raise TypeError("Uptime line must have an integer value: %s" % value) + if not validate: continue + raise ValueError("Uptime line must have an integer value: %s" % value) self.uptime = int(value) elif keyword == "onion-key": - if not block_type or not block_contents: - raise TypeError("Onion key line must be followed by a public key: %s" % line) + if validate and (not block_type or not block_contents): + raise ValueError("Onion key line must be followed by a public key: %s" % line) self.onion_key_type = block_type self.onion_key = block_contents elif keyword == "signing-key": - if not block_type or not block_contents: - raise TypeError("Signing key line must be followed by a public key: %s" % line) + if validate and (not block_type or not block_contents): + raise ValueError("Signing key line must be followed by a public key: %s" % line) self.signing_key_type = block_type self.signing_key = block_contents elif keyword == "router-signature": - if not block_type or not block_contents: - raise TypeError("Router signature line must be followed by a signature block: %s" % line) + if validate and (not block_type or not block_contents): + raise ValueError("Router signature line must be followed by a signature block: %s" % line) self.router_sig_type = block_type self.router_sig = block_contents

1 0

[stem/master] Including exit policy among the attributes
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 838a3f37037b7533e010dbb0563191d228fb4947 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Mar 20 09:00:58 2012 -0700 Including exit policy among the attributes We'll later have a proper ExitPolicy class but that's a small project of its own. Until then this will just provide the list of strings that make up the exit policy. --- stem/descriptor/server_descriptor.py | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 8e3f37b..80b13ac 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -109,6 +109,7 @@ class ServerDescriptorV2(Descriptor): observed_bandwidth (int) - estimated capacity of the relay based on usage in bytes/s (*) platform (str) - operating system and tor version tor_version (stem.version.Version) - version of tor + exit_policy (stem.exit_policy.ExitPolicy) - relay's stated exit policy published (datetime.datetime) - time in GMT when the descriptor was generated (*) fingerprint (str) - fourty hex digits that make up the relay's fingerprint hibernating (bool) - flag to indicate if the relay was hibernating when published (*) @@ -133,6 +134,11 @@ class ServerDescriptorV2(Descriptor): hibernating = False family = unrecognized_entries = [] + # TODO: Until we have a proper ExitPolicy class this is just a list of the + # exit policy strings... + + exit_policy = [] + def __init__(self, contents, validate = True): """ Version 2 server descriptor constructor, created from an individual relay's @@ -163,7 +169,6 @@ class ServerDescriptorV2(Descriptor): # does not matter so breaking it into key / value pairs. entries = {} - exit_policy_lines = [] remaining_contents = contents.split("\n") while remaining_contents: @@ -185,7 +190,7 @@ class ServerDescriptorV2(Descriptor): block_type, block_contents = _get_psudo_pgp_block(remaining_contents) if keyword in ("accept", "reject"): - exit_policy_lines.append("%s %s" % (keyword, value)) + self.exit_policy.append("%s %s" % (keyword, value)) elif keyword in entries: entries[keyword].append((value, block_type, block_contents)) else:

1 0

[stem/master] Parsing descriptor files and annotations
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit e20fd3efae6a0e3a7f647017da9f62ccc83aebcc Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Mar 23 07:31:41 2012 -0700 Parsing descriptor files and annotations Completely untested change to finish the implementation of server descriptors. This parses a cached descriptor file into individual descriptors and annotations. Next comes unit and integ tests to start exercising it in an automated way. --- stem/descriptor/server_descriptor.py | 115 ++++++++++++++++++++++++++++++++-- 1 files changed, 110 insertions(+), 5 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 80b13ac..6d16400 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -53,7 +53,76 @@ def parse_server_descriptors_v2(path, descriptor_file): Iterates over the verion 2 server descriptors in a descriptor file. """ - pass + # Cached descriptors consist of annotations followed by the descriptor + # itself. For instance... + # + # @downloaded-at 2012-03-14 16:31:05 + # @source "145.53.65.130" + # router caerSidi 71.35.143.157 9001 0 0 + # platform Tor 0.2.1.30 on Linux x86_64 + # <rest of the descriptor content> + # router-signature + # -----BEGIN SIGNATURE----- + # <signature for the above descriptor> + # -----END SIGNATURE----- + # + # Metrics descriptor files are the same, but lack any annotations. The + # following simply does the following... + # + # - parse as annotations until we get to ENTRY_START + # - parse as descriptor content until we get to ENTRY_END followed by the + # end of the signature block + # - construct a descriptor and provide it back to the caller + + while descriptor_file: + annotations = _read_until_keyword(ENTRY_START, descriptor_file) + descriptor_content = _read_until_keyword(ENTRY_END, descriptor_file) + + # we've reached the 'router-signature', now include the pgp style block + block_end_prefix = PGP_BLOCK_END.split(' ', 1)[0] + descriptor_content += _read_until_keyword(block_end_prefix, descriptor_file, True) + + # If the file has ending annotations (ie, non-descriptor text after the + # last descriptor) then we won't have any descriptor content at this point. + # This is fine. Those ending annotations are simply never returned to the + # caller. + + if descriptor_content: + yield ServerDescriptorV2(descriptr_content, annotations = annotations) + +def _read_until_keyword(keyword, descriptor_file, inclusive = False): + """ + Reads from the descriptor file until we get to the given keyword or reach the + end of the file. + + Arguments: + keyword (str) - keyword we want to read until + descriptor_file (file) - file with the descriptor content + inclusive (bool) - includes the line with the keyword if True + + Returns: + list with the lines until we find the keyword + """ + + content = [] + + while descriptor_file: + last_position = descriptor_file.tell() + line = descriptor_file.readline() + + if not line: continue # blank line + elif " " in line: line_keyword = line.split(" ", 1)[0] + else: line_keyword = line + + if line_keyword == keyword: + if inclusive: content.append(line) + else: descriptor_file.seek(last_position) + + break + else: + content.append(line) + + return content def _get_psudo_pgp_block(remaining_contents): """ @@ -139,7 +208,7 @@ class ServerDescriptorV2(Descriptor): exit_policy = [] - def __init__(self, contents, validate = True): + def __init__(self, contents, validate = True, annotations = None): """ Version 2 server descriptor constructor, created from an individual relay's descriptor content (as provided by "GETINFO desc/*", cached descriptors, @@ -150,9 +219,10 @@ class ServerDescriptorV2(Descriptor): malformed data. Arguments: - contents (str) - descriptor content provided by the relay - validate (bool) - checks the validity of the descriptor's content if True, - skips these checks otherwise + contents (str) - descriptor content provided by the relay + validate (bool) - checks the validity of the descriptor's content if + True, skips these checks otherwise + annotations (list) - lines that appeared prior to the descriptor Raises: ValueError if the contents is malformed and validate is True @@ -160,6 +230,15 @@ class ServerDescriptorV2(Descriptor): Descriptor.__init__(self, contents) + self._annotation_lines = annotations + self._annotation_dict = {} + + for line in annotations: + if " " in line: + key, value = line.split(" ", 1) + self._annotation_dict[key] = value + else: self._annotation_dict[line] = None + # A descriptor contains a series of 'keyword lines' which are simply a # keyword followed by an optional value. Lines can also be followed by a # signature block. @@ -338,6 +417,32 @@ class ServerDescriptorV2(Descriptor): else: unrecognized_entries.append(line) + def get_annotations(self): + """ + Provides content that appeard prior to the descriptor. If this comes from + the cached-descriptors file then this commonly contains content like... + + @downloaded-at 2012-03-18 21:18:29 + @source "173.254.216.66" + + Returns: + dict with the key/value pairs in our annotations + """ + + return self._annotation_dict + + def get_annotation_lines(self): + """ + Provides the lines of content that appeared prior to the descriptor. This + is the same as the get_annotations() results, but with the unparsed lines + and ordering retained. + + Returns: + list with the lines of annotation that came before this descriptor + """ + + return self._annotation_lines + def is_valid(self): """ Validates that our content matches our signature.

1 0

[stem/master] Merging descriptor.py with module init
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 55db7157d30e62af2fa610f1bb962e8fc8a29817 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Mar 23 09:39:50 2012 -0700 Merging descriptor.py with module init The syntax for using the descriptor module was stupid ("import stem.descriptor.descriptor"). Using the json module as an example for using __init__.py for base functions. Common ones will live in 'stem.descritor' but the contents, like 'stem.descriptor.reader', are also accessable. This is similar to how the os module works, though in that case they're using a lot more python magic to also support multiple operating systems. --- stem/descriptor/__init__.py | 86 ++++++++++++++++++++++++++++++++ stem/descriptor/descriptor.py | 89 ---------------------------------- stem/descriptor/server_descriptor.py | 6 +- 3 files changed, 89 insertions(+), 92 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 0a5ba81..96f1bf5 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -4,3 +4,89 @@ Utilities for parsing and processing descriptor data. __all__ = ["descriptor", "reader", "server_descriptor"] +import os + +import stem.descriptor.server_descriptor + +def parse_descriptors(path, descriptor_file): + """ + Provides an iterator for the descriptors within a given file. + + Arguments: + path (str) - absolute path to the file's location on disk + descriptor_file (file) - opened file with the descriptor contents + + Returns: + iterator that parses the file's contents into descriptors + + Raises: + TypeError if we can't match the contents of the file to a descriptor type + IOError if unable to read from the descriptor_file + """ + + # The tor descriptor specifications do not provide a reliable method for + # identifying a descriptor file's type and version so we need to guess + # based on its filename for resources from the data directory and contents + # for files provided by metrics. + + filename = os.path.basename(path) + + if filename == "cached-descriptors": + # server descriptors from tor's data directory + while descriptor_file: + yield stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file) + + return + + first_line = descriptor_file.readline() + descriptor_file.seek(0) + + if first_line.startswith("router "): + # server descriptor + while descriptor_file: + yield stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file) + + return + + # TODO: implement actual descriptor type recognition and parsing + # TODO: add integ test for non-descriptor text content + yield Descriptor(path, descriptor_file.read()) + +class Descriptor: + """ + Common parent for all types of descriptors. + """ + + def __init__(self, contents): + self._path = None + self._raw_contents = contents + + def get_path(self): + """ + Provides the absolute path that we loaded this descriptor from. + + Returns: + str with the absolute path of the descriptor source + """ + + return self._path + + def get_unrecognized_lines(self): + """ + Provides a list of lines that were either ignored or had data that we did + not know how to process. This is most common due to new descriptor fields + that this library does not yet know how to process. Patches welcome! + + Returns: + list of lines of unrecognized content + """ + + return [] + + def _set_path(self, path): + self._path = path + + def __str__(self): + return self._raw_contents + + diff --git a/stem/descriptor/descriptor.py b/stem/descriptor/descriptor.py deleted file mode 100644 index f535afc..0000000 --- a/stem/descriptor/descriptor.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -Common functionality for descriptors. -""" - -import os - -import stem.descriptor.server_descriptor - -def parse_descriptors(path, descriptor_file): - """ - Provides an iterator for the descriptors within a given file. - - Arguments: - path (str) - absolute path to the file's location on disk - descriptor_file (file) - opened file with the descriptor contents - - Returns: - iterator that parses the file's contents into descriptors - - Raises: - TypeError if we can't match the contents of the file to a descriptor type - IOError if unable to read from the descriptor_file - """ - - # The tor descriptor specifications do not provide a reliable method for - # identifying a descriptor file's type and version so we need to guess - # based on its filename for resources from the data directory and contents - # for files provided by metrics. - - filename = os.path.basename(path) - - if filename == "cached-descriptors": - # server descriptors from tor's data directory - while descriptor_file: - yield stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file) - - return - - first_line = descriptor_file.readline() - descriptor_file.seek(0) - - if first_line.startswith("router "): - # server descriptor - while descriptor_file: - yield stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file) - - return - - # TODO: implement actual descriptor type recognition and parsing - # TODO: add integ test for non-descriptor text content - yield Descriptor(path, descriptor_file.read()) - -class Descriptor: - """ - Common parent for all types of descriptors. - """ - - def __init__(self, contents): - self._path = None - self._raw_contents = contents - - def get_path(self): - """ - Provides the absolute path that we loaded this descriptor from. - - Returns: - str with the absolute path of the descriptor source - """ - - return self._path - - def get_unrecognized_lines(self): - """ - Provides a list of lines that were either ignored or had data that we did - not know how to process. This is most common due to new descriptor fields - that this library does not yet know how to process. Patches welcome! - - Returns: - list of lines of unrecognized content - """ - - return [] - - def _set_path(self, path): - self._path = path - - def __str__(self): - return self._raw_contents - diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 40124f5..bd68f91 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -14,7 +14,7 @@ import datetime import stem.version import stem.util.connection import stem.util.tor_tools -import stem.descriptor.descriptor +import stem.descriptor ENTRY_START = "router" ENTRY_END = "router-signature" @@ -164,7 +164,7 @@ def _get_psudo_pgp_block(remaining_contents): else: return (None, None) -class ServerDescriptorV2(stem.descriptor.descriptor.Descriptor): +class ServerDescriptorV2(stem.descriptor.Descriptor): """ Version 2 server descriptor, as specified in... https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec-v2.txt @@ -230,7 +230,7 @@ class ServerDescriptorV2(stem.descriptor.descriptor.Descriptor): ValueError if the contents is malformed and validate is True """ - stem.descriptor.descriptor.Descriptor.__init__(self, contents) + stem.descriptor.Descriptor.__init__(self, contents) self._annotation_lines = annotations self._annotation_dict = {}

1 0

[stem/master] Pylint corrections for server descriptor
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 245c85ecf35e7324ce008c351e1707b5ef96567c Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Mar 23 08:41:43 2012 -0700 Pylint corrections for server descriptor Corrections from running pylint over the untested server descriptor class (all basic problems that would keep it from running). --- stem/descriptor/server_descriptor.py | 32 +++++++++++++++++--------------- 1 files changed, 17 insertions(+), 15 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 6d16400..a60e9f2 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -88,7 +88,9 @@ def parse_server_descriptors_v2(path, descriptor_file): # caller. if descriptor_content: - yield ServerDescriptorV2(descriptr_content, annotations = annotations) + descriptor = ServerDescriptorV2(descriptor_content, annotations = annotations) + descriptor._set_path(path) + yield descriptor def _read_until_keyword(keyword, descriptor_file, inclusive = False): """ @@ -157,7 +159,7 @@ def _get_psudo_pgp_block(remaining_contents): line = remaining_contents.pop(0) block_lines.append(line) - if line == PGP_BLOCK_END $ block_type: + if line == PGP_BLOCK_END % block_type: return block_type, "\n".join(block_lines) else: return (None, None) @@ -280,7 +282,7 @@ class ServerDescriptorV2(Descriptor): if validate: for keyword in REQUIRED_FIELDS: if not keyword in entries: - raise ValueError("Descriptor must have a '%s' entry" % keyword + raise ValueError("Descriptor must have a '%s' entry" % keyword) for keyword in SINGLE_FIELDS + REQUIRED_FIELDS: if keyword in entries and len(entries[keyword]) > 1: @@ -288,7 +290,7 @@ class ServerDescriptorV2(Descriptor): # parse all the entries into our attributes - for keyword, values in entres.items(): + for keyword, values in entries.items(): # most just work with the first (and only) value value, block_type, block_contents = values[0] @@ -301,7 +303,7 @@ class ServerDescriptorV2(Descriptor): if len(router_comp) != 5: if not validate: continue - raise ValueError("Router line must have five values: %s" % line + raise ValueError("Router line must have five values: %s" % line) if validate: if not stem.util.tor_tools.is_valid_nickname(router_comp[0]): @@ -326,19 +328,19 @@ class ServerDescriptorV2(Descriptor): if len(bandwidth_comp) != 3: if not validate: continue - raise ValueError("Bandwidth line must have three values: %s" % line + raise ValueError("Bandwidth line must have three values: %s" % line) if validate: - if not bandwidth_comp[0].isdigit()): + if not bandwidth_comp[0].isdigit(): raise ValueError("Bandwidth line's average rate isn't numeric: %s" % bandwidth_comp[0]) - elif not bandwidth_comp[1].isdigit()): + elif not bandwidth_comp[1].isdigit(): raise ValueError("Bandwidth line's burst rate isn't numeric: %s" % bandwidth_comp[1]) - elif not bandwidth_comp[2].isdigit()): + elif not bandwidth_comp[2].isdigit(): raise ValueError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2]) - average_bandwidth = int(router_comp[0]) - burst_bandwidth = int(router_comp[1]) - observed_bandwidth = int(router_comp[2]) + self.average_bandwidth = int(router_comp[0]) + self.burst_bandwidth = int(router_comp[1]) + self.observed_bandwidth = int(router_comp[2]) elif keyword == "platform": # "platform" string @@ -350,11 +352,11 @@ class ServerDescriptorV2(Descriptor): # # There's no guerentee that we'll be able to pick out the version. - platform_comp = platform.split() + platform_comp = self.platform.split() if platform_comp[0] == "Tor" and len(platform_comp) >= 2: try: - tor_version = stem.version.Version(platform_comp[1]) + self.tor_version = stem.version.Version(platform_comp[1]) except ValueError: pass elif keyword == "published": # "published" YYYY-MM-DD HH:MM:SS @@ -415,7 +417,7 @@ class ServerDescriptorV2(Descriptor): elif keyword == "family": self.family = value.split(" ") else: - unrecognized_entries.append(line) + self.unrecognized_entries.append(line) def get_annotations(self): """

1 0

[stem/master] Using server descriptor when parsing
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 3b657b57b4cb4ce5304145a7a55051e758cf9cbd Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Mar 23 09:28:55 2012 -0700 Using server descriptor when parsing Parsing server descriptors via the new server descriptor class. This breaks the tests and will need quite a bit of work. :) --- stem/descriptor/__init__.py | 2 +- stem/descriptor/descriptor.py | 14 ++++++++++++-- stem/descriptor/server_descriptor.py | 6 +++--- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 33970ef..0a5ba81 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -2,5 +2,5 @@ Utilities for parsing and processing descriptor data. """ -__all__ = ["descriptor", "reader"] +__all__ = ["descriptor", "reader", "server_descriptor"] diff --git a/stem/descriptor/descriptor.py b/stem/descriptor/descriptor.py index a89e5e5..f535afc 100644 --- a/stem/descriptor/descriptor.py +++ b/stem/descriptor/descriptor.py @@ -4,6 +4,8 @@ Common functionality for descriptors. import os +import stem.descriptor.server_descriptor + def parse_descriptors(path, descriptor_file): """ Provides an iterator for the descriptors within a given file. @@ -28,13 +30,21 @@ def parse_descriptors(path, descriptor_file): filename = os.path.basename(path) if filename == "cached-descriptors": - pass # server descriptors from tor's data directory + # server descriptors from tor's data directory + while descriptor_file: + yield stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file) + + return first_line = descriptor_file.readline() descriptor_file.seek(0) if first_line.startswith("router "): - pass # server descriptor + # server descriptor + while descriptor_file: + yield stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file) + + return # TODO: implement actual descriptor type recognition and parsing # TODO: add integ test for non-descriptor text content diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index a60e9f2..40124f5 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -14,7 +14,7 @@ import datetime import stem.version import stem.util.connection import stem.util.tor_tools -from stem.descriptor.descriptor import Descriptor +import stem.descriptor.descriptor ENTRY_START = "router" ENTRY_END = "router-signature" @@ -164,7 +164,7 @@ def _get_psudo_pgp_block(remaining_contents): else: return (None, None) -class ServerDescriptorV2(Descriptor): +class ServerDescriptorV2(stem.descriptor.descriptor.Descriptor): """ Version 2 server descriptor, as specified in... https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec-v2.txt @@ -230,7 +230,7 @@ class ServerDescriptorV2(Descriptor): ValueError if the contents is malformed and validate is True """ - Descriptor.__init__(self, contents) + stem.descriptor.descriptor.Descriptor.__init__(self, contents) self._annotation_lines = annotations self._annotation_dict = {}

1 0

[stem/master] Cleaning up minor import issues
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit ab8fe646817b585ceda7c6062c3294556f8f71cc Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Mar 23 20:23:17 2012 -0700 Cleaning up minor import issues --- stem/__init__.py | 2 +- stem/descriptor/__init__.py | 4 ++-- stem/descriptor/reader.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stem/__init__.py b/stem/__init__.py index 8a914fa..ceb137f 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -2,5 +2,5 @@ Library for working with the tor process. """ -__all__ = ["connection", "control", "process", "socket", "version"] +__all__ = ["descriptor", "util", "connection", "control", "process", "socket", "version"] diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 96f1bf5..7b05963 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -1,8 +1,8 @@ """ -Utilities for parsing and processing descriptor data. +Package for parsing and processing descriptor data. """ -__all__ = ["descriptor", "reader", "server_descriptor"] +__all__ = ["descriptor", "reader", "server_descriptor", "parse_descriptors", "Descriptor"] import os diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 3b6f5fd..4f043bc 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -70,7 +70,7 @@ import threading import mimetypes import Queue -import stem.descriptor.descriptor +import stem.descriptor # flag to indicate when the reader thread is out of descriptor files to read FINISHED = "DONE" @@ -365,7 +365,7 @@ class DescriptorReader: def _handle_descriptor_file(self, target): try: with open(target) as target_file: - for desc in stem.descriptor.descriptor.parse_descriptors(target, target_file): + for desc in stem.descriptor.parse_descriptors(target, target_file): self._enqueue_descriptor(desc) self._iter_notice.set() except TypeError, exc: @@ -380,7 +380,7 @@ class DescriptorReader: if tar_entry.isfile(): entry = tar_file.extractfile(tar_entry) - for desc in stem.descriptor.descriptor.parse_descriptors(target, entry): + for desc in stem.descriptor.parse_descriptors(target, entry): self._enqueue_descriptor(desc) self._iter_notice.set()

1 0

[stem/master] Server descriptor parsing fixes
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 8dc2d66b7dde16e0709dc5c5ff1ac0c320397f22 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Mar 23 22:05:29 2012 -0700 Server descriptor parsing fixes Resolving the variety of issues that were breaking the integ tests. We still don't have any tests to actually verify the server descriptor parsing, but at least we're now exercising it on some real data... and not dieing in a fire! Progress! --- stem/descriptor/__init__.py | 8 +++++--- stem/descriptor/server_descriptor.py | 24 +++++++++++++----------- test/integ/descriptor/reader.py | 4 ++-- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 4d44bc0..ad17296 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -43,14 +43,16 @@ def parse_descriptors(path, descriptor_file): if first_line.startswith("router "): # server descriptor - while descriptor_file: - yield stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file) + for desc in stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file): + yield desc return # TODO: implement actual descriptor type recognition and parsing # TODO: add integ test for non-descriptor text content - yield Descriptor(path, descriptor_file.read()) + desc = Descriptor(descriptor_file.read()) + desc._set_path(path) + yield desc class Descriptor: """ diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index bd68f91..fe6889b 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -21,7 +21,7 @@ ENTRY_END = "router-signature" KEYWORD_CHAR = "a-zA-Z0-9-" WHITESPACE = " \t" -KEYWORD_LINE = re.compile("^([%s]+)[%s]*([%s]*)$" % (KEYWORD_CHAR, WHITESPACE, KEYWORD_CHAR)) +KEYWORD_LINE = re.compile("^([%s]+)[%s]*(.*)$" % (KEYWORD_CHAR, WHITESPACE)) PGP_BLOCK_START = re.compile("^-----BEGIN ([%s%s]+)-----$" % (KEYWORD_CHAR, WHITESPACE)) PGP_BLOCK_END = "-----END %s-----" @@ -74,7 +74,7 @@ def parse_server_descriptors_v2(path, descriptor_file): # end of the signature block # - construct a descriptor and provide it back to the caller - while descriptor_file: + while True: annotations = _read_until_keyword(ENTRY_START, descriptor_file) descriptor_content = _read_until_keyword(ENTRY_END, descriptor_file) @@ -88,9 +88,10 @@ def parse_server_descriptors_v2(path, descriptor_file): # caller. if descriptor_content: - descriptor = ServerDescriptorV2(descriptor_content, annotations = annotations) + descriptor = ServerDescriptorV2("\n".join(descriptor_content), annotations = annotations) descriptor._set_path(path) yield descriptor + else: return # done parsing descriptors def _read_until_keyword(keyword, descriptor_file, inclusive = False): """ @@ -108,12 +109,14 @@ def _read_until_keyword(keyword, descriptor_file, inclusive = False): content = [] - while descriptor_file: + while True: last_position = descriptor_file.tell() line = descriptor_file.readline() - if not line: continue # blank line - elif " " in line: line_keyword = line.split(" ", 1)[0] + if not line: break # EOF + line = line.strip() + + if " " in line: line_keyword = line.split(" ", 1)[0] else: line_keyword = line if line_keyword == keyword: @@ -259,7 +262,7 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): # ignored. This prefix is being removed in... # https://trac.torproject.org/projects/tor/ticket/5124 - line = line.lstrip("opt ") + if line.startswith("opt "): line = line[4:] line_match = KEYWORD_LINE.match(line) @@ -278,7 +281,6 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): entries[keyword] = [(value, block_type, block_contents)] # validates restrictions about the entries - if validate: for keyword in REQUIRED_FIELDS: if not keyword in entries: @@ -338,9 +340,9 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): elif not bandwidth_comp[2].isdigit(): raise ValueError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2]) - self.average_bandwidth = int(router_comp[0]) - self.burst_bandwidth = int(router_comp[1]) - self.observed_bandwidth = int(router_comp[2]) + self.average_bandwidth = int(bandwidth_comp[0]) + self.burst_bandwidth = int(bandwidth_comp[1]) + self.observed_bandwidth = int(bandwidth_comp[2]) elif keyword == "platform": # "platform" string diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index cdc33c2..7f2a425 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -51,7 +51,7 @@ def _get_raw_tar_descriptors(): for tar_entry in tar_file: if tar_entry.isfile(): entry = tar_file.extractfile(tar_entry) - raw_descriptors.append(entry.read()) + raw_descriptors.append(entry.read().strip()) entry.close() TAR_DESCRIPTORS = raw_descriptors @@ -149,7 +149,7 @@ class TestDescriptorReader(unittest.TestCase): descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor") with open(descriptor_path) as descriptor_file: - descriptor_entries.append(descriptor_file.read()) + descriptor_entries.append(descriptor_file.read().strip()) # running this test multiple times to flush out concurrency issues for i in xrange(15):

1 0

[stem/master] Resolving circular dependency with server_descriptor
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit c91532362bd04d90c35adab22d497ba09ee7cc64 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Mar 23 20:23:57 2012 -0700 Resolving circular dependency with server_descriptor ... damn this was a pita to debug. A long explanation of the issue is: http://stackoverflow.com/questions/8991520/python-conditional-module-object… with-personal-package Short story is that the stem.descriptor module imported server_descriptor prior to being fully initialized which caused a server_descriptor use of stem.descriptor.Descriptor to fail. The confusing was that it was actually reporting failure in using "stem.descriptor" (the module itself) which made no sense because it was listed within sys.modules. It turns out that the python importer uses module attributes rather than sys.modules mappings to resolve statements (ie, it was doing getattr(stem, "descriptor")). However, a module's added as an attribute of its parent when its *finished* importing. This fixes the unit tests, but integ tests are still getting stuck when they use the new server_descriptor parsing. --- stem/descriptor/__init__.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 7b05963..4d44bc0 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -6,8 +6,6 @@ __all__ = ["descriptor", "reader", "server_descriptor", "parse_descriptors", "De import os -import stem.descriptor.server_descriptor - def parse_descriptors(path, descriptor_file): """ Provides an iterator for the descriptors within a given file. @@ -24,6 +22,8 @@ def parse_descriptors(path, descriptor_file): IOError if unable to read from the descriptor_file """ + import stem.descriptor.server_descriptor + # The tor descriptor specifications do not provide a reliable method for # identifying a descriptor file's type and version so we need to guess # based on its filename for resources from the data directory and contents

1 0