- tor-commits - lists.torproject.org

[stem/master] Skeleton for extra-info descriptors
by atagar＠torproject.org 13 May '12

13 May '12

commit 13944b062ff26a5efa178ff8d5552ff1d049574b Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 6 20:08:53 2012 -0700 Skeleton for extra-info descriptors Basic module for parsing extrainfo descriptors. This doesn't actually do any parsing yet, just turning the raw content into a ExtraInfoDescriptor instance. This abstracts code we'll need out of the server_descriptor module so we'll be able to use it. What code there is here is exercised via the runner tests. --- stem/descriptor/__init__.py | 158 ++++++++++++++++++++++- stem/descriptor/extrainfo_descriptor.py | 111 ++++++++++++++++ stem/descriptor/server_descriptor.py | 159 +---------------------- test/integ/descriptor/data/extrainfo_descriptor | 12 ++ 4 files changed, 286 insertions(+), 154 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 2af1fb0..f3a5983 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -8,9 +8,16 @@ Descriptor - Common parent for all descriptor file types. +- __str__ - string that the descriptor was made from """ -__all__ = ["descriptor", "reader", "server_descriptor", "parse_file", "Descriptor"] +__all__ = ["descriptor", "reader", "extrainfo_descriptor", "server_descriptor", "parse_file", "Descriptor"] import os +import re + +KEYWORD_CHAR = "a-zA-Z0-9-" +WHITESPACE = " \t" +KEYWORD_LINE = re.compile("^([%s]+)[%s]*(.*)$" % (KEYWORD_CHAR, WHITESPACE)) +PGP_BLOCK_START = re.compile("^-----BEGIN ([%s%s]+)-----$" % (KEYWORD_CHAR, WHITESPACE)) +PGP_BLOCK_END = "-----END %s-----" def parse_file(path, descriptor_file): """ @@ -28,6 +35,7 @@ def parse_file(path, descriptor_file): IOError if unable to read from the descriptor_file """ + import stem.descriptor.extrainfo_descriptor import stem.descriptor.server_descriptor # The tor descriptor specifications do not provide a reliable method for @@ -44,6 +52,10 @@ def parse_file(path, descriptor_file): for desc in stem.descriptor.server_descriptor.parse_file(descriptor_file): desc._set_path(path) yield desc + elif filename == "cached-extrainfo" or first_line.startswith("extra-info "): + for desc in stem.descriptor.extrainfo_descriptor.parse_file(descriptor_file): + desc._set_path(path) + yield desc else: # unrecognized descriptor type raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line)) @@ -85,3 +97,147 @@ class Descriptor: def __str__(self): return self._raw_contents +def _read_until_keyword(keyword, descriptor_file, inclusive = False): + """ + Reads from the descriptor file until we get to the given keyword or reach the + end of the file. + + Arguments: + keyword (str) - keyword we want to read until + descriptor_file (file) - file with the descriptor content + inclusive (bool) - includes the line with the keyword if True + + Returns: + list with the lines until we find the keyword + """ + + content = [] + + while True: + last_position = descriptor_file.tell() + line = descriptor_file.readline() + if not line: break # EOF + + if " " in line: line_keyword = line.split(" ", 1)[0] + else: line_keyword = line.strip() + + if line_keyword == keyword: + if inclusive: content.append(line) + else: descriptor_file.seek(last_position) + + break + else: + content.append(line) + + return content + +def _get_pseudo_pgp_block(remaining_contents): + """ + Checks if given contents begins with a pseudo-Open-PGP-style block and, if + so, pops it off and provides it back to the caller. + + Arguments: + remaining_contents (list) - lines to be checked for a public key block + + Returns: + str with the armor wrapped contents or None if it doesn't exist + + Raises: + ValueError if the contents starts with a key block but it's malformed (for + instance, if it lacks an ending line) + """ + + if not remaining_contents: + return None # nothing left + + block_match = PGP_BLOCK_START.match(remaining_contents[0]) + + if block_match: + block_type = block_match.groups()[0] + block_lines = [] + + while True: + if not remaining_contents: + raise ValueError("Unterminated pgp style block") + + line = remaining_contents.pop(0) + block_lines.append(line) + + if line == PGP_BLOCK_END % block_type: + return "\n".join(block_lines) + else: + return None + +def _get_descriptor_components(raw_contents, validate, extra_keywords): + """ + Initial breakup of the server descriptor contents to make parsing easier. + + A descriptor contains a series of 'keyword lines' which are simply a keyword + followed by an optional value. Lines can also be followed by a signature + block. + + To get a sublisting with just certain keywords use extra_keywords. This can + be useful if we care about their relative ordering with respect to each + other. For instance, we care about the ordering of 'accept' and 'reject' + entries because this influences the resulting exit policy, but for everything + else in server descriptors the order does not matter. + + Arguments: + raw_contents (str) - descriptor content provided by the relay + validate (bool) - checks the validity of the descriptor's content if + True, skips these checks otherwise + extra_keywords (list) - entity keywords to put into a separate listing with + ordering intact + + Returns: + tuple with the following attributes... + entries (dict) - keyword => (value, pgp key) entries + first_keyword (str) - keyword of the first line + last_keyword (str) - keyword of the last line + extra_entries (list) - lines containing entries matching extra_keywords + """ + + entries = {} + first_keyword = None + last_keyword = None + extra_entries = [] # entries with a keyword in extra_keywords + remaining_lines = raw_contents.split("\n") + + while remaining_lines: + line = remaining_lines.pop(0) + + # last line can be empty + if not line and not remaining_lines: continue + + # Some lines have an 'opt ' for backward compatability. They should be + # ignored. This prefix is being removed in... + # https://trac.torproject.org/projects/tor/ticket/5124 + + if line.startswith("opt "): line = line[4:] + + line_match = KEYWORD_LINE.match(line) + + if not line_match: + if not validate: continue + raise ValueError("Line contains invalid characters: %s" % line) + + keyword, value = line_match.groups() + + if not first_keyword: first_keyword = keyword + last_keyword = keyword + + try: + block_contents = _get_pseudo_pgp_block(remaining_lines) + except ValueError, exc: + if not validate: continue + raise exc + + if keyword in extra_keywords: + extra_entries.append("%s %s" % (keyword, value)) + elif keyword in entries: + entries[keyword].append((value, block_contents)) + else: + entries[keyword] = [(value, block_contents)] + + return entries, first_keyword, last_keyword, extra_entries + diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py new file mode 100644 index 0000000..891ba18 --- /dev/null +++ b/stem/descriptor/extrainfo_descriptor.py @@ -0,0 +1,111 @@ +""" +Parsing for Tor extra-info descriptors. These are published by relays whenever +their server descriptor is published and have a similar format. However, unlike +server descriptors these don't contain information that Tor clients require to +function and as such aren't fetched by default. + +Defined in section 2.2 of the dir-spec, extra-info descriptors contain +interesting but non-vital information such as usage statistics. These documents +cannot be requested of bridges. + +Extra-info descriptors are available from a few sources... + +- if you have 'DownloadExtraInfo 1' in your torrc... + - control port via 'GETINFO extra-info/digest/*' queries + - the 'cached-extrainfo' file in tor's data directory +- tor metrics, at https://metrics.torproject.org/data.html +- directory authorities and mirrors via their DirPort + +parse_file - Iterates over the extra-info descriptors in a file. +ExtraInfoDescriptor - Tor extra-info descriptor. +""" + +import stem.descriptor + +def parse_file(descriptor_file, validate = True): + """ + Iterates over the extra-info descriptors in a file. + + Arguments: + descriptor_file (file) - file with descriptor content + validate (bool) - checks the validity of the descriptor's content if + True, skips these checks otherwise + + Returns: + iterator for ExtraInfoDescriptor instances in the file + + Raises: + ValueError if the contents is malformed and validate is True + IOError if the file can't be read + """ + + while True: + extrainfo_content = stem.descriptor._read_until_keyword("router-signature", descriptor_file) + + # we've reached the 'router-signature', now include the pgp style block + block_end_prefix = stem.descriptor.PGP_BLOCK_END.split(' ', 1)[0] + extrainfo_content += stem.descriptor._read_until_keyword(block_end_prefix, descriptor_file, True) + + if extrainfo_content: + yield ExtraInfoDescriptor("".join(extrainfo_content), validate) + else: break # done parsing file + +class ExtraInfoDescriptor(stem.descriptor.Descriptor): + """ + Extra-info descriptor document. + + Attributes: + nickname (str) - relay's nickname (*) + fingerprint (str) - fourty hex digits that make up the relay's fingerprint (*) + published (datetime.datetime) - time in GMT when the descriptor was generated (*) + geoip_db_digest (str) - sha1 of geoIP database file + + read_history (str) - read-history line, always unset + read_history_end (datetime.datetime) - end of the sampling interval + read_history_interval (int) - seconds per interval + read_history_values (list) - bytes read during each interval (*) + + write_history (str) - write-history line, always unset + write_history_end (datetime.datetime) - end of the sampling interval + write_history_interval (int) - seconds per interval + write_history_values (list) - bytes written during each interval (*) + + (*) required fields, others are left as None if undefined + """ + + def __init__(self, raw_contents, validate = True, annotations = None): + """ + Extra-info descriptor constructor, created from a relay's extra-info + content (as provided by "GETINFO extra-info/digest/*", cached contents, and + metrics). + + By default this validates the descriptor's content as it's parsed. This + validation can be disables to either improve performance or be accepting of + malformed data. + + Arguments: + raw_contents (str) - extra-info content provided by the relay + validate (bool) - checks the validity of the extra-info descriptor if + True, skips these checks otherwise + + Raises: + ValueError if the contents is malformed and validate is True + """ + + stem.descriptor.Descriptor.__init__(self, raw_contents) + + self.nickname = None + self.fingerprint = None + self.published = None + self.geoip_db_digest = None + + self.read_history = None + self.read_history_end = None + self.read_history_interval = None + self.read_history_values = [] + + self.write_history = None + self.write_history_end = None + self.write_history_interval = None + self.write_history_values = [] + diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 551835d..b87ed2a 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -33,12 +33,6 @@ import stem.version import stem.util.connection import stem.util.tor_tools -KEYWORD_CHAR = "a-zA-Z0-9-" -WHITESPACE = " \t" -KEYWORD_LINE = re.compile("^([%s]+)[%s]*(.*)$" % (KEYWORD_CHAR, WHITESPACE)) -PGP_BLOCK_START = re.compile("^-----BEGIN ([%s%s]+)-----$" % (KEYWORD_CHAR, WHITESPACE)) -PGP_BLOCK_END = "-----END %s-----" - # relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( "router", @@ -127,56 +121,21 @@ def parse_file(descriptor_file, validate = True): # to the caller). while True: - annotations = _read_until_keyword("router", descriptor_file) - descriptor_content = _read_until_keyword("router-signature", descriptor_file) + annotations = stem.descriptor._read_until_keyword("router", descriptor_file) + descriptor_content = stem.descriptor._read_until_keyword("router-signature", descriptor_file) # we've reached the 'router-signature', now include the pgp style block - block_end_prefix = PGP_BLOCK_END.split(' ', 1)[0] - descriptor_content += _read_until_keyword(block_end_prefix, descriptor_file, True) + block_end_prefix = stem.descriptor.PGP_BLOCK_END.split(' ', 1)[0] + descriptor_content += stem.descriptor._read_until_keyword(block_end_prefix, descriptor_file, True) if descriptor_content: # strip newlines from annotations annotations = map(str.strip, annotations) descriptor_text = "".join(descriptor_content) - descriptor = RelayDescriptor(descriptor_text, validate, annotations) - yield descriptor + yield RelayDescriptor(descriptor_text, validate, annotations) else: break # done parsing descriptors -def _read_until_keyword(keyword, descriptor_file, inclusive = False): - """ - Reads from the descriptor file until we get to the given keyword or reach the - end of the file. - - Arguments: - keyword (str) - keyword we want to read until - descriptor_file (file) - file with the descriptor content - inclusive (bool) - includes the line with the keyword if True - - Returns: - list with the lines until we find the keyword - """ - - content = [] - - while True: - last_position = descriptor_file.tell() - line = descriptor_file.readline() - if not line: break # EOF - - if " " in line: line_keyword = line.split(" ", 1)[0] - else: line_keyword = line.strip() - - if line_keyword == keyword: - if inclusive: content.append(line) - else: descriptor_file.seek(last_position) - - break - else: - content.append(line) - - return content - class ServerDescriptor(stem.descriptor.Descriptor): """ Common parent for server descriptors. @@ -297,7 +256,7 @@ class ServerDescriptor(stem.descriptor.Descriptor): # does not matter so breaking it into key / value pairs. entries, first_keyword, last_keyword, self.exit_policy = \ - _get_descriptor_components(raw_contents, validate) + stem.descriptor._get_descriptor_components(raw_contents, validate, ("accept", "reject")) self._parse(entries, validate) if validate: self._check_constraints(entries, first_keyword, last_keyword) @@ -816,109 +775,3 @@ class BridgeDescriptor(ServerDescriptor): def _first_keyword(self): return "router" -def _get_descriptor_components(raw_contents, validate): - """ - Initial breakup of the server descriptor contents to make parsing easier. - - A descriptor contains a series of 'keyword lines' which are simply a keyword - followed by an optional value. Lines can also be followed by a signature - block. - - We care about the ordering of 'accept' and 'reject' entries because this - influences the resulting exit policy, but for everything else the order does - not matter so breaking it into key / value pairs. - - Arguments: - raw_contents (str) - descriptor content provided by the relay - validate (bool) - checks the validity of the descriptor's content if - True, skips these checks otherwise - - Returns: - tuple with the following attributes... - entries (dict) - keyword => (value, pgp key) entries - first_keyword (str) - keyword of the first line - last_keyword (str) - keyword of the last line - exit_policy (list) - lines containing the exit policy - """ - - entries = {} - first_keyword = None - last_keyword = None - exit_policy = [] - remaining_lines = raw_contents.split("\n") - - while remaining_lines: - line = remaining_lines.pop(0) - - # last line can be empty - if not line and not remaining_lines: continue - - # Some lines have an 'opt ' for backward compatability. They should be - # ignored. This prefix is being removed in... - # https://trac.torproject.org/projects/tor/ticket/5124 - - if line.startswith("opt "): line = line[4:] - - line_match = KEYWORD_LINE.match(line) - - if not line_match: - if not validate: continue - raise ValueError("Line contains invalid characters: %s" % line) - - keyword, value = line_match.groups() - - if not first_keyword: first_keyword = keyword - last_keyword = keyword - - try: - block_contents = _get_pseudo_pgp_block(remaining_lines) - except ValueError, exc: - if not validate: continue - raise exc - - if keyword in ("accept", "reject"): - exit_policy.append("%s %s" % (keyword, value)) - elif keyword in entries: - entries[keyword].append((value, block_contents)) - else: - entries[keyword] = [(value, block_contents)] - - return entries, first_keyword, last_keyword, exit_policy - -def _get_pseudo_pgp_block(remaining_contents): - """ - Checks if given contents begins with a pseudo-Open-PGP-style block and, if - so, pops it off and provides it back to the caller. - - Arguments: - remaining_contents (list) - lines to be checked for a public key block - - Returns: - str with the armor wrapped contents or None if it doesn't exist - - Raises: - ValueError if the contents starts with a key block but it's malformed (for - instance, if it lacks an ending line) - """ - - if not remaining_contents: - return None # nothing left - - block_match = PGP_BLOCK_START.match(remaining_contents[0]) - - if block_match: - block_type = block_match.groups()[0] - block_lines = [] - - while True: - if not remaining_contents: - raise ValueError("Unterminated pgp style block") - - line = remaining_contents.pop(0) - block_lines.append(line) - - if line == PGP_BLOCK_END % block_type: - return "\n".join(block_lines) - else: - return None - diff --git a/test/integ/descriptor/data/extrainfo_descriptor b/test/integ/descriptor/data/extrainfo_descriptor new file mode 100644 index 0000000..4525afe --- /dev/null +++ b/test/integ/descriptor/data/extrainfo_descriptor @@ -0,0 +1,12 @@ +extra-info NINJA B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48 +published 2012-05-05 17:03:50 +write-history 2012-05-05 17:02:45 (900 s) 1082368,19456,50176,272384,485376,1850368,1132544,1790976,2459648,4091904,6310912,13701120,3209216,3871744,7873536,5440512,7287808,10561536,9979904,11247616,11982848,7590912,10611712,20728832,38534144,6839296,3173376,16678912 +read-history 2012-05-05 17:02:45 (900 s) 3309568,9216,41984,27648,123904,2004992,364544,576512,1607680,3808256,4672512,12783616,2938880,2562048,7348224,3574784,6488064,10954752,9359360,4438016,6286336,6438912,4502528,10720256,38165504,1524736,2336768,8186880 +dirreq-write-history 2012-05-05 17:02:45 (900 s) 0,0,0,227328,349184,382976,738304,1171456,850944,657408,1675264,987136,702464,1335296,587776,1941504,893952,533504,695296,6828032,6326272,1287168,6310912,10085376,1048576,5372928,894976,8610816 +dirreq-read-history 2012-05-05 17:02:45 (900 s) 0,0,0,0,33792,27648,48128,46080,60416,51200,63488,64512,45056,27648,37888,48128,57344,34816,46080,50176,37888,51200,25600,33792,39936,32768,28672,30720 +router-signature +-----BEGIN SIGNATURE----- +K5FSywk7qvw/boA4DQcqkls6Ize5vcBYfhQ8JnOeRQC9+uDxbnpm3qaYN9jZ8myj +k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw +7LZqklu+gVvhMKREpchVqlAwXkWR44VENm24Hs+mT3M= +-----END SIGNATURE-----

1 0

[stem/master] Basic constraint checking for extra-info descriptors
by atagar＠torproject.org 13 May '12

13 May '12

commit 234a90a79624e5503c4bf65db01c14e54fb9b8de Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 6 20:52:21 2012 -0700 Basic constraint checking for extra-info descriptors Parsing the 'extra-info' line and performing basic validation about extra-info descriptor fields (that we have required fields, most fields don't appear more than once, etc). I'll add some unit tests for this after I've added parsing for all of the mandatory fields. --- stem/descriptor/extrainfo_descriptor.py | 102 +++++++++++++++++++++++++++++++ stem/descriptor/server_descriptor.py | 4 +- 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 891ba18..04530c0 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -22,6 +22,52 @@ ExtraInfoDescriptor - Tor extra-info descriptor. import stem.descriptor +# relay descriptors must have exactly one of the following +REQUIRED_FIELDS = ( + "extra-info", + "published", + "router-signature", +) + +# optional entries that can appear at most once +SINGLE_FIELDS = ( + "read-history", + "write-history", + "geoip-db-digest", + "bridge-stats-end", + "bridge-ips", + "dirreq-stats-end", + "dirreq-v2-ips", + "dirreq-v3-ips", + "dirreq-v2-reqs", + "dirreq-v3-reqs", + "dirreq-v2-share", + "dirreq-v3-share", + "dirreq-v2-resp", + "dirreq-v3-resp", + "dirreq-v2-direct-dl", + "dirreq-v3-direct-dl", + "dirreq-v2-tunneled-dl", + "dirreq-v3-tunneled-dl", + "dirreq-read-history", + "dirreq-write-history", + "entry-stats-end", + "entry-ips", + "cell-stats-end", + "cell-processed-cells", + "cell-queued-cells", + "cell-time-in-queue", + "cell-circuits-per-decile", + "conn-bi-direct", + "exit-stats-end", + "exit-kibibytes-written", + "exit-kibibytes-read", + "exit-streams-opened", +) + +FIRST_FIELD = "extra-info" +LAST_FIELD = "router-signature" + def parse_file(descriptor_file, validate = True): """ Iterates over the extra-info descriptors in a file. @@ -108,4 +154,60 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.write_history_end = None self.write_history_interval = None self.write_history_values = [] + + entries, first_keyword, last_keyword, _ = \ + stem.descriptor._get_descriptor_components(raw_contents, validate, ()) + + if validate: + for keyword in REQUIRED_FIELDS: + if not keyword in entries: + raise ValueError("Extra-info descriptor must have a '%s' entry" % keyword) + + for keyword in REQUIRED_FIELDS + SINGLE_FIELDS: + if keyword in entries and len(entries[keyword]) > 1: + raise ValueError("The '%s' entry can only appear once in an extra-info descriptor" % keyword) + if not first_keyword == FIRST_FIELD: + raise ValueError("Extra-info descriptor must start with a '%s' entry" % FIRST_FIELD) + + if not last_keyword == LAST_FIELD: + raise ValueError("Descriptor must end with a '%s' entry" % LAST_FIELD) + + self._parse(entries, validate) + + def _parse(self, entries, validate): + """ + Parses a series of 'keyword => (value, pgp block)' mappings and applies + them as attributes. + + Arguments: + entries (dict) - descriptor contents to be applied + validate (bool) - checks the validity of descriptor content if True + + Raises: + ValueError if an error occures in validation + """ + + for keyword, values in entries.items(): + # most just work with the first (and only) value + value, block_contents = values[0] + + line = "%s %s" % (keyword, value) # original line + if block_contents: line += "\n%s" % block_contents + + if keyword == "extra-info": + # "extra-info" Nickname Fingerprint + extra_info_comp = value.split() + + if len(extra_info_comp) != 2: + if not validate: continue + raise ValueError("Extra-info line must have two values: %s" % line) + + if validate: + if not stem.util.tor_tools.is_valid_nickname(extra_info_comp[0]): + raise ValueError("Extra-info line entry isn't a valid nickname: %s" % extra_info_comp[0]) + elif not stem.util.tor_tools.is_valid_fingerprint(extra_info_comp[1]): + raise ValueError("Tor relay fingerprints consist of fourty hex digits: %s" % extra_info_comp[1]) + + self.nickname = extra_info_comp[0] + self.fingerprint = extra_info_comp[1] diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index b87ed2a..12b0728 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -543,11 +543,11 @@ class ServerDescriptor(stem.descriptor.Descriptor): expected_first_keyword = self._first_keyword() if expected_first_keyword and not first_keyword == expected_first_keyword: - raise ValueError("Descriptor must start with a '%s' entry" % first_keyword) + raise ValueError("Descriptor must start with a '%s' entry" % expected_first_keyword) expected_last_keyword = self._last_keyword() if expected_last_keyword and not last_keyword == expected_last_keyword: - raise ValueError("Descriptor must end with a '%s' entry" % last_keyword) + raise ValueError("Descriptor must end with a '%s' entry" % expected_last_keyword) if not self.exit_policy: raise ValueError("Descriptor must have at least one 'accept' or 'reject' entry")

1 0

[stem/master] Missed a few more 'version 3' labels
by atagar＠torproject.org 13 May '12

13 May '12

commit 7930b74b6586220cd02ee816107a85fc1ba7e4ae Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 6 13:38:45 2012 -0700 Missed a few more 'version 3' labels --- stem/descriptor/server_descriptor.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 6b53866..551835d 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -6,6 +6,7 @@ etc). This information is provided from a few sources... - control port via 'GETINFO desc/*' queries - the 'cached-descriptors' file in tor's data directory - tor metrics, at https://metrics.torproject.org/data.html +- directory authorities and mirrors via their DirPort parse_file - Iterates over the server descriptors in a file. ServerDescriptor - Tor server descriptor. @@ -222,7 +223,7 @@ class ServerDescriptor(stem.descriptor.Descriptor): def __init__(self, raw_contents, validate = True, annotations = None): """ - Version 3 server descriptor constructor, created from an individual relay's + Server descriptor constructor, created from an individual relay's descriptor content (as provided by "GETINFO desc/*", cached descriptors, and metrics). @@ -602,7 +603,7 @@ class ServerDescriptor(stem.descriptor.Descriptor): class RelayDescriptor(ServerDescriptor): """ - Version 3 server descriptor, as specified in... + Server descriptor, as specified in... https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt Attributes: @@ -697,7 +698,7 @@ class RelayDescriptor(ServerDescriptor): class BridgeDescriptor(ServerDescriptor): """ - Version 3 bridge descriptor, as specified in... + Bridge descriptor, as specified in... https://metrics.torproject.org/formats.html#bridgedesc Attributes:

1 0

[stem/master] Unit tests for extrainfo descriptors
by atagar＠torproject.org 13 May '12

13 May '12

commit 77a498ae31d0c95d1cdaa88752965fcea870990d Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 7 09:23:09 2012 -0700 Unit tests for extrainfo descriptors Few basic unit tests for extrainfo descriptors. These are done in a similar fashion to the server descriptors. I'm highly tempted to refactor out some common bits, but at present that would probably hurt code readability more than it would help. This'll change if I keep using the same type of helpers for descriptor unit tests. --- run_tests.py | 2 + stem/descriptor/extrainfo_descriptor.py | 8 ++ test/unit/descriptor/__init__.py | 2 +- test/unit/descriptor/extrainfo_descriptor.py | 131 ++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 1 deletions(-) diff --git a/run_tests.py b/run_tests.py index 818bfb8..bf2c2bc 100755 --- a/run_tests.py +++ b/run_tests.py @@ -21,6 +21,7 @@ import test.unit.socket.control_line import test.unit.socket.control_message import test.unit.descriptor.reader import test.unit.descriptor.server_descriptor +import test.unit.descriptor.extrainfo_descriptor import test.unit.util.conf import test.unit.util.connection import test.unit.util.enum @@ -94,6 +95,7 @@ UNIT_TESTS = ( test.unit.util.tor_tools.TestTorTools, test.unit.descriptor.reader.TestDescriptorReader, test.unit.descriptor.server_descriptor.TestServerDescriptor, + test.unit.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, test.unit.version.TestVersion, test.unit.socket.control_message.TestControlMessage, test.unit.socket.control_line.TestControlLine, diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index fceaf60..e13787f 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -18,6 +18,7 @@ Extra-info descriptors are available from a few sources... parse_file - Iterates over the extra-info descriptors in a file. ExtraInfoDescriptor - Tor extra-info descriptor. + +- get_unrecognized_lines - lines with unrecognized content """ import datetime @@ -159,6 +160,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.write_history_interval = None self.write_history_values = [] + self._unrecognized_lines = [] + entries, first_keyword, last_keyword, _ = \ stem.descriptor._get_descriptor_components(raw_contents, validate, ()) @@ -178,6 +181,9 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self._parse(entries, validate) + def get_unrecognized_lines(self): + return list(self._unrecognized_lines) + def _parse(self, entries, validate): """ Parses a series of 'keyword => (value, pgp block)' mappings and applies @@ -227,4 +233,6 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Router signature line must be followed by a signature block: %s" % line) self.signature = block_contents + else: + self._unrecognized_lines.append(line) diff --git a/test/unit/descriptor/__init__.py b/test/unit/descriptor/__init__.py index ae61cda..cf4fceb 100644 --- a/test/unit/descriptor/__init__.py +++ b/test/unit/descriptor/__init__.py @@ -2,5 +2,5 @@ Unit tests for stem.descriptor. """ -__all__ = ["reader", "server_descriptor"] +__all__ = ["reader", "extrainfo_descriptor", "server_descriptor"] diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py new file mode 100644 index 0000000..15ffe68 --- /dev/null +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -0,0 +1,131 @@ +""" +Unit tests for stem.descriptor.extrainfo_descriptor. +""" + +import unittest +from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor + +CRYPTO_BLOB = """ +K5FSywk7qvw/boA4DQcqkls6Ize5vcBYfhQ8JnOeRQC9+uDxbnpm3qaYN9jZ8myj +k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw +7LZqklu+gVvhMKREpchVqlAwXkWR44VENm24Hs+mT3M= +""" + +EXTRAINFO_DESCRIPTOR_ATTR = ( + ("extra-info", "ninja B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48"), + ("published", "2012-05-05 17:03:50"), + ("router-signature", "\n-----BEGIN SIGNATURE-----%s-----END SIGNATURE-----" % CRYPTO_BLOB), +) + +def _make_descriptor(attr = None, exclude = None): + """ + Constructs a minimal extrainfo descriptor with the given attributes. + + Arguments: + attr (dict) - keyword/value mappings to be included in the descriptor + exclude (list) - mandatory keywords to exclude from the descriptor + + Returns: + str with customized descriptor content + """ + + descriptor_lines = [] + if attr is None: attr = {} + if exclude is None: exclude = [] + attr = dict(attr) # shallow copy since we're destructive + + for keyword, value in EXTRAINFO_DESCRIPTOR_ATTR: + if keyword in exclude: continue + elif keyword in attr: + value = attr[keyword] + del attr[keyword] + + # if this is the last entry then we should dump in any unused attributes + if keyword == "router-signature": + for attr_keyword, attr_value in attr.items(): + descriptor_lines.append("%s %s" % (attr_keyword, attr_value)) + + descriptor_lines.append("%s %s" % (keyword, value)) + + return "\n".join(descriptor_lines) + +class TestExtraInfoDescriptor(unittest.TestCase): + def test_minimal_extrainfo_descriptor(self): + """ + Basic sanity check that we can parse an extrainfo descriptor with minimal + attributes. + """ + + desc_text = _make_descriptor() + desc = ExtraInfoDescriptor(desc_text) + + self.assertEquals("ninja", desc.nickname) + self.assertEquals("B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48", desc.fingerprint) + self.assertTrue(CRYPTO_BLOB in desc.signature) + + def test_unrecognized_line(self): + """ + Includes unrecognized content in the descriptor. + """ + + desc_text = _make_descriptor({"pepperjack": "is oh so tasty!"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(["pepperjack is oh so tasty!"], desc.get_unrecognized_lines()) + + def test_proceeding_line(self): + """ + Includes a line prior to the 'extra-info' entry. + """ + + desc_text = "exit-streams-opened port=80\n" + _make_descriptor() + self._expect_invalid_attr(desc_text) + + def test_trailing_line(self): + """ + Includes a line after the 'router-signature' entry. + """ + + desc_text = _make_descriptor() + "\nexit-streams-opened port=80" + self._expect_invalid_attr(desc_text) + + def test_extrainfo_line_missing_fields(self): + """ + Checks that validation catches when the extra-info line is missing fields + and that without validation both the nickname and fingerprint are left as + None. + """ + + test_entry = ( + "ninja", + "ninja ", + "B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48", + " B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48", + ) + + for entry in test_entry: + desc_text = _make_descriptor({"extra-info": entry}) + desc = self._expect_invalid_attr(desc_text, "nickname") + self.assertEquals(None, desc.nickname) + self.assertEquals(None, desc.fingerprint) + + def _expect_invalid_attr(self, desc_text, attr = None, expected_value = None): + """ + Asserts that construction will fail due to desc_text having a malformed + attribute. If an attr is provided then we check that it matches an expected + value when we're constructed without validation. + """ + + self.assertRaises(ValueError, ExtraInfoDescriptor, desc_text) + desc = ExtraInfoDescriptor(desc_text, validate = False) + + if attr: + # check that the invalid attribute matches the expected value when + # constructed without validation + + self.assertEquals(expected_value, getattr(desc, attr)) + else: + # check a default attribute + self.assertEquals("ninja", desc.nickname) + + return desc +

1 0

[stem/master] Reverting change to recognize 127.0.0.1 as bridges
by atagar＠torproject.org 13 May '12

13 May '12

commit 9042c172f8544912888d174ab18b4a19882db7e3 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 7 09:27:06 2012 -0700 Reverting change to recognize 127.0.0.1 as bridges Reverting change 477d448 since Karsten reports that bridges with that scrubbing scheme should no longer exist. --- stem/descriptor/server_descriptor.py | 9 ++++----- 1 files changed, 4 insertions(+), 5 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 12b0728..8513b86 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -82,15 +82,14 @@ def parse_file(descriptor_file, validate = True): # Handler for bridge descriptors # # Bridge descriptors are scrubbed so their nickname is 'Unnamed' and their - # ip address is in the 10.x.x.x space (priorly '127.0.0.1'), which is - # normally reserved for private networks. Bridge descriptors only come from - # metrics so a file only contains a single descriptor. + # ip address is in the 10.x.x.x space, which is normally reserved for private + # networks. Bridge descriptors only come from metrics so a file only contains + # a single descriptor. first_line = descriptor_file.readline() descriptor_file.seek(0) - if first_line.startswith("router Unnamed 10.") or \ - first_line.startswith("router Unnamed 127.0.0.1"): + if first_line.startswith("router Unnamed 10."): yield BridgeDescriptor(descriptor_file.read()) return

1 0

[stem/master] Helper function for parsing timestamp/intervals
by atagar＠torproject.org 13 May '12

13 May '12

commit b8e08ce69d30d10cfa6879b7205f62e988fea0e1 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue May 8 09:53:13 2012 -0700 Helper function for parsing timestamp/intervals Several extra-info descriptor lines are of the form... 'YYYY-MM-DD HH:MM:SS (NSEC s) *' So making a helper function to parse it. Using it for the server descriptor's read/write-history entries and it improves code readability quite a bit... --- stem/descriptor/extrainfo_descriptor.py | 35 +++++++++++++++++++ stem/descriptor/server_descriptor.py | 57 ++++++++++++------------------ 2 files changed, 58 insertions(+), 34 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index e13787f..ca34ca1 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -21,6 +21,7 @@ ExtraInfoDescriptor - Tor extra-info descriptor. +- get_unrecognized_lines - lines with unrecognized content """ +import re import datetime import stem.descriptor @@ -99,6 +100,40 @@ def parse_file(descriptor_file, validate = True): yield ExtraInfoDescriptor("".join(extrainfo_content), validate) else: break # done parsing file +def _parse_timestamp_and_interval(keyword, content): + """ + Parses a 'YYYY-MM-DD HH:MM:SS (NSEC s) *' entry. + + Arguments: + keyword (str) - line's keyword + content (str) - line content to be parsed + + Returns: + tuple of the form... + (timestamp (datetime), interval (int), remaining content (str)) + + Raises: + ValueError if the content is malformed + """ + + line = "%s %s" % (keyword, content) + content_match = re.match("^(.*) $([0-9]+) s$( .*)?$", content) + + if not content_match: + raise ValueError("Malformed %s line: %s" % (keyword, line)) + + timestamp_str, interval, remainder = content_match.groups() + if remainder: remainder = remainder[1:] # remove leading space + + if not interval.isdigit(): + raise ValueError("%s line's interval wasn't a number: %s" % (keyword, line)) + + try: + timestamp = datetime.datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") + return timestamp, int(interval), remainder + except ValueError: + raise ValueError("%s line's timestamp wasn't parseable: %s" % (keyword, line)) + class ExtraInfoDescriptor(stem.descriptor.Descriptor): """ Extra-info descriptor document. diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 8513b86..2a9d72f 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -29,6 +29,7 @@ import hashlib import datetime import stem.descriptor +import stem.descriptor.extrainfo_descriptor import stem.version import stem.util.connection import stem.util.tor_tools @@ -467,43 +468,31 @@ class ServerDescriptor(stem.descriptor.Descriptor): elif keyword == "eventdns": self.eventdns = value == "1" elif keyword in ("read-history", "write-history"): - is_read = keyword == "read-history" - - if is_read: self.read_history = value - else: self.write_history = value - - value_match = re.match("^(.*) $([0-9]+) s$ (.*)$", value) - - if not value_match: - if not validate: continue - raise ValueError("Malformed %s line: %s" % (keyword, line)) - - end_value, interval_value, history_values = value_match.groups() - try: - end_datetime = datetime.datetime.strptime(end_value, "%Y-%m-%d %H:%M:%S") + timestamp, interval, remainder = \ + stem.descriptor.extrainfo_descriptor._parse_timestamp_and_interval(keyword, value) - if is_read: self.read_history_end = end_datetime - else: self.write_history_end = end_datetime - except ValueError: - if validate: - raise ValueError("%s line's time wasn't parseable: %s" % (keyword, line)) - - if interval_value.isdigit(): - if is_read: self.read_history_interval = int(interval_value) - else: self.write_history_interval = int(interval_value) - elif validate: - raise ValueError("%s line's interval wasn't a number: %s" % (keyword, line)) - - if history_values != '': - for sampling in history_values.split(","): - if sampling.isdigit(): - if is_read: self.read_history_values.append(int(sampling)) - else: self.write_history_values.append(int(sampling)) + try: + if remainder: + history_values = [int(entry) for entry in remainder.split(",")] else: - if validate: - raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) - else: break + history_values = [] + except ValueError: + raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) + + if keyword == "read-history": + self.read_history = value + self.read_history_end = timestamp + self.read_history_interval = interval + self.read_history_values = history_values + else: + self.write_history = value + self.write_history_end = timestamp + self.write_history_interval = interval + self.write_history_values = history_values + except ValueError, exc: + if not validate: continue + else: raise exc else: self._unrecognized_lines.append(line)

1 0

[stem/master] Parsing read/write-history lines in extrainfo descriptors
by atagar＠torproject.org 13 May '12

13 May '12

commit fa6d81cf69d5590f6bcb0ad94cf34a7b313f02c2 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed May 9 08:09:00 2012 -0700 Parsing read/write-history lines in extrainfo descriptors Copying the code for parsing from the server descriptors. I suspect that this isn't yet exercised by the tests - I'm next gonna improve our integ tests then check if this would be appropriate for a unit test too. --- stem/descriptor/extrainfo_descriptor.py | 25 +++++++++++++++++++++++++ 1 files changed, 25 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index ca34ca1..2ea8829 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -263,6 +263,31 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): except ValueError: if validate: raise ValueError("Published line's time wasn't parseable: %s" % line) + elif keyword in ("read-history", "write-history"): + try: + timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value) + + try: + if remainder: + history_values = [int(entry) for entry in remainder.split(",")] + else: + history_values = [] + except ValueError: + raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) + + if keyword == "read-history": + self.read_history = value + self.read_history_end = timestamp + self.read_history_interval = interval + self.read_history_values = history_values + else: + self.write_history = value + self.write_history_end = timestamp + self.write_history_interval = interval + self.write_history_values = history_values + except ValueError, exc: + if not validate: continue + else: raise exc elif keyword == "router-signature": if validate and not block_contents: raise ValueError("Router signature line must be followed by a signature block: %s" % line)

1 0

[stem/master] Extra-info integraion tests
by atagar＠torproject.org 13 May '12

13 May '12

commit 406205c2549e855140723bc42af874248a6b3026 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed May 9 08:42:48 2012 -0700 Extra-info integraion tests ExtraInfo counterparts for parsing a metrics descriptor and tor's cached file. There's some common bits with the server descriptor tests that I plan to move out. --- run_tests.py | 2 + test/integ/descriptor/__init__.py | 2 +- test/integ/descriptor/extrainfo_descriptor.py | 108 +++++++++++++++++++++++++ test/integ/descriptor/server_descriptor.py | 19 ++--- test/runner.py | 1 + 5 files changed, 119 insertions(+), 13 deletions(-) diff --git a/run_tests.py b/run_tests.py index bf2c2bc..85a4f65 100755 --- a/run_tests.py +++ b/run_tests.py @@ -36,6 +36,7 @@ import test.integ.socket.control_message import test.integ.socket.control_socket import test.integ.descriptor.reader import test.integ.descriptor.server_descriptor +import test.integ.descriptor.extrainfo_descriptor import test.integ.util.conf import test.integ.util.system import test.integ.process @@ -108,6 +109,7 @@ INTEG_TESTS = ( test.integ.util.system.TestSystem, test.integ.descriptor.reader.TestDescriptorReader, test.integ.descriptor.server_descriptor.TestServerDescriptor, + test.integ.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, test.integ.version.TestVersion, test.integ.process.TestProcess, test.integ.socket.control_socket.TestControlSocket, diff --git a/test/integ/descriptor/__init__.py b/test/integ/descriptor/__init__.py index b143c2a..ec61ba7 100644 --- a/test/integ/descriptor/__init__.py +++ b/test/integ/descriptor/__init__.py @@ -2,5 +2,5 @@ Integration tests for stem.descriptor.* contents. """ -__all__ = ["reader", "server_descriptor"] +__all__ = ["reader", "extrainfo_descriptor", "server_descriptor"] diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py new file mode 100644 index 0000000..74ac7fb --- /dev/null +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -0,0 +1,108 @@ +""" +Integration tests for stem.descriptor.extrainfo_descriptor. +""" + +import os +import datetime +import unittest + +import stem.descriptor.extrainfo_descriptor +import test.runner + +my_dir = os.path.dirname(__file__) +DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") + +# 'test_cached_descriptor' is a lengthy test and uneffected by testing targets, +# so including a flag to prevent it from being ran multiple times + +RAN_CACHED_DESCRIPTOR_TEST = False + +class TestExtraInfoDescriptor(unittest.TestCase): + is_cached_descriptors_available = None + + def setUp(self): + if self.is_cached_descriptors_available is None: + test_dir = test.runner.get_runner().get_test_dir() + descriptor_path = os.path.join(test_dir, "cached-extrainfo") + self.is_cached_descriptors_available = os.path.exists(descriptor_path) + + def test_metrics_descriptor(self): + """ + Parses and checks our results against an extrainfo descriptor from metrics. + """ + + descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "extrainfo_descriptor") + + descriptor_file = open(descriptor_path) + descriptor_contents = descriptor_file.read() + descriptor_file.close() + + expected_signature = """-----BEGIN SIGNATURE----- +K5FSywk7qvw/boA4DQcqkls6Ize5vcBYfhQ8JnOeRQC9+uDxbnpm3qaYN9jZ8myj +k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw +7LZqklu+gVvhMKREpchVqlAwXkWR44VENm24Hs+mT3M= +-----END SIGNATURE-----""" + + desc = stem.descriptor.extrainfo_descriptor.ExtraInfoDescriptor(descriptor_contents) + self.assertEquals("NINJA", desc.nickname) + self.assertEquals("B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48", desc.fingerprint) + self.assertEquals(datetime.datetime(2012, 5, 5, 17, 3, 50), desc.published) + self.assertEquals(datetime.datetime(2012, 5, 5, 17, 2, 45), desc.read_history_end) + self.assertEquals(900, desc.read_history_interval) + self.assertEquals(datetime.datetime(2012, 5, 5, 17, 2, 45), desc.write_history_end) + self.assertEquals(900, desc.write_history_interval) + self.assertEquals(expected_signature, desc.signature) + + # TODO: still missing dirreq-read-history and dirreq-write-history + #self.assertEquals([], desc.get_unrecognized_lines()) + + # The read-history and write-history lines are pretty long so just checking + # the initial contents for the line and parsed values. + + read_start = "2012-05-05 17:02:45 (900 s) 3309568,9216,41984" + self.assertTrue(desc.read_history.startswith(read_start)) + + read_values_start = [3309568, 9216, 41984, 27648, 123904] + self.assertEquals(read_values_start, desc.read_history_values[:5]) + + write_start = "2012-05-05 17:02:45 (900 s) 1082368,19456,50176,272384" + self.assertTrue(desc.write_history.startswith(write_start)) + + write_values_start = [1082368, 19456, 50176, 272384, 485376] + self.assertEquals(write_values_start, desc.write_history_values[:5]) + + def test_cached_descriptor(self): + """ + Parses the cached descriptor file in our data directory, checking that it + doesn't raise any validation issues and looking for unrecognized descriptor + additions. + """ + + descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-extrainfo") + + if not self.is_cached_descriptors_available: + self.skipTest("(no cached descriptors)") + + global RAN_CACHED_DESCRIPTOR_TEST + + if RAN_CACHED_DESCRIPTOR_TEST: + self.skipTest("(already ran)") + else: + RAN_CACHED_DESCRIPTOR_TEST = True + + with open(descriptor_path) as descriptor_file: + for desc in stem.descriptor.extrainfo_descriptor.parse_file(descriptor_file): + # TODO: uncomment when we're done implementing the ExtraInfoDescriptor class + #unrecognized_lines = desc.get_unrecognized_lines() + unrecognized_lines = [] + + if unrecognized_lines: + # TODO: This isn't actually a problem, and rather than failing we + # should alert the user about these entries at the end of the tests + # (along with new events, getinfo options, and such). For now though + # there doesn't seem to be anything in practice to trigger this so + # failing to get our attention if it does. + + print "Unrecognized descriptor content: %s" % unrecognized_lines + self.fail() + diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 5ad700f..a74e4eb 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -20,17 +20,17 @@ DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") RAN_CACHED_DESCRIPTOR_TEST = False class TestServerDescriptor(unittest.TestCase): - is_descriptors_available = None + is_cached_descriptors_available = None def setUp(self): # If this is our first time running the integ tests and we didn't wait for # a full tor initialization then the cached descriptors won't exist yet. # Noting if they exist or not since some tests need them. - if self.is_descriptors_available is None: + if self.is_cached_descriptors_available is None: test_dir = test.runner.get_runner().get_test_dir() descriptor_path = os.path.join(test_dir, "cached-descriptors") - self.is_descriptors_available = os.path.exists(descriptor_path) + self.is_cached_descriptors_available = os.path.exists(descriptor_path) def test_metrics_descriptor(self): """ @@ -43,8 +43,6 @@ class TestServerDescriptor(unittest.TestCase): descriptor_contents = descriptor_file.read() descriptor_file.close() - expected_published = datetime.datetime(2012, 3, 1, 17, 15, 27) - expected_family = [ "$0CE3CFB1E9CC47B63EA8869813BF6FAB7D4540C1", "$1FD187E8F69A9B74C9202DC16A25B9E7744AB9F6", @@ -85,7 +83,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(stem.version.Version("0.2.1.30"), desc.tor_version) self.assertEquals("Linux x86_64", desc.operating_system) self.assertEquals(588217, desc.uptime) - self.assertEquals(expected_published, desc.published) + self.assertEquals(datetime.datetime(2012, 3, 1, 17, 15, 27), desc.published) self.assertEquals("www.atagar.com/contact", desc.contact) self.assertEquals(["1", "2"], desc.link_protocols) self.assertEquals(["1"], desc.circuit_protocols) @@ -170,7 +168,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-descriptors") - if not self.is_descriptors_available: + if not self.is_cached_descriptors_available: self.skipTest("(no cached descriptors)") global RAN_CACHED_DESCRIPTOR_TEST @@ -211,7 +209,6 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_contents = descriptor_file.read() descriptor_file.close() - expected_published = datetime.datetime(2012, 3, 21, 16, 28, 14) expected_contact = "2048R/F171EC1F Johan Bl\xc3\xa5b\xc3\xa4ck \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf" desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) @@ -225,7 +222,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(stem.version.Version("0.2.2.35"), desc.tor_version) self.assertEquals("Linux x86_64", desc.operating_system) self.assertEquals(3103848, desc.uptime) - self.assertEquals(expected_published, desc.published) + self.assertEquals(datetime.datetime(2012, 3, 21, 16, 28, 14), desc.published) self.assertEquals(expected_contact, desc.contact) self.assertEquals(["1", "2"], desc.link_protocols) self.assertEquals(["1"], desc.circuit_protocols) @@ -303,8 +300,6 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_contents = descriptor_file.read() descriptor_file.close() - expected_published = datetime.datetime(2012, 3, 22, 17, 34, 38) - expected_family = [ "$CE396C72A3D0880F74C064FEA79D68C15BD380B9", "$AB8B00C00B1347BA80A88E548FAC9EDF701D7D0E", @@ -322,7 +317,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(stem.version.Version("0.2.3.12-alpha"), desc.tor_version) self.assertEquals("Linux x86_64", desc.operating_system) self.assertEquals(186, desc.uptime) - self.assertEquals(expected_published, desc.published) + self.assertEquals(datetime.datetime(2012, 3, 22, 17, 34, 38), desc.published) self.assertEquals("somebody", desc.contact) self.assertEquals(["1", "2"], desc.link_protocols) self.assertEquals(["1"], desc.circuit_protocols) diff --git a/test/runner.py b/test/runner.py index 6acc68e..034f1e5 100644 --- a/test/runner.py +++ b/test/runner.py @@ -63,6 +63,7 @@ ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) BASE_TORRC = """# configuration for stem integration tests DataDirectory %s SocksPort 0 +DownloadExtraInfo 1 """ # We make some paths relative to stem's base directory (the one above us)

1 0

[stem/master] Letting runner's get_test_dir() provide resources
by atagar＠torproject.org 13 May '12

13 May '12

commit 46fb6b272e45ab8c569e1439a53a985c5a8d827d Author: Damian Johnson <atagar(a)torproject.org> Date: Wed May 9 08:54:26 2012 -0700 Letting runner's get_test_dir() provide resources The most common use of the integ runner's get_test_dir() method is to get a path for resources in that directory. Providing an optional argument to get that path rather than making the caller use os.join(). --- test/integ/connection/authentication.py | 2 +- test/integ/descriptor/extrainfo_descriptor.py | 5 ++--- test/integ/descriptor/reader.py | 6 +++--- test/integ/descriptor/server_descriptor.py | 5 ++--- test/integ/util/conf.py | 2 +- test/runner.py | 14 ++++++++++---- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/test/integ/connection/authentication.py b/test/integ/connection/authentication.py index c209d2b..9d912b3 100644 --- a/test/integ/connection/authentication.py +++ b/test/integ/connection/authentication.py @@ -236,7 +236,7 @@ class TestAuthenticate(unittest.TestCase): """ auth_type = stem.connection.AuthMethod.COOKIE - auth_value = os.path.join(test.runner.get_runner().get_test_dir(), "fake_cookie") + auth_value = test.runner.get_runner().get_test_dir("fake_cookie") # we need to create a 32 byte cookie file to load from fake_cookie = open(auth_value, "w") diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 74ac7fb..8b59844 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -22,8 +22,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): def setUp(self): if self.is_cached_descriptors_available is None: - test_dir = test.runner.get_runner().get_test_dir() - descriptor_path = os.path.join(test_dir, "cached-extrainfo") + descriptor_path = test.runner.get_runner().get_test_dir("cached-extrainfo") self.is_cached_descriptors_available = os.path.exists(descriptor_path) def test_metrics_descriptor(self): @@ -78,7 +77,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw additions. """ - descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-extrainfo") + descriptor_path = test.runner.get_runner().get_test_dir("cached-extrainfo") if not self.is_cached_descriptors_available: self.skipTest("(no cached descriptors)") diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index 0d4a638..81635e0 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -24,7 +24,7 @@ DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") TAR_DESCRIPTORS = None def _get_processed_files_path(): - return os.path.join(test.runner.get_runner().get_test_dir(), "descriptor_processed_files") + return test.runner.get_runner().get_test_dir("descriptor_processed_files") def _make_processed_files_listing(contents): """ @@ -372,7 +372,7 @@ class TestDescriptorReader(unittest.TestCase): # types are solely based on file extensions so making something that looks # like an png image - test_path = os.path.join(test.runner.get_runner().get_test_dir(), "test.png") + test_path = test.runner.get_runner().get_test_dir("test.png") try: test_file = open(test_path, "w") @@ -399,7 +399,7 @@ class TestDescriptorReader(unittest.TestCase): Listens for a file that's skipped because we lack read permissions. """ - test_path = os.path.join(test.runner.get_runner().get_test_dir(), "secret_file") + test_path = test.runner.get_runner().get_test_dir("secret_file") try: test_file = open(test_path, "w") diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index a74e4eb..c94c709 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -28,8 +28,7 @@ class TestServerDescriptor(unittest.TestCase): # Noting if they exist or not since some tests need them. if self.is_cached_descriptors_available is None: - test_dir = test.runner.get_runner().get_test_dir() - descriptor_path = os.path.join(test_dir, "cached-descriptors") + descriptor_path = test.runner.get_runner().get_test_dir("cached-descriptors") self.is_cached_descriptors_available = os.path.exists(descriptor_path) def test_metrics_descriptor(self): @@ -166,7 +165,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= additions. """ - descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-descriptors") + descriptor_path = test.runner.get_runner().get_test_dir("cached-descriptors") if not self.is_cached_descriptors_available: self.skipTest("(no cached descriptors)") diff --git a/test/integ/util/conf.py b/test/integ/util/conf.py index 13d0579..a429f2a 100644 --- a/test/integ/util/conf.py +++ b/test/integ/util/conf.py @@ -46,7 +46,7 @@ Why are those arrows", coming my way?!?""" def _get_test_config_path(): - return os.path.join(test.runner.get_runner().get_test_dir(), "integ_test_cfg") + return test.runner.get_runner().get_test_dir("integ_test_cfg") def _make_config(contents): """ diff --git a/test/runner.py b/test/runner.py index 034f1e5..5ee9098 100644 --- a/test/runner.py +++ b/test/runner.py @@ -346,18 +346,24 @@ class Runner: return self._custom_opts - def get_test_dir(self): + def get_test_dir(self, resource = None): """ - Provides the absolute path for our testing directory. + Provides the absolute path for our testing directory or a file within it. + + Arguments: + resource (str) - file within our test directory to provide the path for Returns: - str with our test direcectory path + str with our test direcectory's absolute path or that of a file within it Raises: RunnerStopped if we aren't running """ - return self._get("_test_dir") + if resource: + return os.path.join(self._get("_test_dir"), resource) + else: + return self._get("_test_dir") def get_torrc_path(self): """

1 0

[stem/master] Parsing extrainfo published and router-signature lines
by atagar＠torproject.org 13 May '12

13 May '12

commit 7a91e7d85bbc24df130147cc8789e87a5c0f8911 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 7 08:39:34 2012 -0700 Parsing extrainfo published and router-signature lines Parsing the remaining two mandatory fields for extrainfo descriptors. These fields are identical to what's in server descriptors. --- stem/descriptor/extrainfo_descriptor.py | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 04530c0..fceaf60 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -20,6 +20,8 @@ parse_file - Iterates over the extra-info descriptors in a file. ExtraInfoDescriptor - Tor extra-info descriptor. """ +import datetime + import stem.descriptor # relay descriptors must have exactly one of the following @@ -105,6 +107,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): fingerprint (str) - fourty hex digits that make up the relay's fingerprint (*) published (datetime.datetime) - time in GMT when the descriptor was generated (*) geoip_db_digest (str) - sha1 of geoIP database file + signature (str) - signature for this extrainfo descriptor (*) read_history (str) - read-history line, always unset read_history_end (datetime.datetime) - end of the sampling interval @@ -144,6 +147,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.fingerprint = None self.published = None self.geoip_db_digest = None + self.signature = None self.read_history = None self.read_history_end = None @@ -210,4 +214,17 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.nickname = extra_info_comp[0] self.fingerprint = extra_info_comp[1] + elif keyword == "published": + # "published" YYYY-MM-DD HH:MM:SS + + try: + self.published = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + except ValueError: + if validate: + raise ValueError("Published line's time wasn't parseable: %s" % line) + elif keyword == "router-signature": + if validate and not block_contents: + raise ValueError("Router signature line must be followed by a signature block: %s" % line) + + self.signature = block_contents

1 0