- tor-commits - lists.torproject.org

[stem/master] Cleaning up prior descriptor implementations
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 7e9d454ed4eafe6b5461c5ff5170ca5634899edf Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Mar 24 13:41:11 2012 -0700 Cleaning up prior descriptor implementations Adding header pydocs and made a variety of small fixes and naming improvements. --- stem/descriptor/__init__.py | 45 +++++++---------- stem/descriptor/reader.py | 4 +- stem/descriptor/server_descriptor.py | 86 +++++++++++++++++++++++----------- test/integ/descriptor/reader.py | 4 +- 4 files changed, 81 insertions(+), 58 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index ad17296..215864f 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -1,12 +1,18 @@ """ Package for parsing and processing descriptor data. + +parse_file - Iterates over the descriptors in a file. +Descriptor - Common parent for all descriptor file types. + |- get_path - location of the descriptor on disk if it came from a file + |- get_unrecognized_lines - unparsed descriptor content + +- __str__ - string that the descriptor was made from """ -__all__ = ["descriptor", "reader", "server_descriptor", "parse_descriptors", "Descriptor"] +__all__ = ["descriptor", "reader", "server_descriptor", "parse_file", "Descriptor"] import os -def parse_descriptors(path, descriptor_file): +def parse_file(path, descriptor_file): """ Provides an iterator for the descriptors within a given file. @@ -15,7 +21,7 @@ def parse_descriptors(path, descriptor_file): descriptor_file (file) - opened file with the descriptor contents Returns: - iterator that parses the file's contents into descriptors + iterator for Descriptor instances in the file Raises: TypeError if we can't match the contents of the file to a descriptor type @@ -26,33 +32,21 @@ def parse_descriptors(path, descriptor_file): # The tor descriptor specifications do not provide a reliable method for # identifying a descriptor file's type and version so we need to guess - # based on its filename for resources from the data directory and contents - # for files provided by metrics. + # based on... + # - its filename for resources from the tor data directory + # - first line of our contents for files provided by metrics filename = os.path.basename(path) - - if filename == "cached-descriptors": - # server descriptors from tor's data directory - while descriptor_file: - yield stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file) - - return - first_line = descriptor_file.readline() descriptor_file.seek(0) - if first_line.startswith("router "): - # server descriptor - for desc in stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file): + if filename == "cached-descriptors" or first_line.startswith("router "): + for desc in stem.descriptor.server_descriptor.parse_file_v2(descriptor_file): + desc._set_path(path) yield desc - - return - - # TODO: implement actual descriptor type recognition and parsing - # TODO: add integ test for non-descriptor text content - desc = Descriptor(descriptor_file.read()) - desc._set_path(path) - yield desc + else: + # unrecognized descritor type + raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line)) class Descriptor: """ @@ -83,7 +77,7 @@ class Descriptor: list of lines of unrecognized content """ - return [] + raise NotImplementedError def _set_path(self, path): self._path = path @@ -91,4 +85,3 @@ class Descriptor: def __str__(self): return self._raw_contents - diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 4f043bc..511ba04 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -365,7 +365,7 @@ class DescriptorReader: def _handle_descriptor_file(self, target): try: with open(target) as target_file: - for desc in stem.descriptor.parse_descriptors(target, target_file): + for desc in stem.descriptor.parse_file(target, target_file): self._enqueue_descriptor(desc) self._iter_notice.set() except TypeError, exc: @@ -380,7 +380,7 @@ class DescriptorReader: if tar_entry.isfile(): entry = tar_file.extractfile(tar_entry) - for desc in stem.descriptor.parse_descriptors(target, entry): + for desc in stem.descriptor.parse_file(target, entry): self._enqueue_descriptor(desc) self._iter_notice.set() diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index fe6889b..70db1fe 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -6,15 +6,22 @@ etc). This information is provided from a few sources... - control port via 'GETINFO desc/*' queries - the 'cached-descriptors' file in tor's data directory - tor metrics, at https://metrics.torproject.org/data.html + +parse_file_v2 - Iterates over the server descriptors in a file. +ServerDescriptorV2 - Tor server descriptor, version 2. + |- get_unrecognized_lines - lines with unrecognized content + |- get_annotations - dictionary of content prior to the descriptor entry + |- get_annotation_lines - lines that provided the annotations + +- is_valid - checks the signature against the descriptor content """ import re import datetime +import stem.descriptor import stem.version import stem.util.connection import stem.util.tor_tools -import stem.descriptor ENTRY_START = "router" ENTRY_END = "router-signature" @@ -48,9 +55,21 @@ SINGLE_FIELDS = ( "family", ) -def parse_server_descriptors_v2(path, descriptor_file): +def parse_file_v2(descriptor_file, validate = True): """ - Iterates over the verion 2 server descriptors in a descriptor file. + Iterates over the version 2 server descriptors in a file. + + Arguments: + descriptor_file (file) - file with descriptor content + validate (bool) - checks the validity of the descriptor's content if + True, skips these checks otherwise + + Returns: + iterator for ServerDescriptorV2 instances in the file + + Raises: + ValueError if the contents is malformed and validate is True + IOError if the file can't be read """ # Cached descriptors consist of annotations followed by the descriptor @@ -73,6 +92,9 @@ def parse_server_descriptors_v2(path, descriptor_file): # - parse as descriptor content until we get to ENTRY_END followed by the # end of the signature block # - construct a descriptor and provide it back to the caller + # + # Any annotations after the last server descriptor is ignored (never provided + # to the caller). while True: annotations = _read_until_keyword(ENTRY_START, descriptor_file) @@ -82,16 +104,14 @@ def parse_server_descriptors_v2(path, descriptor_file): block_end_prefix = PGP_BLOCK_END.split(' ', 1)[0] descriptor_content += _read_until_keyword(block_end_prefix, descriptor_file, True) - # If the file has ending annotations (ie, non-descriptor text after the - # last descriptor) then we won't have any descriptor content at this point. - # This is fine. Those ending annotations are simply never returned to the - # caller. - if descriptor_content: - descriptor = ServerDescriptorV2("\n".join(descriptor_content), annotations = annotations) - descriptor._set_path(path) + # strip newlines from annotations + annotations = map(str.strip, annotations) + + descriptor_text = "".join(descriptor_content) + descriptor = ServerDescriptorV2(descriptor_text, validate, annotations) yield descriptor - else: return # done parsing descriptors + else: break # done parsing descriptors def _read_until_keyword(keyword, descriptor_file, inclusive = False): """ @@ -112,12 +132,10 @@ def _read_until_keyword(keyword, descriptor_file, inclusive = False): while True: last_position = descriptor_file.tell() line = descriptor_file.readline() - if not line: break # EOF - line = line.strip() if " " in line: line_keyword = line.split(" ", 1)[0] - else: line_keyword = line + else: line_keyword = line.strip() if line_keyword == keyword: if inclusive: content.append(line) @@ -157,13 +175,13 @@ def _get_psudo_pgp_block(remaining_contents): while True: if not remaining_contents: - raise ValueError("Unterminated public key block") + raise ValueError("Unterminated pgp style block") line = remaining_contents.pop(0) block_lines.append(line) if line == PGP_BLOCK_END % block_type: - return block_type, "\n".join(block_lines) + return (block_type, "\n".join(block_lines)) else: return (None, None) @@ -197,7 +215,7 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): contact (str) - relay's contact information family (list) - nicknames or fingerprints of relays it has a declared family with (*) - * required fields, others are left as None if undefined + (*) required fields, others are left as None if undefined """ nickname = address = or_port = socks_port = dir_port = None @@ -206,7 +224,7 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): onion_key = onion_key_type = signing_key = signing_key_type = None router_sig = router_sig_type = contact = None hibernating = False - family = unrecognized_entries = [] + family = unrecognized_lines = [] # TODO: Until we have a proper ExitPolicy class this is just a list of the # exit policy strings... @@ -258,6 +276,9 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): while remaining_contents: line = remaining_contents.pop(0) + # last line can be empty + if not line and not remaining_contents: continue + # Some lines have an 'opt ' for backward compatability. They should be # ignored. This prefix is being removed in... # https://trac.torproject.org/projects/tor/ticket/5124 @@ -271,7 +292,12 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): raise ValueError("Line contains invalid characters: %s" % line) keyword, value = line_match.groups() - block_type, block_contents = _get_psudo_pgp_block(remaining_contents) + + try: + block_type, block_contents = _get_psudo_pgp_block(remaining_contents) + except ValueError, exc: + if not validate: continue + raise exc if keyword in ("accept", "reject"): self.exit_policy.append("%s %s" % (keyword, value)) @@ -291,7 +317,6 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) # parse all the entries into our attributes - for keyword, values in entries.items(): # most just work with the first (and only) value value, block_type, block_contents = values[0] @@ -332,13 +357,15 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): if not validate: continue raise ValueError("Bandwidth line must have three values: %s" % line) - if validate: - if not bandwidth_comp[0].isdigit(): - raise ValueError("Bandwidth line's average rate isn't numeric: %s" % bandwidth_comp[0]) - elif not bandwidth_comp[1].isdigit(): - raise ValueError("Bandwidth line's burst rate isn't numeric: %s" % bandwidth_comp[1]) - elif not bandwidth_comp[2].isdigit(): - raise ValueError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2]) + if not bandwidth_comp[0].isdigit(): + if not validate: continue + raise ValueError("Bandwidth line's average rate isn't numeric: %s" % bandwidth_comp[0]) + elif not bandwidth_comp[1].isdigit(): + if not validate: continue + raise ValueError("Bandwidth line's burst rate isn't numeric: %s" % bandwidth_comp[1]) + elif not bandwidth_comp[2].isdigit(): + if not validate: continue + raise ValueError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2]) self.average_bandwidth = int(bandwidth_comp[0]) self.burst_bandwidth = int(bandwidth_comp[1]) @@ -419,7 +446,10 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): elif keyword == "family": self.family = value.split(" ") else: - self.unrecognized_entries.append(line) + self.unrecognized_lines.append(line) + + def get_unrecognized_lines(self): + return list(unrecognized_lines) def get_annotations(self): """ diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index 7f2a425..cdc33c2 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -51,7 +51,7 @@ def _get_raw_tar_descriptors(): for tar_entry in tar_file: if tar_entry.isfile(): entry = tar_file.extractfile(tar_entry) - raw_descriptors.append(entry.read().strip()) + raw_descriptors.append(entry.read()) entry.close() TAR_DESCRIPTORS = raw_descriptors @@ -149,7 +149,7 @@ class TestDescriptorReader(unittest.TestCase): descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor") with open(descriptor_path) as descriptor_file: - descriptor_entries.append(descriptor_file.read().strip()) + descriptor_entries.append(descriptor_file.read()) # running this test multiple times to flush out concurrency issues for i in xrange(15):

1 0

[stem/master] Integration test for reading non-descriptor files
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 9faad3255491642f87801926512c9cb6d334f6d8 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Mar 24 16:50:03 2012 -0700 Integration test for reading non-descriptor files Integ test for reading non-descriptor plaintext and binary data. --- stem/descriptor/reader.py | 7 ++++++- test/integ/descriptor/data/riddle | 20 ++++++++++++++++++++ test/integ/descriptor/data/tiny.png | Bin 0 -> 188 bytes test/integ/descriptor/reader.py | 18 ++++++++++++++++++ 4 files changed, 44 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 511ba04..de39d54 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -94,7 +94,10 @@ class ParsingFailure(FileSkipped): self.exception = parsing_exception class UnrecognizedType(FileSkipped): - "File's mime type indicates that it isn't descriptor data." + """ + File doesn't contain descriptor data. This could either be due to its file + type or because it doens't conform to a recognizable descriptor type. + """ def __init__(self, mime_type): FileSkipped.__init__(self) @@ -369,6 +372,8 @@ class DescriptorReader: self._enqueue_descriptor(desc) self._iter_notice.set() except TypeError, exc: + self._notify_skip_listeners(target, UnrecognizedType(None)) + except ValueError, exc: self._notify_skip_listeners(target, ParsingFailure(exc)) except IOError, exc: self._notify_skip_listeners(target, ReadFailed(exc)) diff --git a/test/integ/descriptor/data/riddle b/test/integ/descriptor/data/riddle new file mode 100644 index 0000000..ac08f36 --- /dev/null +++ b/test/integ/descriptor/data/riddle @@ -0,0 +1,20 @@ +Riddle by Damian Johnson (May, 2009) + +Curtains raise, they take the stage, + mute actors that never age. +The clouds are cotton, the floor is pine, + the souls bound up in fine twine. + +Crowds will cheer, the players dance, + flowing scene to scene as in a trance. +The world is perfect, they never whine, + for there's not a single will, save mine. + +The play has ended, we take a bow, + but the viewers are the actors now. +When freed from decision, of thought and blame, + we'll walk to another drum just the same. + +answer: +http://www.atagar.com/riddles/answer10.php + diff --git a/test/integ/descriptor/data/tiny.png b/test/integ/descriptor/data/tiny.png new file mode 100644 index 0000000..e86715f Binary files /dev/null and b/test/integ/descriptor/data/tiny.png differ diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index cdc33c2..717a55b 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -287,6 +287,24 @@ class TestDescriptorReader(unittest.TestCase): self.assertEquals(expected_results, reader.get_processed_files()) + def test_skip_nondescriptor_contents(self): + """ + Checks that the reader properly reports when it skips both binary and + plaintext non-descriptor files. + """ + + skip_listener = SkipListener() + reader = stem.descriptor.reader.DescriptorReader([DESCRIPTOR_TEST_DATA]) + reader.register_skip_listener(skip_listener.listener) + + with reader: list(reader) # iterates over all of the descriptors + + self.assertTrue(2, len(skip_listener.results)) + + for skip_path, skip_exception in skip_listener.results: + self.assertTrue(os.path.basename(skip_path) in ("riddle", "tiny.png")) + self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType)) + def test_skip_listener_already_read(self): """ Checks that calling set_processed_files() prior to reading makes us skip

1 0

[stem/master] Missing version 3 attributes
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 7dc7d82bd7b7f70f3dfae21dd607502b4eff6477 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Mar 24 19:38:27 2012 -0700 Missing version 3 attributes Adding the server descriptor attributes that are new as of version 3. The only attributes that I'm still excluding are eventdns (because it's dead) and read/write-history (since they're extra-info now). --- stem/descriptor/server_descriptor.py | 56 +++++++++++++++++++++++++-- test/integ/descriptor/server_descriptor.py | 6 +++ 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index c301840..ceee835 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -33,26 +33,31 @@ PGP_BLOCK_START = re.compile("^-----BEGIN ([%s%s]+)-----$" % (KEYWORD_CHAR, WHIT PGP_BLOCK_END = "-----END %s-----" # entries must have exactly one of the following -# TODO: spec doesn't list 'router-signature', but that's a spec bug I should fix REQUIRED_FIELDS = ( + "router", + "bandwidth", "published", "onion-key", "signing-key", - "bandwidth", "router-signature", ) # optional entries that can appear at most once SINGLE_FIELDS = ( - "contact", - "uptime", + "platform", "fingerprint", "hibernating", + "uptime", + "contact", "read-history", "write-history", "eventdns", - "platform", "family", + "caches-extra-info", + "extra-info-digest", + "hidden-service-dir", + "protocols", + "allow-single-hop-exits", ) def parse_file_v3(descriptor_file, validate = True): @@ -203,7 +208,13 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): uptime (int) - relay's uptime when published in seconds published (datetime.datetime) - time in GMT when the descriptor was generated (*) contact (str) - relay's contact information + link_protocols (list) - link protocols supported by the relay + circuit_protocols (list) - circuit protocols supported by the relay hibernating (bool) - flag to indicate if the relay was hibernating when published (*) + allow_single_hop_exits (bool) - flag to indicate if single hop exiting is allowed from it (*) + extra_info_cache (bool) - flag to indicate if it's a mirror for extra-info documents (*) + extra_info_digest (str) - hex encoded digest of our extra-info document + hidden_service_dir (list) - hidden service descriptor versions that it stores exit_policy (stem.exit_policy.ExitPolicy) - relay's stated exit policy family (list) - nicknames or fingerprints of relays it has a declared family with (*) average_bandwidth (int) - rate of traffic relay is willing to relay in bytes/s (*) @@ -253,7 +264,13 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): self.uptime = None self.published = None self.contact = None + self.link_protocols = None + self.circuit_protocols = None self.hibernating = False + self.allow_single_hop_exits = False + self.extra_info_cache = False + self.extra_info_digest = None + self.hidden_service_dir = None self.family = [] self.average_bandwidth = None self.burst_bandwidth = None @@ -337,6 +354,9 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): for keyword in SINGLE_FIELDS + REQUIRED_FIELDS: if keyword in entries and len(entries[keyword]) > 1: raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) + + if not self.exit_policy: + raise ValueError("Descriptor must have at least one 'accept' or 'reject' entry") # parse all the entries into our attributes for keyword, values in entries.items(): @@ -444,6 +464,23 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): raise ValueError("Hibernating line had an invalid value, must be zero or one: %s" % value) self.hibernating = value == "1" + elif keyword == "allow-single-hop-exits": + self.allow_single_hop_exits = True + elif keyword == "caches-extra-info": + self.extra_info_cache = True + elif keyword == "extra-info-digest": + # this is fourty hex digits which just so happens to be the same a + # fingerprint + + if validate and not stem.util.tor_tools.is_valid_fingerprint(value): + raise ValueError("Hidden service digests should consist of fourty hex digits: %s" % value) + + self.extra_info_digest = value + elif keyword == "hidden-service-dir": + if value: + self.hidden_service_dir = value.split(" ") + else: + self.hidden_service_dir = ["2"] elif keyword == "uptime": if not value.isdigit(): if not validate: continue @@ -470,6 +507,15 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): self.signature = block_contents elif keyword == "contact": self.contact = value + elif keyword == "protocols": + protocols_match = re.match("^Link (.*) Circuit (.*)$", value) + + if protocols_match: + link_versions, circuit_versions = protocols_match.groups() + self.link_protocols = link_versions.split(" ") + self.circuit_protocols = circuit_versions.split(" ") + elif validate: + raise ValueError("Protocols line did not match the expected pattern: %s" % line) elif keyword == "family": self.family = value.split(" ") else: diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 5a070a2..a956606 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -68,7 +68,13 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(588217, desc.uptime) self.assertEquals(expected_published, desc.published) self.assertEquals("www.atagar.com/contact", desc.contact) + self.assertEquals(["1", "2"], desc.link_protocols) + self.assertEquals(["1"], desc.circuit_protocols) self.assertEquals(False, desc.hibernating) + self.assertEquals(False, desc.allow_single_hop_exits) + self.assertEquals(False, desc.extra_info_cache) + self.assertEquals("D225B728768D7EA4B5587C13A7A9D22EBBEE6E66", desc.extra_info_digest) + self.assertEquals(["2"], desc.hidden_service_dir) self.assertEquals(expected_family, desc.family) self.assertEquals(153600, desc.average_bandwidth) self.assertEquals(256000, desc.burst_bandwidth)

1 0

[stem/master] Server descriptor test for metrics content
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit d72c21de710ba78e16b6d0fa29949fb43a2e5eb6 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Mar 24 18:15:27 2012 -0700 Server descriptor test for metrics content Test for parsing a single server descriptor and checking its content. This includes a few fixes and improvements for the ServerDescriptorV2 class, the most intersting of which is that declaring instance variables prior to init makes a single reference. Hence all server descriptors were using the same exit policy. Oops. :) --- run_tests.py | 2 + stem/descriptor/server_descriptor.py | 111 +++++++++++++++++----------- test/integ/descriptor/__init__.py | 2 +- test/integ/descriptor/server_descriptor.py | 83 +++++++++++++++++++++ 4 files changed, 155 insertions(+), 43 deletions(-) diff --git a/run_tests.py b/run_tests.py index e0574d3..3cb259d 100755 --- a/run_tests.py +++ b/run_tests.py @@ -31,6 +31,7 @@ import test.integ.control.base_controller import test.integ.socket.control_message import test.integ.socket.control_socket import test.integ.descriptor.reader +import test.integ.descriptor.server_descriptor import test.integ.util.conf import test.integ.util.system import test.integ.version @@ -98,6 +99,7 @@ INTEG_TESTS = ( test.integ.util.conf.TestConf, test.integ.util.system.TestSystem, test.integ.descriptor.reader.TestDescriptorReader, + test.integ.descriptor.server_descriptor.TestServerDescriptor, test.integ.version.TestVersion, test.integ.socket.control_socket.TestControlSocket, test.integ.socket.control_message.TestControlMessage, diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 70db1fe..7715a96 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -192,45 +192,33 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): Attributes: nickname (str) - relay's nickname (*) + fingerprint (str) - fourty hex digits that make up the relay's fingerprint address (str) - IPv4 address of the relay (*) or_port (int) - port used for relaying (*) socks_port (int) - deprecated attribute, always zero (*) dir_port (int) - deprecated port used for descriptor mirroring (*) - average_bandwidth (int) - rate of traffic relay is willing to relay in bytes/s (*) - burst_bandwidth (int) - rate of traffic relay is willing to burst to in bytes/s (*) - observed_bandwidth (int) - estimated capacity of the relay based on usage in bytes/s (*) platform (str) - operating system and tor version tor_version (stem.version.Version) - version of tor - exit_policy (stem.exit_policy.ExitPolicy) - relay's stated exit policy + operating_system (str) - relay's operating system + uptime (int) - relay's uptime when published in seconds published (datetime.datetime) - time in GMT when the descriptor was generated (*) - fingerprint (str) - fourty hex digits that make up the relay's fingerprint + contact (str) - relay's contact information hibernating (bool) - flag to indicate if the relay was hibernating when published (*) - uptime (int) - relay's uptime when published in seconds + exit_policy (stem.exit_policy.ExitPolicy) - relay's stated exit policy + family (list) - nicknames or fingerprints of relays it has a declared family with (*) + average_bandwidth (int) - rate of traffic relay is willing to relay in bytes/s (*) + burst_bandwidth (int) - rate of traffic relay is willing to burst to in bytes/s (*) + observed_bandwidth (int) - estimated capacity of the relay based on usage in bytes/s (*) onion_key (str) - key used to encrypt EXTEND cells (*) onion_key_type (str) - block type of the onion_key, probably "RSA PUBLIC KEY" (*) signing_key (str) - relay's long-term identity key (*) signing_key_type (str) - block type of the signing_key, probably "RSA PUBLIC KEY" (*) - router_sig (str) - signature for this descriptor (*) - router_sig_type (str) - block type of the router_sig, probably "SIGNATURE" (*) - contact (str) - relay's contact information - family (list) - nicknames or fingerprints of relays it has a declared family with (*) + signature (str) - signature for this descriptor (*) + signature_type (str) - block type of the signature, probably "SIGNATURE" (*) (*) required fields, others are left as None if undefined """ - nickname = address = or_port = socks_port = dir_port = None - average_bandwidth = burst_bandwidth = observed_bandwidth = None - platform = tor_version = published = fingerprint = uptime = None - onion_key = onion_key_type = signing_key = signing_key_type = None - router_sig = router_sig_type = contact = None - hibernating = False - family = unrecognized_lines = [] - - # TODO: Until we have a proper ExitPolicy class this is just a list of the - # exit policy strings... - - exit_policy = [] - def __init__(self, contents, validate = True, annotations = None): """ Version 2 server descriptor constructor, created from an individual relay's @@ -253,14 +241,49 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): stem.descriptor.Descriptor.__init__(self, contents) - self._annotation_lines = annotations - self._annotation_dict = {} + self.nickname = None + self.fingerprint = None + self.address = None + self.or_port = None + self.socks_port = None + self.dir_port = None + self.platform = None + self.tor_version = None + self.operating_system = None + self.uptime = None + self.published = None + self.contact = None + self.hibernating = False + self.family = [] + self.average_bandwidth = None + self.burst_bandwidth = None + self.observed_bandwidth = None + self.onion_key = None + self.onion_key_type = None + self.signing_key = None + self.signing_key_type = None + self.signature = None + self.signature_type = None + + # TODO: Until we have a proper ExitPolicy class this is just a list of the + # exit policy strings... - for line in annotations: - if " " in line: - key, value = line.split(" ", 1) - self._annotation_dict[key] = value - else: self._annotation_dict[line] = None + self.exit_policy = [] + + self._unrecognized_lines = [] + + if annotations: + self._annotation_lines = annotations + self._annotation_dict = {} + + for line in annotations: + if " " in line: + key, value = line.split(" ", 1) + self._annotation_dict[key] = value + else: self._annotation_dict[line] = None + else: + self._annotation_lines = [] + self._annotation_dict = {} # A descriptor contains a series of 'keyword lines' which are simply a # keyword followed by an optional value. Lines can also be followed by a @@ -271,7 +294,6 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): # does not matter so breaking it into key / value pairs. entries = {} - remaining_contents = contents.split("\n") while remaining_contents: line = remaining_contents.pop(0) @@ -343,12 +365,14 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): raise ValueError("Router line's SocksPort should be zero: %s" % router_comp[3]) elif not stem.util.connection.is_valid_port(router_comp[4], allow_zero = True): raise ValueError("Router line's DirPort is invalid: %s" % router_comp[4]) + elif not (router_comp[2].isdigit() and router_comp[3].isdigit() and router_comp[4].isdigit()): + continue self.nickname = router_comp[0] self.address = router_comp[1] - self.or_port = router_comp[2] - self.socks_port = router_comp[3] - self.dir_port = router_comp[4] + self.or_port = int(router_comp[2]) + self.socks_port = int(router_comp[3]) + self.dir_port = int(router_comp[4]) elif keyword == "bandwidth": # "bandwidth" bandwidth-avg bandwidth-burst bandwidth-observed bandwidth_comp = value.split() @@ -379,13 +403,16 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): # version followed by the os like the following... # platform Tor 0.2.2.35 (git-73ff13ab3cc9570d) on Linux x86_64 # - # There's no guerentee that we'll be able to pick out the version. + # There's no guerentee that we'll be able to pick these out the + # version, but might as well try to save our caller the effot. - platform_comp = self.platform.split() + platform_match = re.match("^Tor (\S*).* on (.*)$", self.platform) - if platform_comp[0] == "Tor" and len(platform_comp) >= 2: + if platform_match: + version_str, self.operating_system = platform_match.groups() + try: - self.tor_version = stem.version.Version(platform_comp[1]) + self.tor_version = stem.version.Version(version_str) except ValueError: pass elif keyword == "published": # "published" YYYY-MM-DD HH:MM:SS @@ -439,17 +466,17 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): if validate and (not block_type or not block_contents): raise ValueError("Router signature line must be followed by a signature block: %s" % line) - self.router_sig_type = block_type - self.router_sig = block_contents + self.signature_type = block_type + self.signature = block_contents elif keyword == "contact": self.contact = value elif keyword == "family": self.family = value.split(" ") else: - self.unrecognized_lines.append(line) + self._unrecognized_lines.append(line) def get_unrecognized_lines(self): - return list(unrecognized_lines) + return list(self._unrecognized_lines) def get_annotations(self): """ diff --git a/test/integ/descriptor/__init__.py b/test/integ/descriptor/__init__.py index e1095ca..b143c2a 100644 --- a/test/integ/descriptor/__init__.py +++ b/test/integ/descriptor/__init__.py @@ -2,5 +2,5 @@ Integration tests for stem.descriptor.* contents. """ -__all__ = ["reader"] +__all__ = ["reader", "server_descriptor"] diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py new file mode 100644 index 0000000..34b77a4 --- /dev/null +++ b/test/integ/descriptor/server_descriptor.py @@ -0,0 +1,83 @@ +""" +Integration tests for stem.descriptor.server_descriptor. +""" + +import os +import datetime +import unittest + +import stem.version +import stem.descriptor.server_descriptor + +my_dir = os.path.dirname(__file__) +DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") + +class TestServerDescriptor(unittest.TestCase): + def test_metrics_descriptor(self): + """ + Parses and checks our results against a server descriptor from metrics. + """ + + descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor") + + descriptor_file = open(descriptor_path) + descriptor_contents = descriptor_file.read() + descriptor_file.close() + + expected_published = datetime.datetime(2012, 3, 1, 17, 15, 27) + + expected_family = [ + "$0CE3CFB1E9CC47B63EA8869813BF6FAB7D4540C1", + "$1FD187E8F69A9B74C9202DC16A25B9E7744AB9F6", + "$74FB5EFA6A46DE4060431D515DC9A790E6AD9A7C", + "$77001D8DA9BF445B0F81AA427A675F570D222E6A", + "$B6D83EC2D9E18B0A7A33428F8CFA9C536769E209", + "$D2F37F46182C23AB747787FD657E680B34EAF892", + "$E0BD57A11F00041A9789577C53A1B784473669E4", + "$E5E3E9A472EAF7BE9682B86E92305DB4C71048EF", + ] + + expected_onion_key = """-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAJv5IIWQ+WDWYUdyA/0L8qbIkEVH/cwryZWoIaPAzINfrw1WfNZGtBmg +skFtXhOHHqTRN4GPPrZsAIUOQGzQtGb66IQgT4tO/pj+P6QmSCCdTfhvGfgTCsC+ +WPi4Fl2qryzTb3QO5r5x7T8OsG2IBUET1bLQzmtbC560SYR49IvVAgMBAAE= +-----END RSA PUBLIC KEY-----""" + + expected_signing_key = """-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAKwvOXyztVKnuYvpTKt+nS3XIKeO8dVungi8qGoeS+6gkR6lDtGfBTjd +uE9UIkdAl9zi8/1Ic2wsUNHE9jiS0VgeupITGZY8YOyMJJ/xtV1cqgiWhq1dUYaq +51TOtUogtAPgXPh4J+V8HbFFIcCzIh3qCO/xXo+DSHhv7SSif1VpAgMBAAE= +-----END RSA PUBLIC KEY-----""" + + expected_signature = """-----BEGIN SIGNATURE----- +dskLSPz8beUW7bzwDjR6EVNGpyoZde83Ejvau+5F2c6cGnlu91fiZN3suE88iE6e +758b9ldq5eh5mapb8vuuV3uO+0Xsud7IEOqfxdkmk0GKnUX8ouru7DSIUzUL0zqq +Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= +-----END SIGNATURE-----""" + + desc = stem.descriptor.server_descriptor.ServerDescriptorV2(descriptor_contents) + self.assertEquals("caerSidi", desc.nickname) + self.assertEquals("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", desc.fingerprint) + self.assertEquals("71.35.133.197", desc.address) + self.assertEquals(9001, desc.or_port) + self.assertEquals(0, desc.socks_port) + self.assertEquals(0, desc.dir_port) + self.assertEquals("Tor 0.2.1.30 on Linux x86_64", desc.platform) + self.assertEquals(stem.version.Version("0.2.1.30"), desc.tor_version) + self.assertEquals("Linux x86_64", desc.operating_system) + self.assertEquals(588217, desc.uptime) + self.assertEquals(expected_published, desc.published) + self.assertEquals("www.atagar.com/contact", desc.contact) + self.assertEquals(False, desc.hibernating) + self.assertEquals(expected_family, desc.family) + self.assertEquals(153600, desc.average_bandwidth) + self.assertEquals(256000, desc.burst_bandwidth) + self.assertEquals(104590, desc.observed_bandwidth) + self.assertEquals(["reject *:*"], desc.exit_policy) + self.assertEquals(expected_onion_key, desc.onion_key) + self.assertEquals("RSA PUBLIC KEY", desc.onion_key_type) + self.assertEquals(expected_signing_key, desc.signing_key) + self.assertEquals("RSA PUBLIC KEY", desc.signing_key_type) + self.assertEquals(expected_signature, desc.signature) + self.assertEquals("SIGNATURE", desc.signature_type) +

1 0

[stem/master] Allowing for extra args on lines with multiple
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit e0f8bfcacca963e1cbc7811dd1f9e6e7e104c7f8 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Mar 24 18:50:20 2012 -0700 Allowing for extra args on lines with multiple Well, crap. I've been developing against the version 2 spec on the assumption that was both the majority use case and what my test instance was running. However, that's evidently pretty ancient and I've been running against version 3 all along. First change I'm spotting in the spec is that lines with multiple arguments should ignore extras so doing that. --- stem/descriptor/__init__.py | 2 +- stem/descriptor/server_descriptor.py | 24 ++++++++++++------------ test/integ/descriptor/server_descriptor.py | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 215864f..325ac4b 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -41,7 +41,7 @@ def parse_file(path, descriptor_file): descriptor_file.seek(0) if filename == "cached-descriptors" or first_line.startswith("router "): - for desc in stem.descriptor.server_descriptor.parse_file_v2(descriptor_file): + for desc in stem.descriptor.server_descriptor.parse_file_v3(descriptor_file): desc._set_path(path) yield desc else: diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 7715a96..c301840 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -7,8 +7,8 @@ etc). This information is provided from a few sources... - the 'cached-descriptors' file in tor's data directory - tor metrics, at https://metrics.torproject.org/data.html -parse_file_v2 - Iterates over the server descriptors in a file. -ServerDescriptorV2 - Tor server descriptor, version 2. +parse_file_v3 - Iterates over the server descriptors in a file. +ServerDescriptorV3 - Tor server descriptor, version 3. |- get_unrecognized_lines - lines with unrecognized content |- get_annotations - dictionary of content prior to the descriptor entry |- get_annotation_lines - lines that provided the annotations @@ -55,9 +55,9 @@ SINGLE_FIELDS = ( "family", ) -def parse_file_v2(descriptor_file, validate = True): +def parse_file_v3(descriptor_file, validate = True): """ - Iterates over the version 2 server descriptors in a file. + Iterates over the version 3 server descriptors in a file. Arguments: descriptor_file (file) - file with descriptor content @@ -65,7 +65,7 @@ def parse_file_v2(descriptor_file, validate = True): True, skips these checks otherwise Returns: - iterator for ServerDescriptorV2 instances in the file + iterator for ServerDescriptorV3 instances in the file Raises: ValueError if the contents is malformed and validate is True @@ -109,7 +109,7 @@ def parse_file_v2(descriptor_file, validate = True): annotations = map(str.strip, annotations) descriptor_text = "".join(descriptor_content) - descriptor = ServerDescriptorV2(descriptor_text, validate, annotations) + descriptor = ServerDescriptorV3(descriptor_text, validate, annotations) yield descriptor else: break # done parsing descriptors @@ -185,10 +185,10 @@ def _get_psudo_pgp_block(remaining_contents): else: return (None, None) -class ServerDescriptorV2(stem.descriptor.Descriptor): +class ServerDescriptorV3(stem.descriptor.Descriptor): """ - Version 2 server descriptor, as specified in... - https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec-v2.txt + Version 3 server descriptor, as specified in... + https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt Attributes: nickname (str) - relay's nickname (*) @@ -221,7 +221,7 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): def __init__(self, contents, validate = True, annotations = None): """ - Version 2 server descriptor constructor, created from an individual relay's + Version 3 server descriptor constructor, created from an individual relay's descriptor content (as provided by "GETINFO desc/*", cached descriptors, and metrics). @@ -350,7 +350,7 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): # "router" nickname address ORPort SocksPort DirPort router_comp = value.split() - if len(router_comp) != 5: + if len(router_comp) < 5: if not validate: continue raise ValueError("Router line must have five values: %s" % line) @@ -377,7 +377,7 @@ class ServerDescriptorV2(stem.descriptor.Descriptor): # "bandwidth" bandwidth-avg bandwidth-burst bandwidth-observed bandwidth_comp = value.split() - if len(bandwidth_comp) != 3: + if len(bandwidth_comp) < 3: if not validate: continue raise ValueError("Bandwidth line must have three values: %s" % line) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 34b77a4..5a070a2 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -55,7 +55,7 @@ dskLSPz8beUW7bzwDjR6EVNGpyoZde83Ejvau+5F2c6cGnlu91fiZN3suE88iE6e Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= -----END SIGNATURE-----""" - desc = stem.descriptor.server_descriptor.ServerDescriptorV2(descriptor_contents) + desc = stem.descriptor.server_descriptor.ServerDescriptorV3(descriptor_contents) self.assertEquals("caerSidi", desc.nickname) self.assertEquals("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", desc.fingerprint) self.assertEquals("71.35.133.197", desc.address)

1 0

[stem/master] Dropping attribute for block types
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit d4fa1279f43c585d9551c6e1f94d3917e14eec99 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Mar 24 19:45:06 2012 -0700 Dropping attribute for block types We already have enough attributes, and parsing out the armor type is redundant since it's within the armor itself. Besides, it's not likely to change and I doubt callers will ever care about it. --- stem/descriptor/server_descriptor.py | 32 +++++++++------------------ test/integ/descriptor/server_descriptor.py | 3 -- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index ceee835..598571e 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -161,8 +161,7 @@ def _get_psudo_pgp_block(remaining_contents): remaining_contents (list) - lines to be checked for a public key block Returns: - (str, str) tuple with the block type and the armor wrapped contents, this - returns (None, None) instead if it doesn't exist + str with the armor wrapped contents or None if it doesn't exist Raises: ValueError if the contents starts with a key block but it's malformed (for @@ -170,7 +169,7 @@ def _get_psudo_pgp_block(remaining_contents): """ if not remaining_contents: - return (None, None) # nothing left + return None # nothing left block_match = PGP_BLOCK_START.match(remaining_contents[0]) @@ -186,9 +185,9 @@ def _get_psudo_pgp_block(remaining_contents): block_lines.append(line) if line == PGP_BLOCK_END % block_type: - return (block_type, "\n".join(block_lines)) + return "\n".join(block_lines) else: - return (None, None) + return None class ServerDescriptorV3(stem.descriptor.Descriptor): """ @@ -221,11 +220,8 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): burst_bandwidth (int) - rate of traffic relay is willing to burst to in bytes/s (*) observed_bandwidth (int) - estimated capacity of the relay based on usage in bytes/s (*) onion_key (str) - key used to encrypt EXTEND cells (*) - onion_key_type (str) - block type of the onion_key, probably "RSA PUBLIC KEY" (*) signing_key (str) - relay's long-term identity key (*) - signing_key_type (str) - block type of the signing_key, probably "RSA PUBLIC KEY" (*) signature (str) - signature for this descriptor (*) - signature_type (str) - block type of the signature, probably "SIGNATURE" (*) (*) required fields, others are left as None if undefined """ @@ -276,11 +272,8 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): self.burst_bandwidth = None self.observed_bandwidth = None self.onion_key = None - self.onion_key_type = None self.signing_key = None - self.signing_key_type = None self.signature = None - self.signature_type = None # TODO: Until we have a proper ExitPolicy class this is just a list of the # exit policy strings... @@ -333,7 +326,7 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): keyword, value = line_match.groups() try: - block_type, block_contents = _get_psudo_pgp_block(remaining_contents) + block_contents = _get_psudo_pgp_block(remaining_contents) except ValueError, exc: if not validate: continue raise exc @@ -341,9 +334,9 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): if keyword in ("accept", "reject"): self.exit_policy.append("%s %s" % (keyword, value)) elif keyword in entries: - entries[keyword].append((value, block_type, block_contents)) + entries[keyword].append((value, block_contents)) else: - entries[keyword] = [(value, block_type, block_contents)] + entries[keyword] = [(value, block_contents)] # validates restrictions about the entries if validate: @@ -361,7 +354,7 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): # parse all the entries into our attributes for keyword, values in entries.items(): # most just work with the first (and only) value - value, block_type, block_contents = values[0] + value, block_contents = values[0] line = "%s %s" % (keyword, value) # original line if block_contents: line += "\n%s" % block_contents @@ -488,22 +481,19 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): self.uptime = int(value) elif keyword == "onion-key": - if validate and (not block_type or not block_contents): + if validate and not block_contents: raise ValueError("Onion key line must be followed by a public key: %s" % line) - self.onion_key_type = block_type self.onion_key = block_contents elif keyword == "signing-key": - if validate and (not block_type or not block_contents): + if validate and not block_contents: raise ValueError("Signing key line must be followed by a public key: %s" % line) - self.signing_key_type = block_type self.signing_key = block_contents elif keyword == "router-signature": - if validate and (not block_type or not block_contents): + if validate and not block_contents: raise ValueError("Router signature line must be followed by a signature block: %s" % line) - self.signature_type = block_type self.signature = block_contents elif keyword == "contact": self.contact = value diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index a956606..d027dc6 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -81,9 +81,6 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(104590, desc.observed_bandwidth) self.assertEquals(["reject *:*"], desc.exit_policy) self.assertEquals(expected_onion_key, desc.onion_key) - self.assertEquals("RSA PUBLIC KEY", desc.onion_key_type) self.assertEquals(expected_signing_key, desc.signing_key) - self.assertEquals("RSA PUBLIC KEY", desc.signing_key_type) self.assertEquals(expected_signature, desc.signature) - self.assertEquals("SIGNATURE", desc.signature_type)

1 0

[stem/master] Including deprecated server descriptor attributes
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit b3d357b9f12d9bce72ec411f2681fdf84dcb4139 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Mar 25 12:15:15 2012 -0700 Including deprecated server descriptor attributes I have mixed feelings about cluttering the server descriptor with dead fields, but since we're already this close to being perfectly spec conformant we might as well include this last bit. Adding the fields but logging an info level warning if we see it, and checking in our cached descriptor that they do not exist. --- stem/descriptor/server_descriptor.py | 18 +++++++++++++++++- test/integ/descriptor/server_descriptor.py | 5 +++++ 2 files changed, 22 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 598571e..899cfcc 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -22,6 +22,7 @@ import stem.descriptor import stem.version import stem.util.connection import stem.util.tor_tools +import stem.util.log as log ENTRY_START = "router" ENTRY_END = "router-signature" @@ -199,7 +200,7 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): fingerprint (str) - fourty hex digits that make up the relay's fingerprint address (str) - IPv4 address of the relay (*) or_port (int) - port used for relaying (*) - socks_port (int) - deprecated attribute, always zero (*) + socks_port (int) - (deprecated) always zero (*) dir_port (int) - deprecated port used for descriptor mirroring (*) platform (str) - operating system and tor version tor_version (stem.version.Version) - version of tor @@ -219,6 +220,9 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): average_bandwidth (int) - rate of traffic relay is willing to relay in bytes/s (*) burst_bandwidth (int) - rate of traffic relay is willing to burst to in bytes/s (*) observed_bandwidth (int) - estimated capacity of the relay based on usage in bytes/s (*) + read_history (str) - (deprecated) always unset + write_history (str) - (deprecated) always unset + eventdns (bool) - (deprecated) always unset (*) onion_key (str) - key used to encrypt EXTEND cells (*) signing_key (str) - relay's long-term identity key (*) signature (str) - signature for this descriptor (*) @@ -271,6 +275,9 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): self.average_bandwidth = None self.burst_bandwidth = None self.observed_bandwidth = None + self.read_history = None + self.write_history = None + self.eventdns = True self.onion_key = None self.signing_key = None self.signature = None @@ -508,6 +515,15 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): raise ValueError("Protocols line did not match the expected pattern: %s" % line) elif keyword == "family": self.family = value.split(" ") + elif keyword == "read-history": + log.info("Read an unexpected 'read-history' line in a v3 server descriptor. These should only appear in extra-info. line: %s" % line) + self.read_history = value + elif keyword == "write-history": + log.info("Read an unexpected 'write-history' line in a v3 server descriptor. These should only appear in extra-info. line: %s" % line) + self.write_history = value + elif keyword == "eventdns": + log.info("Read an unexpected 'eventdns' line in a v3 server descriptor. These should be deprecated. line: %s" % line) + self.eventdns = value == "1" else: self._unrecognized_lines.append(line) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index a3a1e65..3782dea 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -109,6 +109,11 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= with open(descriptor_path) as descriptor_file: for desc in stem.descriptor.server_descriptor.parse_file_v3(descriptor_file): + # the following attributes should be deprecated, and not appear in the wild + self.assertEquals(None, desc.read_history) + self.assertEquals(None, desc.write_history) + self.assertEquals(True, desc.eventdns) + unrecognized_lines = desc.get_unrecognized_lines() if unrecognized_lines:

1 0

[stem/master] Integ test to validate our cached descriptor file
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 092d36a17a9318c42f09199e66458e399af68fda Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Mar 24 19:58:48 2012 -0700 Integ test to validate our cached descriptor file Parsing all of the descriptors in our cached descriptors file to see if it has anything that'll make us choke (and by extension indicate either a tor or stem bug). This is an important yet lengthy test, taking around four second. Targets do not effect this test so only running it once for the test run. --- test/integ/descriptor/server_descriptor.py | 37 ++++++++++++++++++++++++++++ 1 files changed, 37 insertions(+), 0 deletions(-) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index d027dc6..a3a1e65 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -8,10 +8,16 @@ import unittest import stem.version import stem.descriptor.server_descriptor +import test.runner my_dir = os.path.dirname(__file__) DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") +# 'test_cached_descriptor' is a lengthy test and uneffected by testing targets, +# so including a flag to prevent it from being ran multiple times + +RAN_CACHED_DESCRIPTOR_TEST = False + class TestServerDescriptor(unittest.TestCase): def test_metrics_descriptor(self): """ @@ -83,4 +89,35 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(expected_onion_key, desc.onion_key) self.assertEquals(expected_signing_key, desc.signing_key) self.assertEquals(expected_signature, desc.signature) + self.assertEquals([], desc.get_unrecognized_lines()) + + def test_cached_descriptor(self): + """ + Parses the cached descriptor file in our data directory, checking that it + doesn't raise any validation issues and looking for unrecognized descriptor + additions. + """ + + global RAN_CACHED_DESCRIPTOR_TEST + + if RAN_CACHED_DESCRIPTOR_TEST: + self.skipTest("(already ran)") + else: + RAN_CACHED_DESCRIPTOR_TEST = True + + descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-descriptors") + + with open(descriptor_path) as descriptor_file: + for desc in stem.descriptor.server_descriptor.parse_file_v3(descriptor_file): + unrecognized_lines = desc.get_unrecognized_lines() + + if unrecognized_lines: + # TODO: This isn't actually a problem, and rather than failing we + # should alert the user about these entries at the end of the tests + # (along with new events, getinfo options, and such). For now though + # there doesn't seem to be anything in practice to trigger this so + # failing to get our attention if it does. + + print "Unrecognized descriptor content: %s" % unrecognized_lines + self.fail()

1 0

[stem/master] Skipping integ test on cached descriptors if not available
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit e1f7537e939a60e7c902e853b03daab9c28b3d33 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Mar 25 12:20:43 2012 -0700 Skipping integ test on cached descriptors if not available The first time we run our integration tests our data directory is not yet fully populated. This causes our test that runs over the cached descriptor file to fail. Adding a check that the cached descriptors exist before running it. --- test/integ/descriptor/server_descriptor.py | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 3782dea..f950c4a 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -98,6 +98,13 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= additions. """ + descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-descriptors") + + # if this is our first time running the integ tests and we didn't wait for + # a full tor initialization then the cached descriptors won't exist yet + if not os.path.exists(descriptor_path): + self.skipTest("(no cached descriptors)") + global RAN_CACHED_DESCRIPTOR_TEST if RAN_CACHED_DESCRIPTOR_TEST: @@ -105,8 +112,6 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= else: RAN_CACHED_DESCRIPTOR_TEST = True - descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-descriptors") - with open(descriptor_path) as descriptor_file: for desc in stem.descriptor.server_descriptor.parse_file_v3(descriptor_file): # the following attributes should be deprecated, and not appear in the wild

1 0

[stem/master] Miswired 'ONLINE' and 'RELATIVE' targets
by atagar＠torproject.org 26 Mar '12

26 Mar '12

commit 1b507eae6bd104e9eba68ad94ff96baa7044b267 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Mar 25 12:31:40 2012 -0700 Miswired 'ONLINE' and 'RELATIVE' targets The test runner still used the 'test.' prefixed configuration keys rather than 'integ.'. This caused the 'ONLINE' and 'RELATIVE' testing targets to be ignored. --- test/runner.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/runner.py b/test/runner.py index 4b61e8a..9169bef 100644 --- a/test/runner.py +++ b/test/runner.py @@ -47,8 +47,8 @@ import test.output CONFIG = stem.util.conf.config_dict("test", { "integ.test_directory": "./test/data", "integ.log": "./test/data/log", - "test.target.online": False, - "test.target.relative_data_dir": False, + "integ.target.online": False, + "integ.target.relative_data_dir": False, }) STATUS_ATTR = (term.Color.BLUE, term.Attr.BOLD) @@ -179,7 +179,7 @@ class Runner: original_cwd, data_dir_path = os.getcwd(), self._test_dir - if CONFIG["test.target.relative_data_dir"]: + if CONFIG["integ.target.relative_data_dir"]: tor_cwd = os.path.dirname(self._test_dir) if not os.path.exists(tor_cwd): os.makedirs(tor_cwd) @@ -199,7 +199,7 @@ class Runner: self._start_tor(tor_cmd) # revert our cwd back to normal - if CONFIG["test.target.relative_data_dir"]: + if CONFIG["integ.target.relative_data_dir"]: os.chdir(original_cwd) except OSError, exc: self.stop() @@ -543,7 +543,7 @@ class Runner: try: # wait to fully complete if we're running tests with network activity, # otherwise finish after local bootstraping - complete_percent = 100 if CONFIG["test.target.online"] else 5 + complete_percent = 100 if CONFIG["integ.target.online"] else 5 # prints output from tor's stdout while it starts up print_init_line = lambda line: test.output.print_line(" %s" % line, *SUBSTATUS_ATTR)

1 0