- tor-commits - lists.torproject.org

[stem/master] Handler for bridge descriptor 'or-address' entries
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 19ba0506cec868e118ddfcf89cd30d44d3b43692 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Apr 11 20:59:56 2012 -0700 Handler for bridge descriptor 'or-address' entries Parsing 'or-address' entries in bridge descriptors. This parsing logic should be moved to the ServerDescriptorV3 class if/when ipv6 support is available to normal relays. --- stem/descriptor/server_descriptor.py | 43 ++++++++++++++++++++++++++++++++++ 1 files changed, 43 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 8548cf2..aedcb31 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -548,6 +548,7 @@ class RelayDescriptorV3(ServerDescriptorV3): # handles fields only in server descriptors for keyword, values in entries.items(): value, block_contents = values[0] + line = "%s %s" % (keyword, value) if keyword == "onion-key": if validate and not block_contents: @@ -586,9 +587,51 @@ class BridgeDescriptorV3(ServerDescriptorV3): """ Version 3 bridge descriptor, as specified in... https://metrics.torproject.org/formats.html#bridgedesc + + Attributes: + address_alt (list) - alternative for our address/or_port attributes, each + entry is a tuple of the form... + (address (str), port (int), is_ipv6 (bool)) """ + def __init__(self, raw_contents, validate = True, annotations = None): + self.address_alt = [] + ServerDescriptorV3.__init__(self, raw_contents, validate, annotations) + def _parse(self, entries, validate): + entries = dict(entries) + + # handles fields only in bridge descriptors + for keyword, values in entries.items(): + if keyword == "or-address": + or_address_entries = [value for (value, _) in values] + + for entry in or_address_entries: + line = "%s %s" % (keyword, entry) + + if not ":" in entry: + if not validate: continue + else: raise ValueError("or-address line missing a colon: %s" % line) + + div = entry.rfind(":") + address, ports = entry[:div], entry[div+1:] + is_ipv6 = address.startswith("[") and address.endswith("]") + if is_ipv6: address = address[1:-1] # remove brackets + + if not ((not is_ipv6 and stem.util.connection.is_valid_ip_address(address)) or + (is_ipv6 and stem.util.connection.is_valid_ipv6_address(address))): + if not validate: continue + else: raise ValueError("or-address line has a malformed address: %s" % line) + + for port in ports.split(","): + if not stem.util.connection.is_valid_port(port): + if not validate: break + else: raise ValueError("or-address line has malformed ports: %s" % line) + + self.address_alt.append((address, port, is_ipv6)) + + del entries["or-address"] + ServerDescriptorV3._parse(self, entries, validate) if validate: self._check_scrubbing()

1 0

[stem/master] Moving relay/bridge differentation into parse_file_v3()
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 785b22f274c30c2e52703c97bee3f2fb5a23e247 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Apr 12 09:36:49 2012 -0700 Moving relay/bridge differentation into parse_file_v3() It was confusing for parse_file_v3() to only work for relay server descriptors so moving the hack to differentiate relay from bridge descriptors there. This is also nice because it moves server descriptor logic out of the general __init__ module and into the one that's specifically for handling server descriptors. --- stem/descriptor/__init__.py | 10 +--------- stem/descriptor/server_descriptor.py | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 0e90546..21f40c2 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -40,15 +40,7 @@ def parse_file(path, descriptor_file): first_line = descriptor_file.readline() descriptor_file.seek(0) - if first_line.startswith("router Unnamed 10."): - # bridge descriptors are scrubbed so their nickname is 'Unnamed' and their - # ip address is in the 10.x.x.x space, which is normally reserved for - # private networks - - desc = stem.descriptor.server_descriptor.BridgeDescriptorV3(descriptor_file.read()) - desc._set_path(path) - yield desc - elif filename == "cached-descriptors" or first_line.startswith("router "): + if filename == "cached-descriptors" or first_line.startswith("router "): for desc in stem.descriptor.server_descriptor.parse_file_v3(descriptor_file): desc._set_path(path) yield desc diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index aedcb31..52bca94 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -64,7 +64,8 @@ SINGLE_FIELDS = ( def parse_file_v3(descriptor_file, validate = True): """ - Iterates over the version 3 server descriptors in a file. + Iterates over the version 3 server descriptors in a file. This can read + either relay or bridge v3 server descriptors. Arguments: descriptor_file (file) - file with descriptor content @@ -72,13 +73,29 @@ def parse_file_v3(descriptor_file, validate = True): True, skips these checks otherwise Returns: - iterator for RelayDescriptorV3 instances in the file + iterator for ServerDescriptorV3 instances in the file Raises: ValueError if the contents is malformed and validate is True IOError if the file can't be read """ + # Handler for bridge descriptors + # + # Bridge descriptors are scrubbed so their nickname is 'Unnamed' and their + # ip address is in the 10.x.x.x space, which is normally reserved for + # private networks. Bride descriptors only come from metrics so a file only + # contains a single descriptor. + + first_line = descriptor_file.readline() + descriptor_file.seek(0) + + if first_line.startswith("router Unnamed 10."): + yield BridgeDescriptorV3(descriptor_file.read()) + return + + # Handler for relay descriptors + # # Cached descriptors consist of annotations followed by the descriptor # itself. For instance... #

1 0

[stem/master] Utility for validating IPv6 addresses
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit dbd6a80ffc65f8d77d6d798b49f8c899e5ef09e9 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Apr 11 20:50:08 2012 -0700 Utility for validating IPv6 addresses Function and testing to check a string is a valid ipv6 address. --- stem/util/connection.py | 40 ++++++++++++++++++++++++++++++++++++---- test/unit/util/connection.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/stem/util/connection.py b/stem/util/connection.py index 9c6ba8b..93030fc 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -6,22 +6,24 @@ https://gitweb.torproject.org/arm.git/blob/HEAD:/src/util/connections.py but for now just moving the parts we need. """ -def is_valid_ip_address(entry): +import re + +def is_valid_ip_address(address): """ Checks if a string is a valid IPv4 address. Arguments: - entry (str) - string to be checked + address (str) - string to be checked Returns: True if input is a valid IPv4 address, False otherwise. """ # checks if theres four period separated values - if not entry.count(".") == 3: return False + if address.count(".") != 3: return False # checks that each value in the octet are decimal values between 0-255 - for entry in entry.split("."): + for entry in address.split("."): if not entry.isdigit() or int(entry) < 0 or int(entry) > 255: return False elif entry[0] == "0" and len(entry) > 1: @@ -29,6 +31,36 @@ def is_valid_ip_address(entry): return True +def is_valid_ipv6_address(address): + """ + Checks if a string is a valid IPv6 address. + + Arguments: + address (str) - string to be checked + + Returns: + True if input is a valid IPv6 address, False otherwise. + """ + + # addresses are made up of eight colon separated groups of four hex digits + # with leading zeros being optional + # https://en.wikipedia.org/wiki/IPv6#Address_format + + colon_count = address.count(":") + + if colon_count > 7: + return False # too many groups + elif colon_count != 7 and not "::" in address: + return False # not enough groups and none are collapsed + elif address.count("::") > 1 or ":::" in address: + return False # multiple groupings of zeros can't be collapsed + + for entry in address.split(":"): + if not re.match("^[0-9a-fA-f]{0,4}$", entry): + return False + + return True + def is_valid_port(entry, allow_zero = False): """ Checks if a string or int is a valid port number. diff --git a/test/unit/util/connection.py b/test/unit/util/connection.py index 01d1c92..b4c7223 100644 --- a/test/unit/util/connection.py +++ b/test/unit/util/connection.py @@ -29,9 +29,37 @@ class TestConnection(unittest.TestCase): for address in valid_addresses: self.assertTrue(stem.util.connection.is_valid_ip_address(address)) + for address in invalid_addresses: self.assertFalse(stem.util.connection.is_valid_ip_address(address)) + def test_is_valid_ipv6_address(self): + """ + Checks the is_valid_ipv6_address function. + """ + + valid_addresses = ( + "fe80:0000:0000:0000:0202:b3ff:fe1e:8329", + "fe80:0:0:0:202:b3ff:fe1e:8329", + "fe80::202:b3ff:fe1e:8329", + "::", + ) + + invalid_addresses = ( + "fe80:0000:0000:0000:0202:b3ff:fe1e:829g", + "fe80:0000:0000:0000:0202:b3ff:fe1e: 8329", + "2001:db8::aaaa::1", + ":::", + ":", + "", + ) + + for address in valid_addresses: + self.assertTrue(stem.util.connection.is_valid_ipv6_address(address)) + + for address in invalid_addresses: + self.assertFalse(stem.util.connection.is_valid_ipv6_address(address)) + def test_is_valid_port(self): """ Checks the is_valid_port function.

1 0

[stem/master] Unit tests for bridge descriptors and related fixes
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 850b1cbbc07554ea369b44c89b3bbfdd1b99fa69 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Apr 12 10:53:28 2012 -0700 Unit tests for bridge descriptors and related fixes Unit testing the new bridge descriptors. This mostly focused on parsing or-address entries and the validation that the class does for scrubbed values. Fixes included having string port values and not checking that or-address addresses were scrubbed. --- stem/descriptor/server_descriptor.py | 10 ++- test/unit/descriptor/server_descriptor.py | 121 ++++++++++++++++++++++++++-- 2 files changed, 121 insertions(+), 10 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 52bca94..dfdde30 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -645,7 +645,7 @@ class BridgeDescriptorV3(ServerDescriptorV3): if not validate: break else: raise ValueError("or-address line has malformed ports: %s" % line) - self.address_alt.append((address, port, is_ipv6)) + self.address_alt.append((address, int(port), is_ipv6)) del entries["or-address"] @@ -665,6 +665,14 @@ class BridgeDescriptorV3(ServerDescriptorV3): elif self.contact and self.contact != "somebody": raise ValueError("Contact line should be scrubbed to be 'somebody', but instead had '%s'" % self.contact) + for address, _, is_ipv6 in self.address_alt: + if not is_ipv6 and not address.startswith("10."): + raise ValueError("or-address line's address should be scrubbed to be '10.x.x.x': %s" % address) + elif is_ipv6 and not address.startswith("fd9f:2e19:3bcf::"): + # TODO: this check isn't quite right because we aren't checking that + # the next grouping of hex digits contains 1-2 digits + raise ValueError("or-address line's address should be scrubbed to be 'fd9f:2e19:3bcf::xx:xxxx': %s" % address) + for line in self.get_unrecognized_lines(): if line.startswith("onion-key "): raise ValueError("Bridge descriptors should have their onion-key scrubbed: %s" % line) diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 42325ff..89a28c6 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -7,7 +7,7 @@ import StringIO import unittest import stem.descriptor.server_descriptor -from stem.descriptor.server_descriptor import RelayDescriptorV3 +from stem.descriptor.server_descriptor import RelayDescriptorV3, BridgeDescriptorV3 CRYPTO_BLOB = """ MIGJAoGBAJv5IIWQ+WDWYUdyA/0L8qbIkEVH/cwryZWoIaPAzINfrw1WfNZGtBmg @@ -15,7 +15,7 @@ skFtXhOHHqTRN4GPPrZsAIUOQGzQtGb66IQgT4tO/pj+P6QmSCCdTfhvGfgTCsC+ WPi4Fl2qryzTb3QO5r5x7T8OsG2IBUET1bLQzmtbC560SYR49IvVAgMBAAE= """ -SAMPLE_DESCRIPTOR_ATTR = ( +RELAY_DESCRIPTOR_ATTR = ( ("router", "caerSidi 71.35.133.197 9001 0 0"), ("published", "2012-03-01 17:15:27"), ("bandwidth", "153600 256000 104590"), @@ -25,13 +25,21 @@ SAMPLE_DESCRIPTOR_ATTR = ( ("router-signature", "\n-----BEGIN SIGNATURE-----%s-----END SIGNATURE-----" % CRYPTO_BLOB), ) -def _make_descriptor(attr = None, exclude = None): +BRIDGE_DESCRIPTOR_ATTR = ( + ("router", "Unnamed 10.45.227.253 9001 0 0"), + ("published", "2012-03-22 17:34:38"), + ("bandwidth", "409600 819200 5120"), + ("reject", "*:*"), +) + +def _make_descriptor(attr = None, exclude = None, is_bridge = False): """ Constructs a minimal server descriptor with the given attributes. Arguments: - attr (dict) - keyword/value mappings to be included in the descriptor - exclude (list) - mandatory keywords to exclude from the descriptor + attr (dict) - keyword/value mappings to be included in the descriptor + exclude (list) - mandatory keywords to exclude from the descriptor + is_bridge (bool) - minimal descriptor is for a bridge if True, relay otherwise Returns: str with customized descriptor content @@ -40,26 +48,35 @@ def _make_descriptor(attr = None, exclude = None): descriptor_lines = [] if attr == None: attr = {} if exclude == None: exclude = [] + desc_attr = BRIDGE_DESCRIPTOR_ATTR if is_bridge else RELAY_DESCRIPTOR_ATTR + attr = dict(attr) # shallow copy since we're destructive - for keyword, value in SAMPLE_DESCRIPTOR_ATTR: + for keyword, value in desc_attr: if keyword in exclude: continue elif keyword in attr: value = attr[keyword] del attr[keyword] # if this is the last entry then we should dump in any unused attributes - if keyword == "router-signature": + if not is_bridge and keyword == "router-signature": for attr_keyword, attr_value in attr.items(): descriptor_lines.append("%s %s" % (attr_keyword, attr_value)) descriptor_lines.append("%s %s" % (keyword, value)) + # bridges don't have a router-signature so simply append any extra attributes + # to the end + if is_bridge: + for attr_keyword, attr_value in attr.items(): + descriptor_lines.append("%s %s" % (attr_keyword, attr_value)) + return "\n".join(descriptor_lines) class TestServerDescriptor(unittest.TestCase): - def test_minimal_descriptor(self): + def test_minimal_relay_descriptor(self): """ - Basic sanity check that we can parse a descriptor with minimal attributes. + Basic sanity check that we can parse a relay server descriptor with minimal + attributes. """ desc_text = _make_descriptor() @@ -255,6 +272,92 @@ class TestServerDescriptor(unittest.TestCase): self.assertEquals(None, desc.socks_port) self.assertEquals(None, desc.dir_port) + def test_minimal_bridge_descriptor(self): + """ + Basic sanity check that we can parse a descriptor with minimal attributes. + """ + + desc_text = _make_descriptor(is_bridge = True) + desc = BridgeDescriptorV3(desc_text) + + self.assertEquals("Unnamed", desc.nickname) + self.assertEquals("10.45.227.253", desc.address) + self.assertEquals(None, desc.fingerprint) + + # check that we don't have crypto fields + self.assertRaises(AttributeError, getattr, desc, "onion_key") + self.assertRaises(AttributeError, getattr, desc, "signing_key") + self.assertRaises(AttributeError, getattr, desc, "signature") + + def test_bridge_unsanitized(self): + """ + Targeted check that individual unsanitized attributes will be detected. + """ + + unsanitized_attr = [ + {"router": "caerSidi 10.45.227.253 9001 0 0"}, + {"router": "Unnamed 75.45.227.253 9001 0 0"}, + {"contact": "Damian"}, + {"or-address": "71.35.133.197:9001"}, + {"or-address": "[12ab:2e19:3bcf::02:9970]:9001"}, + {"onion-key": "\n-----BEGIN RSA PUBLIC KEY-----%s-----END RSA PUBLIC KEY-----" % CRYPTO_BLOB}, + {"signing-key": "\n-----BEGIN RSA PUBLIC KEY-----%s-----END RSA PUBLIC KEY-----" % CRYPTO_BLOB}, + {"router-signature": "\n-----BEGIN SIGNATURE-----%s-----END SIGNATURE-----" % CRYPTO_BLOB}, + ] + + for attr in unsanitized_attr: + try: + desc_text = _make_descriptor(attr, is_bridge = True) + BridgeDescriptorV3(desc_text) + self.fail("Unsanitized attribute wasn't detected: %s %s" % attr.items()[0]) + except ValueError: pass + + def test_bridge_unsanitized_relay(self): + """ + Checks that parsing a normal relay descriptor as a bridge will fail due to + its unsanatized content. + """ + + desc_text = _make_descriptor() + self.assertRaises(ValueError, BridgeDescriptorV3, desc_text) + + def test_or_address_v4(self): + """ + Constructs a bridge descriptor with a sanatized IPv4 or-address entry. + """ + + desc_text = _make_descriptor({"or-address": "10.45.227.253:9001"}, is_bridge = True) + desc = BridgeDescriptorV3(desc_text) + self.assertEquals([("10.45.227.253", 9001, False)], desc.address_alt) + + def test_or_address_v6(self): + """ + Constructs a bridge descriptor with a sanatized IPv6 or-address entry. + """ + + desc_text = _make_descriptor({"or-address": "[fd9f:2e19:3bcf::02:9970]:9001"}, is_bridge = True) + desc = BridgeDescriptorV3(desc_text) + self.assertEquals([("fd9f:2e19:3bcf::02:9970", 9001, True)], desc.address_alt) + + def test_or_address_multiple(self): + """ + Constructs a bridge descriptor with multiple or-address entries and multiple ports. + """ + + desc_text = "\n".join((_make_descriptor(is_bridge = True), + "or-address 10.45.227.253:9001,9005,80", + "or-address [fd9f:2e19:3bcf::02:9970]:443")) + + expected_address_alt = [ + ("10.45.227.253", 9001, False), + ("10.45.227.253", 9005, False), + ("10.45.227.253", 80, False), + ("fd9f:2e19:3bcf::02:9970", 443, True), + ] + + desc = BridgeDescriptorV3(desc_text) + self.assertEquals(expected_address_alt, desc.address_alt) + def _expect_invalid_attr(self, desc_text, attr = None, expected_value = None): """ Asserts that construction will fail due to desc_text having a malformed

1 0

[stem/master] Removing deprecation warnings for server descriptor fields
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 0d478c57866c868c63be50714887c1b1535fbd0f Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 13 20:11:37 2012 -0700 Removing deprecation warnings for server descriptor fields The read/write-history and eventdns server descriptor fields are part of the spec but no longer used. I was logging a notice that they're deprecated when we encountered them, but this is neither useful nor does it make sense. We might be parsing old metrics archives where those attributes are perfectly valid. Integ tests include a check that those aren't seen in the wild, which is what I really care about. --- stem/descriptor/server_descriptor.py | 4 ---- 1 files changed, 0 insertions(+), 4 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index dfdde30..0a6cd29 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -26,7 +26,6 @@ import stem.descriptor import stem.version import stem.util.connection import stem.util.tor_tools -import stem.util.log as log KEYWORD_CHAR = "a-zA-Z0-9-" WHITESPACE = " \t" @@ -473,13 +472,10 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): elif keyword == "family": self.family = value.split(" ") elif keyword == "read-history": - log.info("Read an unexpected 'read-history' line in a v3 server descriptor. These should only appear in extra-info. line: %s" % line) self.read_history = value elif keyword == "write-history": - log.info("Read an unexpected 'write-history' line in a v3 server descriptor. These should only appear in extra-info. line: %s" % line) self.write_history = value elif keyword == "eventdns": - log.info("Read an unexpected 'eventdns' line in a v3 server descriptor. These should be deprecated. line: %s" % line) self.eventdns = value == "1" else: self._unrecognized_lines.append(line)

1 0

[stem/master] Renaming calculate_digest() to just digest()
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit ed0e8ac4d0dca0c58cc17e2314839351590038b8 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 13 22:39:33 2012 -0700 Renaming calculate_digest() to just digest() --- stem/descriptor/server_descriptor.py | 5 +++-- test/integ/descriptor/server_descriptor.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 9abe06b..e91cab8 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -10,7 +10,8 @@ etc). This information is provided from a few sources... parse_file_v3 - Iterates over the server descriptors in a file. ServerDescriptorV3 - Tor server descriptor, version 3. | |- RelayDescriptorV3 - Server descriptor for a relay. - | | +- is_valid - checks the signature against the descriptor content + | | |- is_valid - checks the signature against the descriptor content + | | +- digest - calculates the digest value for our content | | | +- BridgeDescriptorV3 - Scrubbed server descriptor for a bridge. | @@ -558,7 +559,7 @@ class RelayDescriptorV3(ServerDescriptorV3): raise NotImplementedError # TODO: implement - def calculate_digest(self): + def digest(self): """ Provides the base64 encoded sha1 of our content. This value is part of the server descriptor entry for this relay. diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index c4265d7..b7577da 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -229,7 +229,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(["reject *:*"], desc.exit_policy) self.assertEquals([], desc.get_unrecognized_lines()) - def test_calculate_digest(self): + def test_digest(self): """ Checks that the digest for a descriptor matches its consensus digest value. """ @@ -267,5 +267,5 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= desc_content = desc_content[desc_content.find("=\n") + 2:] desc = stem.descriptor.server_descriptor.RelayDescriptorV3(desc_content) - self.assertEquals(consensus_digest, desc.calculate_digest()) + self.assertEquals(consensus_digest, desc.digest())

1 0

[stem/master] Removing timeout from descriptor reader enqueue calls
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit b56bb55187d6baa00510b98c6ae1c91739476acf Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 13 20:36:33 2012 -0700 Removing timeout from descriptor reader enqueue calls The reader had a timeout for enqueue operations so it could periodically check if we were stopped while waiting to insert a descriptor that we've read. As Karsten thought we were able to do better. Dropped the timeout by clearing our queue on shutdown (to unblock any enqueue calls in process), and skip further enqueuing while we're shutting down. Both iterating and stopping are under a read lock so we don't need to worry about this changing the order in which descriptors are provided to callers. --- stem/descriptor/reader.py | 28 +++++++++++++++------------- 1 files changed, 15 insertions(+), 13 deletions(-) diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index bbedefc..66ed65a 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -302,6 +302,13 @@ class DescriptorReader: with self._reader_thread_lock: self._is_stopped.set() self._iter_notice.set() + + # clears our queue to unblock enqueue calls + try: + while True: + self._unreturned_descriptors.get_nowait() + except Queue.Empty: pass + self._reader_thread.join() self._reader_thread = None @@ -352,7 +359,10 @@ class DescriptorReader: self._notify_skip_listeners(target, UnrecognizedType(target_type)) self._processed_files = new_processed_files - self._enqueue_descriptor(FINISHED) + + if not self._is_stopped.is_set(): + self._unreturned_descriptors.put(FINISHED) + self._iter_notice.set() def __iter__(self): @@ -371,7 +381,8 @@ class DescriptorReader: try: with open(target) as target_file: for desc in stem.descriptor.parse_file(target, target_file): - self._enqueue_descriptor(desc) + if self._is_stopped.is_set(): return + self._unreturned_descriptors.put(desc) self._iter_notice.set() except TypeError, exc: self._notify_skip_listeners(target, UnrecognizedType(None)) @@ -388,7 +399,8 @@ class DescriptorReader: entry = tar_file.extractfile(tar_entry) for desc in stem.descriptor.parse_file(target, entry): - self._enqueue_descriptor(desc) + if self._is_stopped.is_set(): return + self._unreturned_descriptors.put(desc) self._iter_notice.set() entry.close() @@ -397,16 +409,6 @@ class DescriptorReader: except IOError, exc: self._notify_skip_listeners(target, ReadFailed(exc)) - def _enqueue_descriptor(self, descriptor): - # blocks until there is either room for the descriptor or we're stopped - - while True: - try: - self._unreturned_descriptors.put(descriptor, timeout = 0.1) - return - except Queue.Full: - if self._is_stopped.is_set(): return - def _notify_skip_listeners(self, path, exception): for listener in self._skip_listeners: listener(path, exception)

1 0

[stem/master] Implementing calculate_digest() for server descriptors
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 8c1b5b6053e08bce55b9bc935daa492f727c68e8 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 13 22:19:33 2012 -0700 Implementing calculate_digest() for server descriptors Implementation and test for a function to get the digest value for a relay server descriptor. This is the same value found in the network status entry. This was an addition suggested by Karsten. --- stem/descriptor/server_descriptor.py | 28 ++++++++++++++ test/integ/descriptor/server_descriptor.py | 57 ++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 0a6cd29..9abe06b 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -20,6 +20,8 @@ ServerDescriptorV3 - Tor server descriptor, version 3. """ import re +import base64 +import hashlib import datetime import stem.descriptor @@ -542,6 +544,7 @@ class RelayDescriptorV3(ServerDescriptorV3): self.onion_key = None self.signing_key = None self.signature = None + self._digest = None ServerDescriptorV3.__init__(self, raw_contents, validate, annotations) @@ -555,6 +558,31 @@ class RelayDescriptorV3(ServerDescriptorV3): raise NotImplementedError # TODO: implement + def calculate_digest(self): + """ + Provides the base64 encoded sha1 of our content. This value is part of the + server descriptor entry for this relay. + + Returns: + str with the digest value for this server descriptor + """ + + if self._digest == None: + # our digest is calculated from everything except our signature + raw_content, ending = str(self), "\nrouter-signature\n" + raw_content = raw_content[:raw_content.find(ending) + len(ending)] + + digest_sha1 = hashlib.sha1(raw_content).digest() + digest = base64.b64encode(digest_sha1) + + # TODO: I'm not sure why but the base64 decodings have an anomalous '=' + # ending which the network status entries don't have. Tad puzzled, but + # for now stripping it so we match. + + self._digest = digest[:-1] + + return self._digest + def _parse(self, entries, validate): entries = dict(entries) # shallow copy since we're destructive diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 19b2a6d..c4265d7 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -6,6 +6,7 @@ import os import datetime import unittest +import stem.control import stem.version import stem.descriptor.server_descriptor import test.runner @@ -19,6 +20,18 @@ DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") RAN_CACHED_DESCRIPTOR_TEST = False class TestServerDescriptor(unittest.TestCase): + is_descriptors_available = None + + def setUp(self): + # If this is our first time running the integ tests and we didn't wait for + # a full tor initialization then the cached descriptors won't exist yet. + # Noting if they exist or not since some tests need them. + + if self.is_descriptors_available == None: + test_dir = test.runner.get_runner().get_test_dir() + descriptor_path = os.path.join(test_dir, "cached-descriptors") + self.is_descriptors_available = os.path.exists(descriptor_path) + def test_metrics_descriptor(self): """ Parses and checks our results against a server descriptor from metrics. @@ -100,9 +113,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-descriptors") - # if this is our first time running the integ tests and we didn't wait for - # a full tor initialization then the cached descriptors won't exist yet - if not os.path.exists(descriptor_path): + if not self.is_descriptors_available: self.skipTest("(no cached descriptors)") global RAN_CACHED_DESCRIPTOR_TEST @@ -217,4 +228,44 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(5120, desc.observed_bandwidth) self.assertEquals(["reject *:*"], desc.exit_policy) self.assertEquals([], desc.get_unrecognized_lines()) + + def test_calculate_digest(self): + """ + Checks that the digest for a descriptor matches its consensus digest value. + """ + + # TODO: Remove manual parsing with proper objects when we have them... + # - parsing of consensus_digest with the NetworkStatus class + # - low level msg() calls with Controller + + if not self.is_descriptors_available: + self.skipTest("(no cached descriptors)") + + with test.runner.get_runner().get_tor_socket() as control_socket: + controller = stem.control.BaseController(control_socket) + + # picking one of the directory authorities (gabelmoo) since they're + # pretty stable and this is trivial to revise if they change + + fingerprint = "F2044413DAC2E02E3D6BCF4735A19BCA1DE97281" + desc_entry = controller.msg("GETINFO desc/id/%s" % fingerprint) + ns_entry = controller.msg("GETINFO ns/id/%s" % fingerprint) + + # parse the consensus digest from the ns_entry + consensus_digest = None + for line in str(ns_entry).split("\n"): + if line.startswith("r "): + consensus_digest = line.split()[3] + break + + if not consensus_digest: + self.fail("Malformed network descriptor: %s" % ns_entry) + + # parse the descriptor content from the desc_entry + + desc_content = list(desc_entry)[0] + desc_content = desc_content[desc_content.find("=\n") + 2:] + + desc = stem.descriptor.server_descriptor.RelayDescriptorV3(desc_content) + self.assertEquals(consensus_digest, desc.calculate_digest())

1 0

[stem/master] Removing restriction that descriptor socks port is zero
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit c64f663c6f3215eb4e492cfe60bf7be46f50a1d0 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 13 22:53:36 2012 -0700 Removing restriction that descriptor socks port is zero The socks port of server descriptors is depricated and always zero, but this isn't necessarily true for archives. --- stem/descriptor/server_descriptor.py | 4 ++-- test/integ/descriptor/server_descriptor.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index e91cab8..9b74e2b 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -355,8 +355,8 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): raise ValueError("Router line entry isn't a valid IPv4 address: %s" % router_comp[1]) elif not stem.util.connection.is_valid_port(router_comp[2], allow_zero = True): raise ValueError("Router line's ORPort is invalid: %s" % router_comp[2]) - elif router_comp[3] != "0": - raise ValueError("Router line's SocksPort should be zero: %s" % router_comp[3]) + elif not stem.util.connection.is_valid_port(router_comp[3], allow_zero = True): + raise ValueError("Router line's SocksPort is invalid: %s" % router_comp[3]) elif not stem.util.connection.is_valid_port(router_comp[4], allow_zero = True): raise ValueError("Router line's DirPort is invalid: %s" % router_comp[4]) elif not (router_comp[2].isdigit() and router_comp[3].isdigit() and router_comp[4].isdigit()): diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index b7577da..24d9ee5 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -129,6 +129,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(None, desc.read_history) self.assertEquals(None, desc.write_history) self.assertEquals(True, desc.eventdns) + self.assertEquals(0, desc.socks_port) unrecognized_lines = desc.get_unrecognized_lines()

1 0

[stem/master] Removing integ test for descriptor digests
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit b8f6fd2c1a601676be30dd2850211f8fedfb3c14 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 13 22:57:33 2012 -0700 Removing integ test for descriptor digests Dropping the server descriptor integ test for digest(). It was a valiant effort, but is flaky because it'll always fail when the consensus or descriptor changes and are out of sync. --- test/integ/descriptor/server_descriptor.py | 41 +--------------------------- 1 files changed, 1 insertions(+), 40 deletions(-) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 24d9ee5..f1af13c 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -103,6 +103,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(expected_signing_key, desc.signing_key) self.assertEquals(expected_signature, desc.signature) self.assertEquals([], desc.get_unrecognized_lines()) + self.assertEquals("LHsnvqsEtOJFnYnKbVzRzF+Vpok", desc.digest()) def test_cached_descriptor(self): """ @@ -229,44 +230,4 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(5120, desc.observed_bandwidth) self.assertEquals(["reject *:*"], desc.exit_policy) self.assertEquals([], desc.get_unrecognized_lines()) - - def test_digest(self): - """ - Checks that the digest for a descriptor matches its consensus digest value. - """ - - # TODO: Remove manual parsing with proper objects when we have them... - # - parsing of consensus_digest with the NetworkStatus class - # - low level msg() calls with Controller - - if not self.is_descriptors_available: - self.skipTest("(no cached descriptors)") - - with test.runner.get_runner().get_tor_socket() as control_socket: - controller = stem.control.BaseController(control_socket) - - # picking one of the directory authorities (gabelmoo) since they're - # pretty stable and this is trivial to revise if they change - - fingerprint = "F2044413DAC2E02E3D6BCF4735A19BCA1DE97281" - desc_entry = controller.msg("GETINFO desc/id/%s" % fingerprint) - ns_entry = controller.msg("GETINFO ns/id/%s" % fingerprint) - - # parse the consensus digest from the ns_entry - consensus_digest = None - for line in str(ns_entry).split("\n"): - if line.startswith("r "): - consensus_digest = line.split()[3] - break - - if not consensus_digest: - self.fail("Malformed network descriptor: %s" % ns_entry) - - # parse the descriptor content from the desc_entry - - desc_content = list(desc_entry)[0] - desc_content = desc_content[desc_content.find("=\n") + 2:] - - desc = stem.descriptor.server_descriptor.RelayDescriptorV3(desc_content) - self.assertEquals(consensus_digest, desc.digest())

1 0