commit 1f868090e2d641ddcb49d02bd15b5894f5bf6923 Author: Damian Johnson atagar@torproject.org Date: Tue Sep 18 09:36:56 2012 -0700

Parsing the directory-signature and unrecognized lines

Finishing up with the footer. It doesn't make sense for the DirectorySignature or DirectoryAuthority to be Descriptor subclasses (cuz... well, they aren't descriptors). However, I like having this struct class rather than providing our callers with a tuple list. I should probably do this for other descriptor documents too... --- stem/descriptor/networkstatus.py | 111 ++++++++++-------------- test/integ/descriptor/networkstatus.py | 16 ++-- test/unit/descriptor/networkstatus/document.py | 59 ++++++++++--- 3 files changed, 98 insertions(+), 88 deletions(-)

diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index e142728..a7bca7a 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -36,7 +36,7 @@ The documents can be obtained from any of the following sources... +- MicrodescriptorConsensus - Microdescriptor flavoured consensus documents RouterStatusEntry - Router descriptor; contains information about a Tor relay +- RouterMicrodescriptor - Router microdescriptor; contains information that doesn't change frequently - DirectorySignature - Network status document's directory signature + DocumentSignature - Signature of a document by a directory authority DirectoryAuthority - Directory authority defined in a v3 network status document """

@@ -216,7 +216,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :var list known_flags: ***** list of known router flags :var list params: ***** dict of parameter(str) => value(int) mappings :var list directory_authorities: ***** list of DirectoryAuthority objects that have generated this document - :var list directory_signatures: ***** list of signatures this document has + :var list signatures: ***** DocumentSignature of the authorities that have signed the document

**Consensus Attributes:** :var int consensus_method: method version used to generate this consensus @@ -243,7 +243,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): super(NetworkStatusDocument, self).__init__(raw_content)

self.directory_authorities = [] - self.directory_signatures = [] + self.signatures = []

self.version = None self.is_consensus = True @@ -262,6 +262,8 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): self.params = dict(DEFAULT_PARAMS) if default_params else {} self.bandwidth_weights = {}

+ self._unrecognized_lines = [] + document_file = StringIO(raw_content) header, footer, routers_end = _get_document_content(document_file, validate)

@@ -290,13 +292,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): return bool(self.consensus_method >= method or filter(lambda x: x >= method, self.consensus_methods))

def get_unrecognized_lines(self): - """ - Returns any unrecognized trailing lines. - - :returns: a list of unrecognized trailing lines - """ - - return self.unrecognized_lines + return list(self._unrecognized_lines)

def _parse(self, header, footer, validate): """ @@ -445,6 +441,15 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): actual_label = ', '.join(weight_keys)

raise ValueError("A network status document's 'bandwidth-weights' entries should be '%s', got '%s'" % (expected_label, actual_label)) + elif keyword == "directory-signature": + if not " " in value or not block_contents: + if not validate: continue + raise ValueError("Authority signatures in a network status document are expected to be of the form 'directory-signature FINGERPRINT KEY_DIGEST\nSIGNATURE', got:\n%s" % line) + + fingerprint, key_digest = value.split(" ", 1) + self.signatures.append(DocumentSignature(fingerprint, key_digest, block_contents, validate)) + else: + self._unrecognized_lines.append(line)

# doing this validation afterward so we know our 'is_consensus' and # 'is_vote' attributes @@ -483,17 +488,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):

_read_keyword_line("directory-footer", content, False, True) _read_keyword_line("bandwidth-weights", content, False, True) - - while _peek_keyword(content) == "directory-signature": - signature_data = _read_until_keywords("directory-signature", content, False, True) - self.directory_signatures.append(DirectorySignature("".join(signature_data))) - - remainder = content.read() - - if remainder: - self.unrecognized_lines = remainder.split("\n") - else: - self.unrecognized_lines = [] + _read_keyword_line("directory-signature", content, False, True)

def _check_for_missing_and_disallowed_fields(self, header_entries, footer_entries): """ @@ -706,60 +701,44 @@ class DirectoryAuthority(stem.descriptor.Descriptor):

return self.unrecognized_lines

-class DirectorySignature(stem.descriptor.Descriptor): +# TODO: microdescriptors have a slightly different format (including a +# 'method') - should probably be a subclass +class DocumentSignature(object): """ - Contains directory signatures in a v3 network status document. + Directory signature of a v3 network status document.

- :var str identity: signature identity - :var str key_digest: signature key digest - :var str method: method used to generate the signature - :var str signature: the signature data - """ + :var str identity: fingerprint of the authority that made the signature + :var str key_digest: digest of the signing key + :var str signature: document signature + :param bool validate: checks validity if True

- def __init__(self, raw_content, validate = True): - """ - Parse a directory signature entry in a v3 network status document and - provide a DirectorySignature object. - - :param str raw_content: raw directory signature entry information - :param bool validate: True if the document is to be validated, False otherwise - - :raises: ValueError if the raw data is invalid - """ - - super(DirectorySignature, self).__init__(raw_content) - self.identity, self.key_digest, self.method, self.signature = None, None, None, None - content = raw_content.splitlines() - - signature_line = _read_keyword_line_str("directory-signature", content, validate).split(" ") - - if len(signature_line) == 2: - self.identity, self.key_digest = signature_line - if len(signature_line) == 3: - # for microdescriptor consensuses - # This 'method' seems to be undocumented 8-8-12 - self.method, self.identity, self.key_digest = signature_line - - self.signature = _get_pseudo_pgp_block(content) - self.unrecognized_lines = content - if self.unrecognized_lines and validate: - raise ValueError("Unrecognized trailing data in directory signature") + :raises: ValueError if a validity check fails + """

- def get_unrecognized_lines(self): - """ - Returns any unrecognized lines. + def __init__(self, identity, key_digest, signature, validate = True): + # Checking that these attributes are valid. Technically the key + # digest isn't a fingerprint, but it has the same characteristics.

- :returns: a list of unrecognized lines - """ + if validate: + if not stem.util.tor_tools.is_valid_fingerprint(identity): + raise ValueError("Malformed fingerprint (%s) in the document signature" % (identity)) + + if not stem.util.tor_tools.is_valid_fingerprint(key_digest): + raise ValueError("Malformed key digest (%s) in the document signature" % (key_digest))

- return self.unrecognized_lines + self.identity = identity + self.key_digest = key_digest + self.signature = signature

def __cmp__(self, other): - if not isinstance(other, DirectorySignature): + if not isinstance(other, DocumentSignature): return 1

- # attributes are all derived from content, so we can simply use that to check - return str(self) > str(other) + for attr in ("identity", "key_digest", "signature"): + if getattr(self, attr) > getattr(other, attr): return 1 + elif getattr(self, attr) < getattr(other, attr): return -1 + + return 0

class RouterStatusEntry(stem.descriptor.Descriptor): """ @@ -1033,7 +1012,7 @@ class MicrodescriptorConsensus(NetworkStatusDocument): :var list params: dict of parameter(str) => value(int) mappings :var list directory_authorities: ***** list of DirectoryAuthority objects that have generated this document :var dict bandwidth_weights: **~** dict of weight(str) => value(int) mappings - :var list directory_signatures: ***** list of signatures this document has + :var list signatures: ***** list of signatures this document has

| ***** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined | **~** attribute appears only in consensuses diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index d77a36f..aa91ab3 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -143,10 +143,10 @@ HFXB4497LzESysYJ/4jJY83E5vLjhv+igIxD9LU6lf6ftkGeF+lNmIAIEKaMts8H mfWcW0b+jsrXcJoCxV5IrwCDF3u1aC3diwZY6yiG186pwWbOwE41188XI2DeYPwE I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= -----END SIGNATURE-----""" - self.assertEquals(8, len(desc.directory_signatures)) - self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.directory_signatures[0].identity) - self.assertEquals("BF112F1C6D5543CFD0A32215ACABD4197B5279AD", desc.directory_signatures[0].key_digest) - self.assertEquals(expected_signature, desc.directory_signatures[0].signature) + self.assertEquals(8, len(desc.signatures)) + self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.signatures[0].identity) + self.assertEquals("BF112F1C6D5543CFD0A32215ACABD4197B5279AD", desc.signatures[0].key_digest) + self.assertEquals(expected_signature, desc.signatures[0].signature)

def test_metrics_vote(self): """ @@ -261,10 +261,10 @@ fskXN84wB3mXfo+yKGSt0AcDaaPuU3NwMR3ROxWgLN0KjAaVi2eV9PkPCsQkcgw3 JZ/1HL9sHyZfo6bwaC6YSM9PNiiY6L7rnGpS7UkHiFI+M96VCMorvjm5YPs3FioJ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= -----END SIGNATURE-----""" - self.assertEquals(1, len(desc.directory_signatures)) - self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_signatures[0].identity) - self.assertEquals("D5C30C15BB3F1DA27669C2D88439939E8F418FCF", desc.directory_signatures[0].key_digest) - self.assertEquals(expected_signature, desc.directory_signatures[0].signature) + self.assertEquals(1, len(desc.signatures)) + self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.signatures[0].identity) + self.assertEquals("D5C30C15BB3F1DA27669C2D88439939E8F418FCF", desc.signatures[0].key_digest) + self.assertEquals(expected_signature, desc.signatures[0].signature)

def test_cached_microdesc_consensus(self): """ diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py index 8ed1f0e..a0fe3a8 100644 --- a/test/unit/descriptor/networkstatus/document.py +++ b/test/unit/descriptor/networkstatus/document.py @@ -7,7 +7,16 @@ import unittest

import stem.version from stem.descriptor import Flag -from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, NetworkStatusDocument, DirectorySignature +from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, NetworkStatusDocument, DocumentSignature + +sig_block = """\ +-----BEGIN SIGNATURE----- +e1XH33ITaUYzXu+dK04F2dZwR4PhcOQgIuK859KGpU77/6lRuggiX/INk/4FJanJ +ysCTE1K4xk4fH3N1Tzcv/x/gS4LUlIZz3yKfBnj+Xh3w12Enn9V1Gm1Vrhl+/YWH +eweONYRZTTvgsB+aYsCoBuoBBpbr4Swlu64+85F44o4= +-----END SIGNATURE-----""" + +SIG = DocumentSignature("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", "BF112F1C6D5543CFD0A32215ACABD4197B5279AD", sig_block)

NETWORK_STATUS_DOCUMENT_ATTR = { "network-status-version": "3", @@ -21,16 +30,9 @@ NETWORK_STATUS_DOCUMENT_ATTR = { "voting-delay": "300 300", "known-flags": "Authority BadExit Exit Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid", "directory-footer": "", - "directory-signature": "\n".join(( - "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 BF112F1C6D5543CFD0A32215ACABD4197B5279AD", - "-----BEGIN SIGNATURE-----", - "e1XH33ITaUYzXu+dK04F2dZwR4PhcOQgIuK859KGpU77/6lRuggiX/INk/4FJanJ", - "ysCTE1K4xk4fH3N1Tzcv/x/gS4LUlIZz3yKfBnj+Xh3w12Enn9V1Gm1Vrhl+/YWH", - "eweONYRZTTvgsB+aYsCoBuoBBpbr4Swlu64+85F44o4=", - "-----END SIGNATURE-----")), + "directory-signature": "%s %s\n%s" % (SIG.identity, SIG.key_digest, SIG.signature), }

-SIG = DirectorySignature("directory-signature " + NETWORK_STATUS_DOCUMENT_ATTR["directory-signature"])

def get_network_status_document(attr = None, exclude = None, routers = None): """ @@ -119,7 +121,7 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertEqual(DEFAULT_PARAMS, document.params) self.assertEqual([], document.directory_authorities) self.assertEqual({}, document.bandwidth_weights) - self.assertEqual([SIG], document.directory_signatures) + self.assertEqual([SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines())

def test_minimal_vote(self): @@ -151,7 +153,7 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertEqual(DEFAULT_PARAMS, document.params) self.assertEqual([], document.directory_authorities) self.assertEqual({}, document.bandwidth_weights) - self.assertEqual([SIG], document.directory_signatures) + self.assertEqual([SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines())

def test_missing_fields(self): @@ -170,6 +172,15 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertRaises(ValueError, NetworkStatusDocument, content) NetworkStatusDocument(content, False) # constructs without validation

+ def test_unrecognized_line(self): + """ + Includes unrecognized content in the document. + """ + + content = get_network_status_document({"pepperjack": "is oh so tasty!"}) + document = NetworkStatusDocument(content) + self.assertEquals(["pepperjack is oh so tasty!"], document.get_unrecognized_lines()) + def test_misordered_fields(self): """ Rearranges our descriptor fields. @@ -533,14 +544,14 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertRaises(ValueError, NetworkStatusDocument, content)

document = NetworkStatusDocument(content, False) - self.assertEqual([SIG], document.directory_signatures) + self.assertEqual([SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines())

# excludes a footer from a version that shouldn't have it

content = get_network_status_document({"consensus-method": "8"}, ("directory-footer", "directory-signature")) document = NetworkStatusDocument(content) - self.assertEqual([], document.directory_signatures) + self.assertEqual([], document.signatures) self.assertEqual([], document.get_unrecognized_lines())

def test_footer_with_value(self): @@ -552,7 +563,7 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertRaises(ValueError, NetworkStatusDocument, content)

document = NetworkStatusDocument(content, False) - self.assertEqual([SIG], document.directory_signatures) + self.assertEqual([SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines())

def test_bandwidth_wights_ok(self): @@ -649,4 +660,24 @@ class TestNetworkStatusDocument(unittest.TestCase):

document = NetworkStatusDocument(content, False) self.assertEquals(expected, document.bandwidth_weights) + + def test_malformed_signature(self): + """ + Provides malformed or missing content in the 'directory-signature' line. + """ + + test_values = ( + "", + "\n", + "blarg", + ) + + for test_value in test_values: + for test_attr in xrange(3): + attrs = [SIG.identity, SIG.key_digest, SIG.signature] + attrs[test_attr] = test_value + + content = get_network_status_document({"directory-signature": "%s %s\n%s" % tuple(attrs)}) + self.assertRaises(ValueError, NetworkStatusDocument, content) + NetworkStatusDocument(content, False) # checks that it's still parseable without validation