commit cf475d58dfa4d42e982eca6307e2a61e7545147e Author: Damian Johnson <atagar@torproject.org> Date: Thu Sep 6 09:14:09 2012 -0700 Minimal unit test for network status documents Adding a unit test for the minimal valid network status document (plus a consensus-method field since that influences validation). This uncovered some bugs with the NetworkStatusDocument class... * The network_status_version field misdocumented as being an int (it was actually a str). We need it to be a str for microdescriptors so simply changed the pydoc. * The consensus-method and bandwidth-weights are documented in the spec as being optional fields. The parser errored with a stacktrace when consensus-method was missing, and gave a validation error if there isn't a bandwidth-weights. * Inappropriate validation error if there was unrecognized content. * The get_unrecognized_lines() method is documented as providing a list of lines. The NetworkStatusDocument returned a string instead. * Off-by-one error that caused consensus-method 9 documents to skip parsing footers. --- stem/descriptor/networkstatus.py | 28 ++++-- test/unit/descriptor/networkstatus.py | 173 ++++++++++++++++++++++++++------ 2 files changed, 159 insertions(+), 42 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 168c768..6f1f7b1 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -198,11 +198,11 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :var tuple routers: RouterStatusEntry contained in the document - :var int network_status_version: **\*** document version + :var str network_status_version: **\*** document version :var str vote_status: **\*** status of the vote (is either "vote" or "consensus") + :var int consensus_method: **~** consensus method used to generate a consensus :var list consensus_methods: **^** A list of supported consensus generation methods (integers) :var datetime published: **^** time when the document was published - :var int consensus_method: **~** consensus method used to generate a consensus :var datetime valid_after: **\*** time when the consensus becomes valid :var datetime fresh_until: **\*** time until when the consensus is considered to be fresh :var datetime valid_until: **\*** time until when the consensus is valid @@ -318,7 +318,8 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): self.published = _strptime(_read_keyword_line("published", content, validate, True), validate, True) else: read_keyword_line("consensus-method", True) - self.consensus_method = int(self.consensus_method) + if self.consensus_method != None: + self.consensus_method = int(self.consensus_method) map(read_keyword_line, ["valid-after", "fresh-until", "valid-until"]) self.valid_after = _strptime(self.valid_after, validate) @@ -345,7 +346,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): self.directory_authorities.append(DirectoryAuthority(dirauth_data, vote, validate)) # footer section - if self.consensus_method > 9 or vote and filter(lambda x: x >= 9, self.consensus_methods): + if self.consensus_method >= 9 or vote and filter(lambda x: x >= 9, self.consensus_methods): if _peek_keyword(content) == "directory-footer": content.readline() elif validate: @@ -353,17 +354,19 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): if not vote: read_keyword_line("bandwidth-weights", True) - if _bandwidth_weights_regex.match(self.bandwidth_weights): + if self.bandwidth_weights != None and _bandwidth_weights_regex.match(self.bandwidth_weights): self.bandwidth_weights = dict([(weight.split("=")[0], int(weight.split("=")[1])) for weight in self.bandwidth_weights.split(" ")]) - elif validate: - raise ValueError("Invalid bandwidth-weights line") while _peek_keyword(content) == "directory-signature": signature_data = _read_until_keywords("directory-signature", content, False, True) self.directory_signatures.append(DirectorySignature("".join(signature_data))) - self.unrecognized_lines = content.read() - if validate and self.unrecognized_lines: raise ValueError("Unrecognized trailing data") + remainder = content.read() + + if remainder: + self.unrecognized_lines = content.read().split("\n") + else: + self.unrecognized_lines = [] def _check_for_missing_and_disallowed_fields(self, is_consensus, header_entries, footer_entries): """ @@ -508,6 +511,13 @@ class DirectorySignature(stem.descriptor.Descriptor): """ return self.unrecognized_lines + + def __cmp__(self, other): + if not isinstance(other, DirectorySignature): + return 1 + + # attributes are all derived from content, so we can simply use that to check + return str(self) > str(other) class RouterStatusEntry(stem.descriptor.Descriptor): """ diff --git a/test/unit/descriptor/networkstatus.py b/test/unit/descriptor/networkstatus.py index c7672e6..db1270e 100644 --- a/test/unit/descriptor/networkstatus.py +++ b/test/unit/descriptor/networkstatus.py @@ -5,15 +5,89 @@ Unit tests for stem.descriptor.networkstatus. import datetime import unittest -from stem.descriptor.networkstatus import Flag, RouterStatusEntry, _decode_fingerprint +from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, Flag, NetworkStatusDocument, RouterStatusEntry, DirectorySignature, _decode_fingerprint from stem.version import Version from stem.exit_policy import MicrodescriptorExitPolicy +NETWORK_STATUS_DOCUMENT_ATTR = { + "network-status-version": "3", + "vote-status": "consensus", + "consensus-method": "9", + "published": "2012-09-02 22:00:00", + "valid-after": "2012-09-02 22:00:00", + "fresh-until": "2012-09-02 22:00:00", + "valid-until": "2012-09-02 22:00:00", + "voting-delay": "300 300", + "known-flags": "Authority BadExit Exit Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid", + "directory-footer": "", + "directory-signature": "\n".join(( + "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 BF112F1C6D5543CFD0A32215ACABD4197B5279AD", + "-----BEGIN SIGNATURE-----", + "e1XH33ITaUYzXu+dK04F2dZwR4PhcOQgIuK859KGpU77/6lRuggiX/INk/4FJanJ", + "ysCTE1K4xk4fH3N1Tzcv/x/gS4LUlIZz3yKfBnj+Xh3w12Enn9V1Gm1Vrhl+/YWH", + "eweONYRZTTvgsB+aYsCoBuoBBpbr4Swlu64+85F44o4=", + "-----END SIGNATURE-----")), +} + ROUTER_STATUS_ENTRY_ATTR = ( ("r", "caerSidi p1aag7VwarGxqctS7/fS0y5FU+s oQZFLYe9e4A7bOkWKR7TaNxb0JE 2012-08-06 11:19:31 71.35.150.29 9001 0"), ("s", "Fast Named Running Stable Valid"), ) +def get_network_status_document(attr = None, exclude = None, routers = None): + """ + Constructs a minimal network status document with the given attributes. This + places attributes in the proper order to be valid. + + :param dict attr: keyword/value mappings to be included in the entry + :param list exclude: mandatory keywords to exclude from the entry + :param list routers: lines with router status entry content + + :returns: str with customized router status entry content + """ + + descriptor_lines = [] + if attr is None: attr = {} + if exclude is None: exclude = [] + if routers is None: routers = [] + attr = dict(attr) # shallow copy since we're destructive + + is_vote = attr.get("vote-status") == "vote" + is_consensus = not is_vote + + header_content, footer_content = [], [] + + for content, entries in ((header_content, HEADER_STATUS_DOCUMENT_FIELDS), + (footer_content, FOOTER_STATUS_DOCUMENT_FIELDS)): + for field, in_votes, in_consensus, is_mandatory in entries: + if field in exclude: continue + + if not field in attr: + # Skip if it's not mandatory for this type of document. An exception is + # made for the consensus' consensus-method field since it influences + # validation, and is only missing for consensus-method lower than 2. + + if field == "consensus-method" and is_consensus: + pass + elif not is_mandatory or not ((is_consensus and in_consensus) or (is_vote and in_vote)): + continue + + if field in attr: + value = attr[keyword] + del attr[keyword] + elif field in NETWORK_STATUS_DOCUMENT_ATTR: + value = NETWORK_STATUS_DOCUMENT_ATTR[field] + + if value: value = " %s" % value + content.append(field + value) + + remainder = [] + for attr_keyword, attr_value in attr.items(): + if attr_value: attr_value = " %s" % attr_value + remainder.append(attr_keyword + attr_value) + + return "\n".join(header_content + remainder + routers + footer_content) + def get_router_status_entry(attr = None, exclude = None): """ Constructs a minimal router status entry with the given attributes. @@ -67,7 +141,40 @@ class TestNetworkStatus(unittest.TestCase): self.assertRaises(ValueError, _decode_fingerprint, arg, True) self.assertEqual(None, _decode_fingerprint(arg, False)) - def test_rse_minimal(self): + def test_document_minimal(self): + """ + Parses a minimal network status document. + """ + + document = NetworkStatusDocument(get_network_status_document()) + + expected_known_flags = [Flag.AUTHORITY, Flag.BADEXIT, Flag.EXIT, + Flag.FAST, Flag.GUARD, Flag.HSDIR, Flag.NAMED, Flag.RUNNING, + Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID] + + sig = DirectorySignature("directory-signature " + NETWORK_STATUS_DOCUMENT_ATTR["directory-signature"]) + + self.assertEqual((), document.routers) + self.assertEqual("3", document.network_status_version) + self.assertEqual("consensus", document.vote_status) + self.assertEqual(9, document.consensus_method) + self.assertEqual([], document.consensus_methods) + self.assertEqual(None, document.published) + self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.valid_after) + self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.fresh_until) + self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.valid_until) + self.assertEqual(300, document.vote_delay) + self.assertEqual(300, document.dist_delay) + self.assertEqual([], document.client_versions) + self.assertEqual([], document.server_versions) + self.assertEqual(expected_known_flags, document.known_flags) + self.assertEqual(None, document.params) + self.assertEqual([], document.directory_authorities) + self.assertEqual(None, document.bandwidth_weights) + self.assertEqual([sig], document.directory_signatures) + self.assertEqual([], document.get_unrecognized_lines()) + + def test_entry_minimal(self): """ Parses a minimal router status entry. """ @@ -93,21 +200,21 @@ class TestNetworkStatus(unittest.TestCase): self.assertEqual(None, entry.microdescriptor_hashes) self.assertEqual([], entry.get_unrecognized_lines()) - def test_rse_missing_fields(self): + def test_entry_missing_fields(self): """ Parses a router status entry that's missing fields. """ content = get_router_status_entry(exclude = ('r', 's')) - self._expect_invalid_rse_attr(content, "address") + self._expect_invalid_entry_attr(content, "address") content = get_router_status_entry(exclude = ('r',)) - self._expect_invalid_rse_attr(content, "address") + self._expect_invalid_entry_attr(content, "address") content = get_router_status_entry(exclude = ('s',)) - self._expect_invalid_rse_attr(content, "flags") + self._expect_invalid_entry_attr(content, "flags") - def test_rse_unrecognized_lines(self): + def test_entry_unrecognized_lines(self): """ Parses a router status entry with new keywords. """ @@ -116,15 +223,15 @@ class TestNetworkStatus(unittest.TestCase): entry = RouterStatusEntry(content, None) self.assertEquals(['z New tor feature: sparkly unicorns!'], entry.get_unrecognized_lines()) - def test_rse_proceeding_line(self): + def test_entry_proceeding_line(self): """ Includes content prior to the 'r' line. """ content = 'z some stuff\n' + get_router_status_entry() - self._expect_invalid_rse_attr(content, "_unrecognized_lines", ['z some stuff']) + self._expect_invalid_entry_attr(content, "_unrecognized_lines", ['z some stuff']) - def test_rse_blank_lines(self): + def test_entry_blank_lines(self): """ Includes blank lines, which should be ignored. """ @@ -133,7 +240,7 @@ class TestNetworkStatus(unittest.TestCase): entry = RouterStatusEntry(content, None) self.assertEqual("Tor 0.2.2.35", entry.version_line) - def test_rse_missing_r_field(self): + def test_entry_missing_r_field(self): """ Excludes fields from the 'r' line. """ @@ -155,9 +262,9 @@ class TestNetworkStatus(unittest.TestCase): r_line = ' '.join(test_components) content = get_router_status_entry({'r': r_line}) - self._expect_invalid_rse_attr(content, attr) + self._expect_invalid_entry_attr(content, attr) - def test_rse_malformed_nickname(self): + def test_entry_malformed_nickname(self): """ Parses an 'r' line with a malformed nickname. """ @@ -184,9 +291,9 @@ class TestNetworkStatus(unittest.TestCase): if value == "": value = None - self._expect_invalid_rse_attr(content, "nickname", value) + self._expect_invalid_entry_attr(content, "nickname", value) - def test_rse_malformed_fingerprint(self): + def test_entry_malformed_fingerprint(self): """ Parses an 'r' line with a malformed fingerprint. """ @@ -200,9 +307,9 @@ class TestNetworkStatus(unittest.TestCase): for value in test_values: r_line = ROUTER_STATUS_ENTRY_ATTR[0][1].replace("p1aag7VwarGxqctS7/fS0y5FU+s", value) content = get_router_status_entry({'r': r_line}) - self._expect_invalid_rse_attr(content, "fingerprint") + self._expect_invalid_entry_attr(content, "fingerprint") - def test_rse_malformed_published_date(self): + def test_entry_malformed_published_date(self): """ Parses an 'r' line with a malformed published date. """ @@ -226,9 +333,9 @@ class TestNetworkStatus(unittest.TestCase): for value in test_values: r_line = ROUTER_STATUS_ENTRY_ATTR[0][1].replace("2012-08-06 11:19:31", value) content = get_router_status_entry({'r': r_line}) - self._expect_invalid_rse_attr(content, "published") + self._expect_invalid_entry_attr(content, "published") - def test_rse_malformed_address(self): + def test_entry_malformed_address(self): """ Parses an 'r' line with a malformed address. """ @@ -244,9 +351,9 @@ class TestNetworkStatus(unittest.TestCase): for value in test_values: r_line = ROUTER_STATUS_ENTRY_ATTR[0][1].replace("71.35.150.29", value) content = get_router_status_entry({'r': r_line}) - self._expect_invalid_rse_attr(content, "address", value) + self._expect_invalid_entry_attr(content, "address", value) - def test_rse_malformed_port(self): + def test_entry_malformed_port(self): """ Parses an 'r' line with a malformed ORPort or DirPort. """ @@ -276,9 +383,9 @@ class TestNetworkStatus(unittest.TestCase): expected = int(value) if value.isdigit() else None content = get_router_status_entry({'r': r_line}) - self._expect_invalid_rse_attr(content, attr, expected) + self._expect_invalid_entry_attr(content, attr, expected) - def test_rse_flags(self): + def test_entry_flags(self): """ Handles a variety of flag inputs. """ @@ -304,9 +411,9 @@ class TestNetworkStatus(unittest.TestCase): for s_line, expected in test_values.items(): content = get_router_status_entry({'s': s_line}) - self._expect_invalid_rse_attr(content, "flags", expected) + self._expect_invalid_entry_attr(content, "flags", expected) - def test_rse_versions(self): + def test_entry_versions(self): """ Handles a variety of version inputs. """ @@ -326,9 +433,9 @@ class TestNetworkStatus(unittest.TestCase): # tries an invalid input content = get_router_status_entry({'v': "Tor ugabuga"}) - self._expect_invalid_rse_attr(content, "version") + self._expect_invalid_entry_attr(content, "version") - def test_rse_bandwidth(self): + def test_entry_bandwidth(self): """ Handles a variety of 'w' lines. """ @@ -363,9 +470,9 @@ class TestNetworkStatus(unittest.TestCase): for w_line in test_values: content = get_router_status_entry({'w': w_line}) - self._expect_invalid_rse_attr(content) + self._expect_invalid_entry_attr(content) - def test_rse_exit_policy(self): + def test_entry_exit_policy(self): """ Handles a variety of 'p' lines. """ @@ -390,9 +497,9 @@ class TestNetworkStatus(unittest.TestCase): for p_line in test_values: content = get_router_status_entry({'p': p_line}) - self._expect_invalid_rse_attr(content, "exit_policy") + self._expect_invalid_entry_attr(content, "exit_policy") - def test_rse_microdescriptor_hashes(self): + def test_entry_microdescriptor_hashes(self): """ Handles a variety of 'm' lines. """ @@ -417,7 +524,7 @@ class TestNetworkStatus(unittest.TestCase): # try without a document content = get_router_status_entry({'m': "8,9,10,11,12"}) - self._expect_invalid_rse_attr(content, "microdescriptor_hashes") + self._expect_invalid_entry_attr(content, "microdescriptor_hashes") # tries some invalid inputs test_values = ( @@ -430,7 +537,7 @@ class TestNetworkStatus(unittest.TestCase): content = get_router_status_entry({'m': m_line}) self.assertRaises(ValueError, RouterStatusEntry, content, mock_document) - def _expect_invalid_rse_attr(self, content, attr = None, expected_value = None): + def _expect_invalid_entry_attr(self, content, attr = None, expected_value = None): """ Asserts that construction will fail due to content having a malformed attribute. If an attr is provided then we check that it matches an expected