October 2012 - tor-commits - lists.torproject.org

[stem/master] Moving network status document mocking into util
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 8f89166499dafcd155022e07bbf39eafd7cccf4e Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Sep 28 09:05:35 2012 -0700 Moving network status document mocking into util The network status document has a few oddities compared to the other descriptor types. It includes... - ordering constraints outside of the mandatory fields - keyword-only lines - router status entries - different mandatory fields based on if it's a vote or consensus Imho this new version handles these a lot more elegantly than the prior helper that was embedded in the network status document unit tests. --- test/mocking.py | 86 ++++++++++- test/unit/descriptor/networkstatus/document.py | 202 ++++++------------------ 2 files changed, 135 insertions(+), 153 deletions(-) diff --git a/test/mocking.py b/test/mocking.py index 3d4a615..7008a2a 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -40,6 +40,7 @@ import stem.response import stem.socket import stem.descriptor.server_descriptor import stem.descriptor.extrainfo_descriptor +import stem.descriptor.networkstatus # Once we've mocked a function we can't rely on its __module__ or __name__ # attributes, so instead we associate a unique 'mock_id' attribute that maps @@ -59,6 +60,11 @@ skFtXhOHHqTRN4GPPrZsAIUOQGzQtGb66IQgT4tO/pj+P6QmSCCdTfhvGfgTCsC+ WPi4Fl2qryzTb3QO5r5x7T8OsG2IBUET1bLQzmtbC560SYR49IvVAgMBAAE= """ +DOC_SIG = stem.descriptor.networkstatus.DocumentSignature( + "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", + "BF112F1C6D5543CFD0A32215ACABD4197B5279AD", + "-----BEGIN SIGNATURE-----%s-----END SIGNATURE-----" % CRYPTO_BLOB) + RELAY_SERVER_HEADER = ( ("router", "caerSidi 71.35.133.197 9001 0 0"), ("published", "2012-03-01 17:15:27"), @@ -116,6 +122,28 @@ KEY_CERTIFICATE_FOOTER = ( ("dir-key-certification", "\n-----BEGIN SIGNATURE-----%s-----END SIGNATURE-----" % CRYPTO_BLOB), ) +NETWORK_STATUS_DOCUMENT_HEADER = ( + ("network-status-version", "3"), + ("vote-status", "consensus"), + ("consensus-methods", None), + ("consensus-method", None), + ("published", None), + ("valid-after", "2012-09-02 22:00:00"), + ("fresh-until", "2012-09-02 22:00:00"), + ("valid-until", "2012-09-02 22:00:00"), + ("voting-delay", "300 300"), + ("client-versions", None), + ("server-versions", None), + ("known-flags", "Authority BadExit Exit Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid"), + ("params", None), +) + +NETWORK_STATUS_DOCUMENT_FOOTER = ( + ("directory-footer", ""), + ("bandwidth-weights", None), + ("directory-signature", "%s %s\n%s" % (DOC_SIG.identity, DOC_SIG.key_digest, DOC_SIG.signature)), +) + def no_op(): def _no_op(*args): pass return _no_op @@ -397,9 +425,18 @@ def _get_descriptor_content(attr = None, exclude = (), header_template = (), foo value = attr[keyword] del attr[keyword] - content.append("%s %s" % (keyword, value)) + if value is None: continue + elif value == "": + content.append(keyword) + else: + content.append("%s %s" % (keyword, value)) + + remainder = [] + + for k, v in attr.items(): + if v: remainder.append("%s %s" % (k, v)) + else: remainder.append(k) - remainder = ["%s %s" % (k, v) for k, v in attr.items()] return "\n".join(header_content + remainder + footer_content) def get_relay_server_descriptor(attr = None, exclude = (), content = False): @@ -516,3 +553,48 @@ def get_key_certificate(attr = None, exclude = (), content = False): else: return stem.descriptor.networkstatus.KeyCertificate(desc_content, validate = True) +def get_network_status_document(attr = None, exclude = (), routers = None, content = False): + """ + Provides the descriptor content for... + stem.descriptor.networkstatus.NetworkStatusDocument + + :param dict attr: keyword/value mappings to be included in the descriptor + :param list exclude: mandatory keywords to exclude from the descriptor + :param list routers: router status entries to include in the document + :param bool content: provides the str content of the descriptor rather than the class if True + + :returns: NetworkStatusDocument for the requested descriptor content + """ + + if attr is None: + attr = {} + + # add defaults only found in a vote or consensus + + if attr.get("vote-status") == "vote": + extra_defaults = { + "consensus-methods": "1 9", + "published": "2012-09-02 22:00:00", + } + else: + extra_defaults = { + "consensus-method": "9", + } + + for k, v in extra_defaults.items(): + if not (k in attr or (exclude and k in exclude)): + attr[k] = v + + desc_content = _get_descriptor_content(attr, exclude, NETWORK_STATUS_DOCUMENT_HEADER, NETWORK_STATUS_DOCUMENT_FOOTER) + + if routers: + # inject the routers between the header and footer + footer_div = desc_content.find("\ndirectory-footer") + 1 + router_content = "\n".join([str(r) for r in routers]) + "\n" + desc_content = desc_content[:footer_div] + router_content + desc_content[footer_div:] + + if content: + return desc_content + else: + return stem.descriptor.networkstatus.NetworkStatusDocument(desc_content, validate = True) + diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py index 80fe6f2..8a01ec6 100644 --- a/test/unit/descriptor/networkstatus/document.py +++ b/test/unit/descriptor/networkstatus/document.py @@ -9,94 +9,7 @@ import StringIO import stem.version from stem.descriptor import Flag from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, RouterStatusEntry, NetworkStatusDocument, DocumentSignature, parse_file -from test.mocking import get_router_status_entry - -sig_block = """\ ------BEGIN SIGNATURE----- -e1XH33ITaUYzXu+dK04F2dZwR4PhcOQgIuK859KGpU77/6lRuggiX/INk/4FJanJ -ysCTE1K4xk4fH3N1Tzcv/x/gS4LUlIZz3yKfBnj+Xh3w12Enn9V1Gm1Vrhl+/YWH -eweONYRZTTvgsB+aYsCoBuoBBpbr4Swlu64+85F44o4= ------END SIGNATURE-----""" - -SIG = DocumentSignature("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", "BF112F1C6D5543CFD0A32215ACABD4197B5279AD", sig_block) - -NETWORK_STATUS_DOCUMENT_ATTR = { - "network-status-version": "3", - "vote-status": "consensus", - "consensus-methods": "1 9", - "consensus-method": "9", - "published": "2012-09-02 22:00:00", - "valid-after": "2012-09-02 22:00:00", - "fresh-until": "2012-09-02 22:00:00", - "valid-until": "2012-09-02 22:00:00", - "voting-delay": "300 300", - "known-flags": "Authority BadExit Exit Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid", - "directory-footer": "", - "directory-signature": "%s %s\n%s" % (SIG.identity, SIG.key_digest, SIG.signature), -} - -def get_network_status_document(attr = None, exclude = None, routers = None): - """ - Constructs a minimal network status document with the given attributes. This - places attributes in the proper order to be valid. - - :param dict attr: keyword/value mappings to be included in the entry - :param list exclude: mandatory keywords to exclude from the entry - :param list routers: lines with router status entry content - - :returns: str with customized router status entry content - """ - - descriptor_lines = [] - if attr is None: attr = {} - if exclude is None: exclude = [] - if routers is None: routers = [] - attr = dict(attr) # shallow copy since we're destructive - - is_vote = attr.get("vote-status") == "vote" - is_consensus = not is_vote - - header_content, footer_content = [], [] - - for content, entries in ((header_content, HEADER_STATUS_DOCUMENT_FIELDS), - (footer_content, FOOTER_STATUS_DOCUMENT_FIELDS)): - for field, in_votes, in_consensus, is_mandatory in entries: - if field in exclude: continue - - if not field in attr: - # Skip if it's not mandatory for this type of document. An exception is - # made for the consensus' consensus-method and consensus-methods fields - # since it influences validation, and is only missing for - # consensus-method lower than 2. - - if field == "consensus-method" and is_consensus: - pass - elif field == "consensus-methods" and is_vote: - pass - elif not is_mandatory or not ((is_consensus and in_consensus) or (is_vote and in_votes)): - continue - - if field in attr: - value = attr[field] - del attr[field] - elif field in NETWORK_STATUS_DOCUMENT_ATTR: - value = NETWORK_STATUS_DOCUMENT_ATTR[field] - - if value: value = " %s" % value - content.append(field + value) - - remainder = [] - for attr_keyword, attr_value in attr.items(): - if attr_value: attr_value = " %s" % attr_value - remainder.append(attr_keyword + attr_value) - - # join the routers into a single block, then split it into lines - if routers: - router_lines = ("\n".join([str(r) for r in routers])).split("\n") - else: - router_lines = [] - - return "\n".join(header_content + remainder + router_lines + footer_content) +from test.mocking import get_router_status_entry, get_network_status_document, CRYPTO_BLOB, DOC_SIG class TestNetworkStatusDocument(unittest.TestCase): def test_minimal_consensus(self): @@ -104,7 +17,7 @@ class TestNetworkStatusDocument(unittest.TestCase): Parses a minimal network status document. """ - document = NetworkStatusDocument(get_network_status_document()) + document = get_network_status_document() expected_known_flags = [Flag.AUTHORITY, Flag.BADEXIT, Flag.EXIT, Flag.FAST, Flag.GUARD, Flag.HSDIR, Flag.NAMED, Flag.RUNNING, @@ -128,7 +41,7 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertEqual(DEFAULT_PARAMS, document.params) self.assertEqual((), document.directory_authorities) self.assertEqual({}, document.bandwidth_weights) - self.assertEqual([SIG], document.signatures) + self.assertEqual([DOC_SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines()) def test_minimal_vote(self): @@ -136,7 +49,7 @@ class TestNetworkStatusDocument(unittest.TestCase): Parses a minimal network status document. """ - document = NetworkStatusDocument(get_network_status_document({"vote-status": "vote"})) + document = get_network_status_document({"vote-status": "vote"}) expected_known_flags = [Flag.AUTHORITY, Flag.BADEXIT, Flag.EXIT, Flag.FAST, Flag.GUARD, Flag.HSDIR, Flag.NAMED, Flag.RUNNING, @@ -160,7 +73,7 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertEqual(DEFAULT_PARAMS, document.params) self.assertEqual((), document.directory_authorities) self.assertEqual({}, document.bandwidth_weights) - self.assertEqual([SIG], document.signatures) + self.assertEqual([DOC_SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines()) def test_parse_file(self): @@ -170,12 +83,12 @@ class TestNetworkStatusDocument(unittest.TestCase): entry1 = get_router_status_entry({'s': "Fast"}) entry2 = get_router_status_entry({'s': "Valid"}) - content = get_network_status_document(routers = (entry1, entry2)) + content = get_network_status_document(routers = (entry1, entry2), content = True) # the document that the entries refer to should actually be the minimal # descriptor (ie, without the entries) - expected_document = NetworkStatusDocument(get_network_status_document()) + expected_document = get_network_status_document() descriptor_file = StringIO.StringIO(content) entries = list(parse_file(descriptor_file)) @@ -196,7 +109,7 @@ class TestNetworkStatusDocument(unittest.TestCase): for entries in (HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS): for field, in_votes, in_consensus, is_mandatory in entries: if is_mandatory and ((is_consensus and in_consensus) or (is_vote and in_votes)): - content = get_network_status_document(attr, exclude = (field,)) + content = get_network_status_document(attr, exclude = (field,), content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) NetworkStatusDocument(content, False) # constructs without validation @@ -205,8 +118,7 @@ class TestNetworkStatusDocument(unittest.TestCase): Includes unrecognized content in the document. """ - content = get_network_status_document({"pepperjack": "is oh so tasty!"}) - document = NetworkStatusDocument(content) + document = get_network_status_document({"pepperjack": "is oh so tasty!"}) self.assertEquals(["pepperjack is oh so tasty!"], document.get_unrecognized_lines()) def test_misordered_fields(self): @@ -216,12 +128,12 @@ class TestNetworkStatusDocument(unittest.TestCase): for is_consensus in (True, False): attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"} - lines = get_network_status_document(attr).split("\n") + lines = get_network_status_document(attr, content = True).split("\n") for i in xrange(len(lines) - 1): # once we reach the crypto blob we're done since swapping those won't # be detected - if lines[i].startswith("e1XH33"): break + if lines[i].startswith(CRYPTO_BLOB[1:10]): break # swaps this line with the one after it test_lines = list(lines) @@ -239,7 +151,7 @@ class TestNetworkStatusDocument(unittest.TestCase): for is_consensus in (True, False): attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"} - lines = get_network_status_document(attr).split("\n") + lines = get_network_status_document(attr, content = True).split("\n") for i in xrange(len(lines)): # Stop when we hit the 'directory-signature' for a couple reasons... @@ -264,11 +176,10 @@ class TestNetworkStatusDocument(unittest.TestCase): different document version with the v3 parser. """ - content = get_network_status_document({"network-status-version": "3"}) - document = NetworkStatusDocument(content) + document = get_network_status_document({"network-status-version": "3"}) self.assertEquals("3", document.version) - content = get_network_status_document({"network-status-version": "4"}) + content = get_network_status_document({"network-status-version": "4"}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -279,12 +190,11 @@ class TestNetworkStatusDocument(unittest.TestCase): Parses the vote-status field. """ - content = get_network_status_document({"vote-status": "vote"}) - document = NetworkStatusDocument(content) + document = get_network_status_document({"vote-status": "vote"}) self.assertEquals(False, document.is_consensus) self.assertEquals(True, document.is_vote) - content = get_network_status_document({"vote-status": "consensus"}) + content = get_network_status_document({"vote-status": "consensus"}, content = True) document = NetworkStatusDocument(content) self.assertEquals(True, document.is_consensus) self.assertEquals(False, document.is_vote) @@ -296,7 +206,7 @@ class TestNetworkStatusDocument(unittest.TestCase): ) for test_value in test_values: - content = get_network_status_document({"vote-status": test_value}) + content = get_network_status_document({"vote-status": test_value}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -308,12 +218,11 @@ class TestNetworkStatusDocument(unittest.TestCase): Parses the consensus-methods field. """ - content = get_network_status_document({"vote-status": "vote", "consensus-methods": "12 3 1 780"}) - document = NetworkStatusDocument(content) + document = get_network_status_document({"vote-status": "vote", "consensus-methods": "12 3 1 780"}) self.assertEquals([12, 3, 1, 780], document.consensus_methods) # check that we default to including consensus-method 1 - content = get_network_status_document({"vote-status": "vote"}, ("consensus-methods",)) + content = get_network_status_document({"vote-status": "vote"}, ("consensus-methods",), content = True) document = NetworkStatusDocument(content, False) self.assertEquals([1], document.consensus_methods) self.assertEquals(None, document.consensus_method) @@ -327,7 +236,7 @@ class TestNetworkStatusDocument(unittest.TestCase): ) for test_value, expected_consensus_methods in test_values: - content = get_network_status_document({"vote-status": "vote", "consensus-methods": test_value}) + content = get_network_status_document({"vote-status": "vote", "consensus-methods": test_value}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -338,12 +247,11 @@ class TestNetworkStatusDocument(unittest.TestCase): Parses the consensus-method field. """ - content = get_network_status_document({"consensus-method": "12"}) - document = NetworkStatusDocument(content) + document = get_network_status_document({"consensus-method": "12"}) self.assertEquals(12, document.consensus_method) # check that we default to being consensus-method 1 - content = get_network_status_document(exclude = ("consensus-method",)) + content = get_network_status_document(exclude = ("consensus-method",), content = True) document = NetworkStatusDocument(content, False) self.assertEquals(1, document.consensus_method) self.assertEquals([], document.consensus_methods) @@ -357,7 +265,7 @@ class TestNetworkStatusDocument(unittest.TestCase): ) for test_value in test_values: - content = get_network_status_document({"consensus-method": test_value}) + content = get_network_status_document({"consensus-method": test_value}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -372,7 +280,7 @@ class TestNetworkStatusDocument(unittest.TestCase): expected = datetime.datetime(2012, 9, 2, 22, 0, 0) test_value = "2012-09-02 22:00:00" - content = get_network_status_document({ + document = get_network_status_document({ "vote-status": "vote", "published": test_value, "valid-after": test_value, @@ -380,7 +288,6 @@ class TestNetworkStatusDocument(unittest.TestCase): "valid-until": test_value, }) - document = NetworkStatusDocument(content) self.assertEquals(expected, document.published) self.assertEquals(expected, document.valid_after) self.assertEquals(expected, document.fresh_until) @@ -398,7 +305,7 @@ class TestNetworkStatusDocument(unittest.TestCase): attr = field.replace('-', '_') for test_value in test_values: - content = get_network_status_document({"vote-status": "vote", field: test_value}) + content = get_network_status_document({"vote-status": "vote", field: test_value}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -409,8 +316,7 @@ class TestNetworkStatusDocument(unittest.TestCase): Parses the voting-delay field. """ - content = get_network_status_document({"voting-delay": "12 345"}) - document = NetworkStatusDocument(content) + document = get_network_status_document({"voting-delay": "12 345"}) self.assertEquals(12, document.vote_delay) self.assertEquals(345, document.dist_delay) @@ -423,7 +329,7 @@ class TestNetworkStatusDocument(unittest.TestCase): ) for test_value in test_values: - content = get_network_status_document({"voting-delay": test_value}) + content = get_network_status_document({"voting-delay": test_value}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -439,8 +345,7 @@ class TestNetworkStatusDocument(unittest.TestCase): expected = [stem.version.Version("1.2.3.4"), stem.version.Version("56.789.12.34-alpha")] test_value = "1.2.3.4,56.789.12.34-alpha" - content = get_network_status_document({"client-versions": test_value, "server-versions": test_value}) - document = NetworkStatusDocument(content) + document = get_network_status_document({"client-versions": test_value, "server-versions": test_value}) self.assertEquals(expected, document.client_versions) self.assertEquals(expected, document.server_versions) @@ -455,7 +360,7 @@ class TestNetworkStatusDocument(unittest.TestCase): attr = field.replace('-', '_') for test_value, expected_value in test_values: - content = get_network_status_document({field: test_value}) + content = get_network_status_document({field: test_value}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -478,8 +383,7 @@ class TestNetworkStatusDocument(unittest.TestCase): ) for test_value, expected_value in test_values: - content = get_network_status_document({"known-flags": test_value}) - document = NetworkStatusDocument(content) + document = get_network_status_document({"known-flags": test_value}) self.assertEquals(expected_value, document.known_flags) def test_params(self): @@ -487,18 +391,17 @@ class TestNetworkStatusDocument(unittest.TestCase): General testing for the 'params' line, exercising the happy cases. """ - content = get_network_status_document({"params": "CircuitPriorityHalflifeMsec=30000 bwauthpid=1 unrecognized=-122"}) - document = NetworkStatusDocument(content) + document = get_network_status_document({"params": "CircuitPriorityHalflifeMsec=30000 bwauthpid=1 unrecognized=-122"}) self.assertEquals(30000, document.params["CircuitPriorityHalflifeMsec"]) self.assertEquals(1, document.params["bwauthpid"]) self.assertEquals(-122, document.params["unrecognized"]) # empty params line - content = get_network_status_document({"params": ""}) + content = get_network_status_document({"params": ""}, content = True) document = NetworkStatusDocument(content, default_params = True) self.assertEquals(DEFAULT_PARAMS, document.params) - content = get_network_status_document({"params": ""}) + content = get_network_status_document({"params": ""}, content = True) document = NetworkStatusDocument(content, default_params = False) self.assertEquals({}, document.params) @@ -515,7 +418,7 @@ class TestNetworkStatusDocument(unittest.TestCase): ) for test_value in test_values: - content = get_network_status_document({"params": test_value}) + content = get_network_status_document({"params": test_value}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -544,7 +447,7 @@ class TestNetworkStatusDocument(unittest.TestCase): ) for test_value, expected_value, is_ok in test_values: - content = get_network_status_document({"params": test_value}) + content = get_network_status_document({"params": test_value}, content = True) if is_ok: document = NetworkStatusDocument(content, default_params = False) @@ -559,7 +462,7 @@ class TestNetworkStatusDocument(unittest.TestCase): Check that the 'params' line is rejected if out of order. """ - content = get_network_status_document({"params": "unrecognized=-122 bwauthpid=1"}) + content = get_network_status_document({"params": "unrecognized=-122 bwauthpid=1"}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False, default_params = False) @@ -571,17 +474,16 @@ class TestNetworkStatusDocument(unittest.TestCase): introduced. """ - content = get_network_status_document({"consensus-method": "8"}) + content = get_network_status_document({"consensus-method": "8"}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) - self.assertEqual([SIG], document.signatures) + self.assertEqual([DOC_SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines()) # excludes a footer from a version that shouldn't have it - content = get_network_status_document({"consensus-method": "8"}, ("directory-footer", "directory-signature")) - document = NetworkStatusDocument(content) + document = get_network_status_document({"consensus-method": "8"}, ("directory-footer", "directory-signature")) self.assertEqual([], document.signatures) self.assertEqual([], document.get_unrecognized_lines()) @@ -590,11 +492,11 @@ class TestNetworkStatusDocument(unittest.TestCase): Tries to parse a descriptor with content on the 'directory-footer' line. """ - content = get_network_status_document({"directory-footer": "blarg"}) + content = get_network_status_document({"directory-footer": "blarg"}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) - self.assertEqual([SIG], document.signatures) + self.assertEqual([DOC_SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines()) def test_bandwidth_wights_ok(self): @@ -612,8 +514,7 @@ class TestNetworkStatusDocument(unittest.TestCase): weight_entries.append("%s=%i" % (key, value)) expected[key] = value - content = get_network_status_document({"bandwidth-weights": " ".join(weight_entries)}) - document = NetworkStatusDocument(content) + document = get_network_status_document({"bandwidth-weights": " ".join(weight_entries)}) self.assertEquals(expected, document.bandwidth_weights) def test_bandwidth_wights_malformed(self): @@ -633,7 +534,7 @@ class TestNetworkStatusDocument(unittest.TestCase): for test_value in test_values: weight_entry = base_weight_entry.replace("Wbe=5", test_value) - content = get_network_status_document({"bandwidth-weights": weight_entry}) + content = get_network_status_document({"bandwidth-weights": weight_entry}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -647,7 +548,7 @@ class TestNetworkStatusDocument(unittest.TestCase): weight_entry = " ".join(["%s=5" % e for e in reversed(BANDWIDTH_WEIGHT_ENTRIES)]) expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES]) - content = get_network_status_document({"bandwidth-weights": weight_entry}) + content = get_network_status_document({"bandwidth-weights": weight_entry}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -661,7 +562,7 @@ class TestNetworkStatusDocument(unittest.TestCase): weight_entry = " ".join(["%s=5" % e for e in BANDWIDTH_WEIGHT_ENTRIES]) expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES]) - content = get_network_status_document({"vote-status": "vote", "bandwidth-weights": weight_entry}) + content = get_network_status_document({"vote-status": "vote", "bandwidth-weights": weight_entry}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -674,7 +575,7 @@ class TestNetworkStatusDocument(unittest.TestCase): # try parsing an empty value - content = get_network_status_document({"bandwidth-weights": ""}) + content = get_network_status_document({"bandwidth-weights": ""}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -686,7 +587,7 @@ class TestNetworkStatusDocument(unittest.TestCase): weight_entries = ["%s=5" % e for e in BANDWIDTH_WEIGHT_ENTRIES if e != missing_entry] expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES if e != missing_entry]) - content = get_network_status_document({"bandwidth-weights": " ".join(weight_entries)}) + content = get_network_status_document({"bandwidth-weights": " ".join(weight_entries)}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) @@ -705,10 +606,10 @@ class TestNetworkStatusDocument(unittest.TestCase): for test_value in test_values: for test_attr in xrange(3): - attrs = [SIG.identity, SIG.key_digest, SIG.signature] + attrs = [DOC_SIG.identity, DOC_SIG.key_digest, DOC_SIG.signature] attrs[test_attr] = test_value - content = get_network_status_document({"directory-signature": "%s %s\n%s" % tuple(attrs)}) + content = get_network_status_document({"directory-signature": "%s %s\n%s" % tuple(attrs)}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) NetworkStatusDocument(content, False) # checks that it's still parseable without validation @@ -721,15 +622,14 @@ class TestNetworkStatusDocument(unittest.TestCase): entry1 = get_router_status_entry({'s': "Fast"}) entry2 = get_router_status_entry({'s': "Valid"}) - content = get_network_status_document(routers = (entry1, entry2)) + document = get_network_status_document(routers = (entry1, entry2)) - document = NetworkStatusDocument(content) self.assertEquals((entry1, entry2), document.routers) # try with an invalid RouterStatusEntry entry3 = RouterStatusEntry(get_router_status_entry({'r': "ugabuga"}, content = True), False) - content = get_network_status_document(routers = (entry3,)) + content = get_network_status_document(routers = (entry3,), content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False)

1 0

[stem/master] Moving router status entry mocking into util
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit d1ed7c2f9be307d8968ba56fd1906fcd44c1ab25 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Sep 27 09:40:06 2012 -0700 Moving router status entry mocking into util --- test/mocking.py | 33 +++++++- test/unit/descriptor/networkstatus/document.py | 12 ++-- test/unit/descriptor/networkstatus/entry.py | 98 +++++++---------------- 3 files changed, 65 insertions(+), 78 deletions(-) diff --git a/test/mocking.py b/test/mocking.py index c867664..2dd3319 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -27,6 +27,7 @@ calling :func:`test.mocking.revert_mocking`. get_bridge_server_descriptor - stem.descriptor.server_descriptor.BridgeDescriptor get_relay_extrainfo_descriptor - stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor get_bridge_extrainfo_descriptor - stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor + get_router_status_entry - stem.descriptor.networkstatus.RouterStatusEntry """ import inspect @@ -96,6 +97,11 @@ BRIDGE_EXTRAINFO_FOOTER = ( ("router-digest", "006FD96BA35E7785A6A3B8B75FE2E2435A13BDB4"), ) +ROUTER_STATUS_ENTRY_HEADER = ( + ("r", "caerSidi p1aag7VwarGxqctS7/fS0y5FU+s oQZFLYe9e4A7bOkWKR7TaNxb0JE 2012-08-06 11:19:31 71.35.150.29 9001 0"), + ("s", "Fast Named Running Stable Valid"), +) + def no_op(): def _no_op(*args): pass return _no_op @@ -399,7 +405,7 @@ def get_relay_server_descriptor(attr = None, exclude = (), content = False): if content: return desc_content else: - return stem.descriptor.server_descriptor.RelayDescriptor(desc_content, validate = False) + return stem.descriptor.server_descriptor.RelayDescriptor(desc_content, validate = True) def get_bridge_server_descriptor(attr = None, exclude = (), content = False): """ @@ -418,7 +424,7 @@ def get_bridge_server_descriptor(attr = None, exclude = (), content = False): if content: return desc_content else: - return stem.descriptor.server_descriptor.BridgeDescriptor(desc_content, validate = False) + return stem.descriptor.server_descriptor.BridgeDescriptor(desc_content, validate = True) def get_relay_extrainfo_descriptor(attr = None, exclude = (), content = False): """ @@ -437,7 +443,7 @@ def get_relay_extrainfo_descriptor(attr = None, exclude = (), content = False): if content: return desc_content else: - return stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor(desc_content, validate = False) + return stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor(desc_content, validate = True) def get_bridge_extrainfo_descriptor(attr = None, exclude = (), content = False): """ @@ -456,5 +462,24 @@ def get_bridge_extrainfo_descriptor(attr = None, exclude = (), content = False): if content: return desc_content else: - return stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(desc_content, validate = False) + return stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(desc_content, validate = True) + +def get_router_status_entry(attr = None, exclude = (), content = False): + """ + Provides the descriptor content for... + stem.descriptor.networkstatus.RouterStatusEntry + + :param dict attr: keyword/value mappings to be included in the descriptor + :param list exclude: mandatory keywords to exclude from the descriptor + :param bool content: provides the str content of the descriptor rather than the class if True + + :returns: RouterStatusEntry for the requested descriptor content + """ + + desc_content = _get_descriptor_content(attr, exclude, ROUTER_STATUS_ENTRY_HEADER) + + if content: + return desc_content + else: + return stem.descriptor.networkstatus.RouterStatusEntry(desc_content, validate = True) diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py index 42d2842..80fe6f2 100644 --- a/test/unit/descriptor/networkstatus/document.py +++ b/test/unit/descriptor/networkstatus/document.py @@ -9,7 +9,7 @@ import StringIO import stem.version from stem.descriptor import Flag from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, RouterStatusEntry, NetworkStatusDocument, DocumentSignature, parse_file -from test.unit.descriptor.networkstatus.entry import get_router_status_entry +from test.mocking import get_router_status_entry sig_block = """\ -----BEGIN SIGNATURE----- @@ -168,8 +168,8 @@ class TestNetworkStatusDocument(unittest.TestCase): Try parsing a document via the parse_file() function. """ - entry1 = RouterStatusEntry(get_router_status_entry({'s': "Fast"})) - entry2 = RouterStatusEntry(get_router_status_entry({'s': "Valid"})) + entry1 = get_router_status_entry({'s': "Fast"}) + entry2 = get_router_status_entry({'s': "Valid"}) content = get_network_status_document(routers = (entry1, entry2)) # the document that the entries refer to should actually be the minimal @@ -719,8 +719,8 @@ class TestNetworkStatusDocument(unittest.TestCase): document. """ - entry1 = RouterStatusEntry(get_router_status_entry({'s': "Fast"})) - entry2 = RouterStatusEntry(get_router_status_entry({'s': "Valid"})) + entry1 = get_router_status_entry({'s': "Fast"}) + entry2 = get_router_status_entry({'s': "Valid"}) content = get_network_status_document(routers = (entry1, entry2)) document = NetworkStatusDocument(content) @@ -728,7 +728,7 @@ class TestNetworkStatusDocument(unittest.TestCase): # try with an invalid RouterStatusEntry - entry3 = RouterStatusEntry(get_router_status_entry({'r': "ugabuga"}), False) + entry3 = RouterStatusEntry(get_router_status_entry({'r': "ugabuga"}, content = True), False) content = get_network_status_document(routers = (entry3,)) self.assertRaises(ValueError, NetworkStatusDocument, content) diff --git a/test/unit/descriptor/networkstatus/entry.py b/test/unit/descriptor/networkstatus/entry.py index 6380c62..a0ece4d 100644 --- a/test/unit/descriptor/networkstatus/entry.py +++ b/test/unit/descriptor/networkstatus/entry.py @@ -9,40 +9,7 @@ from stem.descriptor import Flag from stem.descriptor.networkstatus import RouterStatusEntry, _decode_fingerprint from stem.version import Version from stem.exit_policy import MicrodescriptorExitPolicy - -ROUTER_STATUS_ENTRY_ATTR = ( - ("r", "caerSidi p1aag7VwarGxqctS7/fS0y5FU+s oQZFLYe9e4A7bOkWKR7TaNxb0JE 2012-08-06 11:19:31 71.35.150.29 9001 0"), - ("s", "Fast Named Running Stable Valid"), -) - -def get_router_status_entry(attr = None, exclude = None): - """ - Constructs a minimal router status entry with the given attributes. - - :param dict attr: keyword/value mappings to be included in the entry - :param list exclude: mandatory keywords to exclude from the entry - - :returns: str with customized router status entry content - """ - - descriptor_lines = [] - if attr is None: attr = {} - if exclude is None: exclude = [] - attr = dict(attr) # shallow copy since we're destructive - - for keyword, value in ROUTER_STATUS_ENTRY_ATTR: - if keyword in exclude: continue - elif keyword in attr: - value = attr[keyword] - del attr[keyword] - - descriptor_lines.append("%s %s" % (keyword, value)) - - # dump in any unused attributes - for attr_keyword, attr_value in attr.items(): - descriptor_lines.append("%s %s" % (attr_keyword, attr_value)) - - return "\n".join(descriptor_lines) +from test.mocking import get_router_status_entry, ROUTER_STATUS_ENTRY_HEADER class TestRouterStatusEntry(unittest.TestCase): def test_fingerprint_decoding(self): @@ -73,7 +40,7 @@ class TestRouterStatusEntry(unittest.TestCase): Parses a minimal router status entry. """ - entry = RouterStatusEntry(get_router_status_entry()) + entry = get_router_status_entry() expected_flags = set([Flag.FAST, Flag.NAMED, Flag.RUNNING, Flag.STABLE, Flag.VALID]) self.assertEqual(None, entry.document) @@ -99,13 +66,13 @@ class TestRouterStatusEntry(unittest.TestCase): Parses a router status entry that's missing fields. """ - content = get_router_status_entry(exclude = ('r', 's')) + content = get_router_status_entry(exclude = ('r', 's'), content = True) self._expect_invalid_attr(content, "address") - content = get_router_status_entry(exclude = ('r',)) + content = get_router_status_entry(exclude = ('r',), content = True) self._expect_invalid_attr(content, "address") - content = get_router_status_entry(exclude = ('s',)) + content = get_router_status_entry(exclude = ('s',), content = True) self._expect_invalid_attr(content, "flags") def test_unrecognized_lines(self): @@ -113,8 +80,7 @@ class TestRouterStatusEntry(unittest.TestCase): Parses a router status entry with new keywords. """ - content = get_router_status_entry({'z': 'New tor feature: sparkly unicorns!'}) - entry = RouterStatusEntry(content) + entry = get_router_status_entry({'z': 'New tor feature: sparkly unicorns!'}) self.assertEquals(['z New tor feature: sparkly unicorns!'], entry.get_unrecognized_lines()) def test_proceeding_line(self): @@ -122,7 +88,7 @@ class TestRouterStatusEntry(unittest.TestCase): Includes content prior to the 'r' line. """ - content = 'z some stuff\n' + get_router_status_entry() + content = 'z some stuff\n' + get_router_status_entry(content = True) self._expect_invalid_attr(content, "_unrecognized_lines", ['z some stuff']) def test_blank_lines(self): @@ -130,7 +96,7 @@ class TestRouterStatusEntry(unittest.TestCase): Includes blank lines, which should be ignored. """ - content = get_router_status_entry() + "\n\nv Tor 0.2.2.35\n\n" + content = get_router_status_entry(content = True) + "\n\nv Tor 0.2.2.35\n\n" entry = RouterStatusEntry(content) self.assertEqual("Tor 0.2.2.35", entry.version_line) @@ -155,7 +121,7 @@ class TestRouterStatusEntry(unittest.TestCase): test_components.remove(value) r_line = ' '.join(test_components) - content = get_router_status_entry({'r': r_line}) + content = get_router_status_entry({'r': r_line}, content = True) self._expect_invalid_attr(content, attr) def test_malformed_nickname(self): @@ -170,8 +136,8 @@ class TestRouterStatusEntry(unittest.TestCase): ) for value in test_values: - r_line = ROUTER_STATUS_ENTRY_ATTR[0][1].replace("caerSidi", value) - content = get_router_status_entry({'r': r_line}) + r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("caerSidi", value) + content = get_router_status_entry({'r': r_line}, content = True) # TODO: Initial whitespace is consumed as part of the keyword/value # divider. This is a bug in the case of V3 router status entries, but @@ -199,8 +165,8 @@ class TestRouterStatusEntry(unittest.TestCase): ) for value in test_values: - r_line = ROUTER_STATUS_ENTRY_ATTR[0][1].replace("p1aag7VwarGxqctS7/fS0y5FU+s", value) - content = get_router_status_entry({'r': r_line}) + r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("p1aag7VwarGxqctS7/fS0y5FU+s", value) + content = get_router_status_entry({'r': r_line}, content = True) self._expect_invalid_attr(content, "fingerprint") def test_malformed_published_date(self): @@ -225,8 +191,8 @@ class TestRouterStatusEntry(unittest.TestCase): ) for value in test_values: - r_line = ROUTER_STATUS_ENTRY_ATTR[0][1].replace("2012-08-06 11:19:31", value) - content = get_router_status_entry({'r': r_line}) + r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("2012-08-06 11:19:31", value) + content = get_router_status_entry({'r': r_line}, content = True) self._expect_invalid_attr(content, "published") def test_malformed_address(self): @@ -243,8 +209,8 @@ class TestRouterStatusEntry(unittest.TestCase): ) for value in test_values: - r_line = ROUTER_STATUS_ENTRY_ATTR[0][1].replace("71.35.150.29", value) - content = get_router_status_entry({'r': r_line}) + r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("71.35.150.29", value) + content = get_router_status_entry({'r': r_line}, content = True) self._expect_invalid_attr(content, "address", value) def test_malformed_port(self): @@ -265,7 +231,7 @@ class TestRouterStatusEntry(unittest.TestCase): if not include_or_port and not include_dir_port: continue - r_line = ROUTER_STATUS_ENTRY_ATTR[0][1] + r_line = ROUTER_STATUS_ENTRY_HEADER[0][1] if include_or_port: r_line = r_line.replace(" 9001 ", " %s " % value) @@ -276,7 +242,7 @@ class TestRouterStatusEntry(unittest.TestCase): attr = "or_port" if include_or_port else "dir_port" expected = int(value) if value.isdigit() else None - content = get_router_status_entry({'r': r_line}) + content = get_router_status_entry({'r': r_line}, content = True) self._expect_invalid_attr(content, attr, expected) def test_flags(self): @@ -292,8 +258,7 @@ class TestRouterStatusEntry(unittest.TestCase): } for s_line, expected in test_values.items(): - content = get_router_status_entry({'s': s_line}) - entry = RouterStatusEntry(content) + entry = get_router_status_entry({'s': s_line}) self.assertEquals(expected, entry.flags) # tries some invalid inputs @@ -304,7 +269,7 @@ class TestRouterStatusEntry(unittest.TestCase): } for s_line, expected in test_values.items(): - content = get_router_status_entry({'s': s_line}) + content = get_router_status_entry({'s': s_line}, content = True) self._expect_invalid_attr(content, "flags", expected) def test_versions(self): @@ -320,13 +285,12 @@ class TestRouterStatusEntry(unittest.TestCase): } for v_line, expected in test_values.items(): - content = get_router_status_entry({'v': v_line}) - entry = RouterStatusEntry(content) + entry = get_router_status_entry({'v': v_line}) self.assertEquals(expected, entry.version) self.assertEquals(v_line, entry.version_line) # tries an invalid input - content = get_router_status_entry({'v': "Tor ugabuga"}) + content = get_router_status_entry({'v': "Tor ugabuga"}, content = True) self._expect_invalid_attr(content, "version") def test_bandwidth(self): @@ -342,8 +306,7 @@ class TestRouterStatusEntry(unittest.TestCase): } for w_line, expected in test_values.items(): - content = get_router_status_entry({'w': w_line}) - entry = RouterStatusEntry(content) + entry = get_router_status_entry({'w': w_line}) self.assertEquals(expected[0], entry.bandwidth) self.assertEquals(expected[1], entry.measured) self.assertEquals(expected[2], entry.unrecognized_bandwidth_entries) @@ -363,7 +326,7 @@ class TestRouterStatusEntry(unittest.TestCase): ) for w_line in test_values: - content = get_router_status_entry({'w': w_line}) + content = get_router_status_entry({'w': w_line}, content = True) self._expect_invalid_attr(content) def test_exit_policy(self): @@ -377,8 +340,7 @@ class TestRouterStatusEntry(unittest.TestCase): } for p_line, expected in test_values.items(): - content = get_router_status_entry({'p': p_line}) - entry = RouterStatusEntry(content) + entry = get_router_status_entry({'p': p_line}) self.assertEquals(expected, entry.exit_policy) # tries some invalid inputs @@ -390,7 +352,7 @@ class TestRouterStatusEntry(unittest.TestCase): ) for p_line in test_values: - content = get_router_status_entry({'p': p_line}) + content = get_router_status_entry({'p': p_line}, content = True) self._expect_invalid_attr(content, "exit_policy") def test_microdescriptor_hashes(self): @@ -413,12 +375,12 @@ class TestRouterStatusEntry(unittest.TestCase): mock_document.__dict__["is_consensus"] = False for m_line, expected in test_values.items(): - content = get_router_status_entry({'m': m_line}) + content = get_router_status_entry({'m': m_line}, content = True) entry = RouterStatusEntry(content, document = mock_document) self.assertEquals(expected, entry.microdescriptor_hashes) # try without a document - content = get_router_status_entry({'m': "8,9,10,11,12"}) + content = get_router_status_entry({'m': "8,9,10,11,12"}, content = True) self._expect_invalid_attr(content, "microdescriptor_hashes") # tries some invalid inputs @@ -429,7 +391,7 @@ class TestRouterStatusEntry(unittest.TestCase): ) for m_line in test_values: - content = get_router_status_entry({'m': m_line}) + content = get_router_status_entry({'m': m_line}, content = True) self.assertRaises(ValueError, RouterStatusEntry, content, True, mock_document) def _expect_invalid_attr(self, content, attr = None, expected_value = None):

1 0

[stem/master] Boilerplate KeyCertificate unit tests
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 92b691c5e9ace6803a5ed24d42e7ec41665595c8 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 29 10:19:16 2012 -0700 Boilerplate KeyCertificate unit tests General unit tests that I've been including with most descriptor types. I should probably include these in a more systematic fasion... --- stem/descriptor/networkstatus.py | 4 +- .../descriptor/networkstatus/key_certificate.py | 52 +++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 01fb2f3..ad74442 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -877,9 +877,11 @@ class KeyCertificate(stem.descriptor.Descriptor): raise ValueError("Key certificate's address isn't a valid IPv4 address: %s" % line) elif not stem.util.connection.is_valid_port(dirport): raise ValueError("Key certificate's dirport is invalid: %s" % line) + elif not dirport.isdigit(): + continue self.address = address - self.dir_port = dirport + self.dir_port = int(dirport) elif keyword == 'fingerprint': # "fingerprint" fingerprint diff --git a/test/unit/descriptor/networkstatus/key_certificate.py b/test/unit/descriptor/networkstatus/key_certificate.py index 703b61d..e3f62e0 100644 --- a/test/unit/descriptor/networkstatus/key_certificate.py +++ b/test/unit/descriptor/networkstatus/key_certificate.py @@ -6,7 +6,7 @@ import datetime import unittest from stem.descriptor.networkstatus import KeyCertificate -from test.mocking import get_key_certificate, CRYPTO_BLOB +from test.mocking import get_key_certificate, CRYPTO_BLOB, KEY_CERTIFICATE_HEADER, KEY_CERTIFICATE_FOOTER class TestKeyCertificate(unittest.TestCase): def test_minimal(self): @@ -27,4 +27,54 @@ class TestKeyCertificate(unittest.TestCase): self.assertEqual(None, certificate.crosscert) self.assertTrue(CRYPTO_BLOB in certificate.certification) self.assertEqual([], certificate.get_unrecognized_lines()) + + def test_unrecognized_line(self): + """ + Includes unrecognized content in the descriptor. + """ + + certificate = get_key_certificate({"pepperjack": "is oh so tasty!"}) + self.assertEquals(["pepperjack is oh so tasty!"], certificate.get_unrecognized_lines()) + + def test_first_and_last_lines(self): + """ + Includes a non-mandatory field before the 'dir-key-certificate-version' + line or after the 'dir-key-certification' line. + """ + + content = get_key_certificate(content = True) + + for cert_text in ("dir-address 127.0.0.1:80\n" + content, + content + "\ndir-address 127.0.0.1:80"): + self.assertRaises(ValueError, KeyCertificate, cert_text) + + certificate = KeyCertificate(cert_text, False) + self.assertEqual("127.0.0.1", certificate.address) + self.assertEqual(80, certificate.dir_port) + + def test_missing_fields(self): + """ + Parse a key certificate where a mandatory field is missing. + """ + + mandatory_fields = [entry[0] for entry in KEY_CERTIFICATE_HEADER + KEY_CERTIFICATE_FOOTER] + + for excluded_field in mandatory_fields: + content = get_key_certificate(exclude = (excluded_field,), content = True) + self.assertRaises(ValueError, KeyCertificate, content) + + certificate = KeyCertificate(content, False) + + if excluded_field == "fingerprint": + self.assertEqual(3, certificate.version) + else: + self.assertEqual("27B6B5996C426270A5C95488AA5BCEB6BCC86956", certificate.fingerprint) + + def test_blank_lines(self): + """ + Includes blank lines, which should be ignored. + """ + + certificate = get_key_certificate({"dir-key-published": "2011-11-28 21:51:04\n\n\n"}) + self.assertEqual(datetime.datetime(2011, 11, 28, 21, 51, 4), certificate.published)

1 0

[stem/master] Dropping a coupel unused imports
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 50cc258286cd31da9571f8010f5f24b2e56bf386 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Sep 28 09:32:21 2012 -0700 Dropping a coupel unused imports Ran pylint over the recently refactored files. It caught a couple of unused imports. --- test/unit/descriptor/extrainfo_descriptor.py | 2 +- test/unit/descriptor/networkstatus/document.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index e8d4cac..7ae38a7 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -4,7 +4,7 @@ Unit tests for stem.descriptor.extrainfo_descriptor. import datetime import unittest -from stem.descriptor.extrainfo_descriptor import RelayExtraInfoDescriptor, BridgeExtraInfoDescriptor, DirResponses, DirStats +from stem.descriptor.extrainfo_descriptor import RelayExtraInfoDescriptor, DirResponses, DirStats from test.mocking import get_relay_extrainfo_descriptor, get_bridge_extrainfo_descriptor, CRYPTO_BLOB class TestExtraInfoDescriptor(unittest.TestCase): diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py index 8a01ec6..0d9f237 100644 --- a/test/unit/descriptor/networkstatus/document.py +++ b/test/unit/descriptor/networkstatus/document.py @@ -8,7 +8,7 @@ import StringIO import stem.version from stem.descriptor import Flag -from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, RouterStatusEntry, NetworkStatusDocument, DocumentSignature, parse_file +from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, RouterStatusEntry, NetworkStatusDocument, parse_file from test.mocking import get_router_status_entry, get_network_status_document, CRYPTO_BLOB, DOC_SIG class TestNetworkStatusDocument(unittest.TestCase):

1 0

[stem/master] Unit tests for KeyCertificate fields
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 7986afa40937657391346dea5b4aa6aeb561c870 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 29 11:13:34 2012 -0700 Unit tests for KeyCertificate fields Remaining unit tests that I'm planning to do for the KeyCertificate. Oddly this is the first time that I've added a unit test for malformed content in a key block field (oops). --- stem/descriptor/__init__.py | 5 +- .../descriptor/networkstatus/key_certificate.py | 109 ++++++++++++++++++++ 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index b921058..9fa1f9e 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -340,15 +340,16 @@ def _get_pseudo_pgp_block(remaining_contents): if block_match: block_type = block_match.groups()[0] block_lines = [] + end_line = PGP_BLOCK_END % block_type while True: if not remaining_contents: - raise ValueError("Unterminated pgp style block") + raise ValueError("Unterminated pgp style block (looking for '%s'):\n%s" % (end_line, "\n".join(block_lines))) line = remaining_contents.pop(0) block_lines.append(line) - if line == PGP_BLOCK_END % block_type: + if line == end_line: return "\n".join(block_lines) else: return None diff --git a/test/unit/descriptor/networkstatus/key_certificate.py b/test/unit/descriptor/networkstatus/key_certificate.py index e3f62e0..e37337b 100644 --- a/test/unit/descriptor/networkstatus/key_certificate.py +++ b/test/unit/descriptor/networkstatus/key_certificate.py @@ -77,4 +77,113 @@ class TestKeyCertificate(unittest.TestCase): certificate = get_key_certificate({"dir-key-published": "2011-11-28 21:51:04\n\n\n"}) self.assertEqual(datetime.datetime(2011, 11, 28, 21, 51, 4), certificate.published) + + def test_version(self): + """ + Parses the dir-key-certificate-version field, including trying to handle a + different certificate version with the v3 parser. + """ + + certificate = get_key_certificate({"dir-key-certificate-version": "3"}) + self.assertEquals(3, certificate.version) + + content = get_key_certificate({"dir-key-certificate-version": "4"}, content = True) + self.assertRaises(ValueError, KeyCertificate, content) + self.assertEquals(4, KeyCertificate(content, False).version) + + content = get_key_certificate({"dir-key-certificate-version": "boo"}, content = True) + self.assertRaises(ValueError, KeyCertificate, content) + self.assertEquals(None, KeyCertificate(content, False).version) + + def test_dir_address(self): + """ + Parses the dir-address field. + """ + + certificate = get_key_certificate({"dir-address": "127.0.0.1:80"}) + self.assertEqual("127.0.0.1", certificate.address) + self.assertEqual(80, certificate.dir_port) + + test_values = ( + ("", None, None), + (" ", None, None), + ("127.0.0.1", None, None), + ("127.0.0.1:", None, None), + ("80", None, None), + (":80", "", 80), + ("127.0.0.1a:80", "127.0.0.1a", 80), + ("127.0.0.1:80a", None, None), + ) + + for test_value, expected_address, expected_port in test_values: + content = get_key_certificate({"dir-address": test_value}, content = True) + self.assertRaises(ValueError, KeyCertificate, content) + + certificate = KeyCertificate(content, False) + self.assertEqual(expected_address, certificate.address) + self.assertEqual(expected_port, certificate.dir_port) + + def test_fingerprint(self): + """ + Parses the fingerprint field. + """ + + test_values = ( + "", + " ", + "27B6B5996C426270A5C95488AA5BCEB6BCC8695", + "27B6B5996C426270A5C95488AA5BCEB6BCC869568", + ) + + for test_value in test_values: + content = get_key_certificate({"fingerprint": test_value}, content = True) + self.assertRaises(ValueError, KeyCertificate, content) + + certificate = KeyCertificate(content, False) + self.assertEqual(test_value.strip(), certificate.fingerprint) + + def test_time_fields(self): + """ + Parses the dir-key-published and dir-key-expires fields, which both have + datetime content. + """ + + test_values = ( + "", + " ", + "2012-12-12", + "2012-12-12 01:01:", + "2012-12-12 01:a1:01", + ) + + for field, attr in (("dir-key-published", "published"), ("dir-key-expires", "expires")): + for test_value in test_values: + content = get_key_certificate({field: test_value}, content = True) + self.assertRaises(ValueError, KeyCertificate, content) + + certificate = KeyCertificate(content, False) + self.assertEquals(None, getattr(certificate, attr)) + + def test_key_blocks(self): + """ + Parses the dir-identity-key, dir-signing-key, dir-key-crosscert, and + dir-key-certification fields which all just have signature content. + """ + + # the only non-mandatory field that we haven't exercised yet is dir-key-crosscert + + certificate = get_key_certificate({"dir-key-crosscert": "\n-----BEGIN ID SIGNATURE-----%s-----END ID SIGNATURE-----" % CRYPTO_BLOB}) + self.assertTrue(CRYPTO_BLOB in certificate.crosscert) + + test_value = "\n-----BEGIN ID SIGNATURE-----%s-----END UGABUGA SIGNATURE-----" % CRYPTO_BLOB + + for field, attr in (('dir-identity-key', 'identity_key'), + ('dir-signing-key', 'signing_key'), + ('dir-key-crosscert', 'crosscert'), + ('dir-key-certification', 'certification')): + content = get_key_certificate({field: test_value}, content = True) + self.assertRaises(ValueError, KeyCertificate, content) + + certificate = KeyCertificate(content, False) + self.assertEquals(None, getattr(certificate, attr))

1 0

[stem/master] Unit tests for minimal DirectoryAuthority instances
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 11070d9a91e0c00b4b5854fc935c5fc7a871e97f Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 29 12:41:24 2012 -0700 Unit tests for minimal DirectoryAuthority instances Adding unit tests for the minimal vote and consensus directory authority entry. Presently this is just exercising the old parsing code (which I broke in a few places during some of my earlier refactoring). --- run_tests.py | 2 + stem/descriptor/networkstatus.py | 29 +++++++----- test/mocking.py | 38 ++++++++++++++++ test/unit/descriptor/networkstatus/__init__.py | 2 +- .../networkstatus/directory_authority.py | 47 ++++++++++++++++++++ 5 files changed, 105 insertions(+), 13 deletions(-) diff --git a/run_tests.py b/run_tests.py index 54e74b2..8f76dd3 100755 --- a/run_tests.py +++ b/run_tests.py @@ -21,6 +21,7 @@ import test.unit.descriptor.reader import test.unit.descriptor.server_descriptor import test.unit.descriptor.extrainfo_descriptor import test.unit.descriptor.networkstatus.entry +import test.unit.descriptor.networkstatus.directory_authority import test.unit.descriptor.networkstatus.key_certificate import test.unit.descriptor.networkstatus.document import test.unit.response.control_line @@ -118,6 +119,7 @@ UNIT_TESTS = ( test.unit.descriptor.server_descriptor.TestServerDescriptor, test.unit.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, test.unit.descriptor.networkstatus.entry.TestRouterStatusEntry, + test.unit.descriptor.networkstatus.directory_authority.TestDirectoryAuthority, test.unit.descriptor.networkstatus.key_certificate.TestKeyCertificate, test.unit.descriptor.networkstatus.document.TestNetworkStatusDocument, test.unit.exit_policy.rule.TestExitPolicyRule, diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 7c95f96..8147a26 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -699,6 +699,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): self.nickname = None self.fingerprint = None + self.hostname = None self.address = None self.dir_port = None self.or_port = None @@ -712,7 +713,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): self._unrecognized_lines = [] #self._parse(raw_contents, validate, is_vote) - self._parse_old(raw_contents, validate, is_vote) + self._parse_old(raw_content, validate, is_vote) def _parse(self, content, validate, is_vote): """ @@ -753,26 +754,24 @@ class DirectoryAuthority(stem.descriptor.Descriptor): #if validate and len(values) > 1 and keyword in ('r', 's', 'v', 'w', 'p'): # raise ValueError("Router status entries can only have a single '%s' line, got %i:\n%s" % (key, len(values), content)) - def _parse_old(self, content, validate, is_vote): - self.nickname, self.fingerprint, self.address, self.ip = None, None, None, None - self.dir_port, self.or_port, self.legacy_dir_key = None, None, None - self.key_certificate, self.contact, self.vote_digest = None, None, None - + def _parse_old(self, raw_content, validate, is_vote): content = StringIO(raw_content) dir_source = _read_keyword_line("dir-source", content, validate) - self.nickname, self.fingerprint, self.address, self.ip, self.dir_port, self.or_port = dir_source.split(" ") + self.nickname, self.fingerprint, self.hostname, self.address, self.dir_port, self.or_port = dir_source.split(" ") self.dir_port = int(self.dir_port) self.or_port = int(self.or_port) self.contact = _read_keyword_line("contact", content, validate) - if vote: + if is_vote: self.legacy_dir_key = _read_keyword_line("legacy-dir-key", content, validate, True) self.key_certificate = KeyCertificate(content.read(), validate) else: self.vote_digest = _read_keyword_line("vote-digest", content, True, validate) - self.unrecognized_lines = content.read() - if self.unrecognized_lines and validate: - raise ValueError("Unrecognized trailing data in directory authority information") + + remainder = content.read() + + if remainder: + self._unrecognized_lines = remainder.split("\n") def get_unrecognized_lines(self): """ @@ -781,7 +780,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): :returns: a list of unrecognized lines """ - return self.unrecognized_lines + return self._unrecognized_lines class KeyCertificate(stem.descriptor.Descriptor): """ @@ -931,6 +930,12 @@ class KeyCertificate(stem.descriptor.Descriptor): """ return self._unrecognized_lines + + def __cmp__(self, other): + if not isinstance(other, KeyCertificate): + return 1 + + return str(self) > str(other) # TODO: microdescriptors have a slightly different format (including a # 'method') - should probably be a subclass diff --git a/test/mocking.py b/test/mocking.py index 7008a2a..89ea2f6 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -28,7 +28,9 @@ calling :func:`test.mocking.revert_mocking`. get_relay_extrainfo_descriptor - stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor get_bridge_extrainfo_descriptor - stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor get_router_status_entry - stem.descriptor.networkstatus.RouterStatusEntry + get_directory_authority - stem.descriptor.networkstatus.DirectoryAuthority get_key_certificate - stem.descriptor.networkstatus.KeyCertificate + get_network_status_document - stem.descriptor.networkstatus.NetworkStatusDocument """ import inspect @@ -109,6 +111,11 @@ ROUTER_STATUS_ENTRY_HEADER = ( ("s", "Fast Named Running Stable Valid"), ) +AUTHORITY_HEADER = ( + ("dir-source", "turtles 27B6B5996C426270A5C95488AA5BCEB6BCC86956 no.place.com 76.73.17.194 9030 9090"), + ("contact", "Mike Perry <email>"), +) + KEY_CERTIFICATE_HEADER = ( ("dir-key-certificate-version", "3"), ("fingerprint", "27B6B5996C426270A5C95488AA5BCEB6BCC86956"), @@ -534,6 +541,37 @@ def get_router_status_entry(attr = None, exclude = (), content = False): else: return stem.descriptor.networkstatus.RouterStatusEntry(desc_content, validate = True) +def get_directory_authority(attr = None, exclude = (), is_vote = False, content = False): + """ + Provides the descriptor content for... + stem.descriptor.networkstatus.DirectoryAuthority + + :param dict attr: keyword/value mappings to be included in the descriptor + :param list exclude: mandatory keywords to exclude from the descriptor + :param bool is_vote: True if this is for a vote, False if it's for a consensus + :param bool content: provides the str content of the descriptor rather than the class if True + + :returns: DirectoryAuthority for the requested descriptor content + """ + + if attr is None: + attr = {} + + if not is_vote: + # entries from a consensus also have a mandatory 'vote-digest' field + if not ('vote-digest' in attr or (exclude and 'vote-digest' in exclude)): + attr['vote-digest'] = '0B6D1E9A300B895AA2D0B427F92917B6995C3C1C' + + desc_content = _get_descriptor_content(attr, exclude, AUTHORITY_HEADER) + + if is_vote: + desc_content += "\n" + str(get_key_certificate()) + + if content: + return desc_content + else: + return stem.descriptor.networkstatus.DirectoryAuthority(desc_content, validate = True, is_vote = is_vote) + def get_key_certificate(attr = None, exclude = (), content = False): """ Provides the descriptor content for... diff --git a/test/unit/descriptor/networkstatus/__init__.py b/test/unit/descriptor/networkstatus/__init__.py index d8c9657..b2314cc 100644 --- a/test/unit/descriptor/networkstatus/__init__.py +++ b/test/unit/descriptor/networkstatus/__init__.py @@ -2,5 +2,5 @@ Unit tests for stem.descriptor.networkstatus. """ -__all__ = ["entry", "key_certificate", "document"] +__all__ = ["entry", "directory_authority", "key_certificate", "document"] diff --git a/test/unit/descriptor/networkstatus/directory_authority.py b/test/unit/descriptor/networkstatus/directory_authority.py new file mode 100644 index 0000000..dd4c5fa --- /dev/null +++ b/test/unit/descriptor/networkstatus/directory_authority.py @@ -0,0 +1,47 @@ +""" +Unit tests for the DirectoryAuthority of stem.descriptor.networkstatus. +""" + +import unittest + +from test.mocking import get_directory_authority, get_key_certificate + +class TestDirectoryAuthority(unittest.TestCase): + def test_minimal_consensus_authority(self): + """ + Parses a minimal directory authority for a consensus. + """ + + authority = get_directory_authority() + + self.assertEqual("turtles", authority.nickname) + self.assertEqual("27B6B5996C426270A5C95488AA5BCEB6BCC86956", authority.fingerprint) + self.assertEqual("no.place.com", authority.hostname) + self.assertEqual("76.73.17.194", authority.address) + self.assertEqual(9030, authority.dir_port) + self.assertEqual(9090, authority.or_port) + self.assertEqual("Mike Perry <email>", authority.contact) + self.assertEqual("0B6D1E9A300B895AA2D0B427F92917B6995C3C1C", authority.vote_digest) + self.assertEqual(None, authority.legacy_dir_key) + self.assertEqual(None, authority.key_certificate) + self.assertEqual([], authority.get_unrecognized_lines()) + + def test_minimal_vote_authority(self): + """ + Parses a minimal directory authority for a vote. + """ + + authority = get_directory_authority(is_vote = True) + + self.assertEqual("turtles", authority.nickname) + self.assertEqual("27B6B5996C426270A5C95488AA5BCEB6BCC86956", authority.fingerprint) + self.assertEqual("no.place.com", authority.hostname) + self.assertEqual("76.73.17.194", authority.address) + self.assertEqual(9030, authority.dir_port) + self.assertEqual(9090, authority.or_port) + self.assertEqual("Mike Perry <email>", authority.contact) + self.assertEqual(None, authority.vote_digest) + self.assertEqual(None, authority.legacy_dir_key) + self.assertEqual(get_key_certificate(), authority.key_certificate) + self.assertEqual([], authority.get_unrecognized_lines()) +

1 0

[stem/master] Dropping the old KeyCertificate class
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit e52f0ec96b01db7da8002479b9a864a6f18f6db8 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 29 11:17:09 2012 -0700 Dropping the old KeyCertificate class We have a new shiny and tested KeyCertificate class so dropping the old one. --- stem/descriptor/__init__.py | 92 -------------------------------------- stem/descriptor/networkstatus.py | 4 +- 2 files changed, 2 insertions(+), 94 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 9fa1f9e..362500a 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -441,95 +441,3 @@ def _strptime(string, validate = True, optional = False): def line_matches_keyword(keyword, line): return re.search("^(opt )?" + re.escape(keyword) + "($| )", line) -class KeyCertificate(Descriptor): - """ - Directory key certificate. - - :var str key_certificate_version: **\*** version of the key certificate (Should be "3") - :var str ip: IP address on which the directory authority is listening - :var int port: port on which the directory authority is listening - :var str fingerprint: **\*** hex encoded fingerprint of the authority's identity key - :var str identity_key: **\*** long term authority identity key - :var datetime published: **\*** time (in GMT) when this document & the key were last generated - :var str expires: **\*** time (in GMT) after which this key becomes invalid - :var str signing_key: **\*** directory server's public signing key - :var str crosscert: signature made using certificate's signing key - :var str certification: **\*** signature of this key certificate signed with the identity key - - **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined - """ - - def __init__(self, raw_content, validate = True): - """ - Parse a key certificate entry and provide a KeyCertificate object. - - :param str raw_content: raw key certificate information - :param bool validate: True if the document is to be validated, False otherwise - - :raises: ValueError if the raw data is invalid - """ - - super(KeyCertificate, self).__init__(raw_content) - self.key_certificate_version, self.ip, self.port = None, None, None - self.fingerprint, self.identity_key, self.published = None, None, None - self.expires, self.signing_key, self.crosscert = None, None, None - self.certification = None - content = raw_content.splitlines() - seen_keywords = set() - - self.key_certificate_version = _read_keyword_line_str("dir-key-certificate-version", content) - if validate and self.key_certificate_version != "3": - raise ValueError("Unrecognized dir-key-certificate-version") - - def read_keyword_line(keyword): - if validate and keyword in seen_keywords: - raise ValueError("Invalid key certificate: '%s' appears twice" % keyword) - seen_keywords.add(keyword) - return _read_keyword_line_str(keyword, content, validate) - - while content: - if line_matches_keyword("dir-address", content[0]): - line = read_keyword_line("dir-address") - try: - self.ip, self.port = line.rsplit(":", 1) - self.port = int(self.port) - except Exception: - if validate: raise ValueError("Invalid dir-address line: %s" % line) - elif line_matches_keyword("fingerprint", content[0]): - self.fingerprint = read_keyword_line("fingerprint") - elif line_matches_keyword("dir-identity-key", content[0]): - read_keyword_line("dir-identity-key") - self.identity_key = _get_pseudo_pgp_block(content) - elif line_matches_keyword("dir-key-published", content[0]): - self.published = _strptime(read_keyword_line("dir-key-published")) - elif line_matches_keyword("dir-key-expires", content[0]): - self.expires = _strptime(read_keyword_line("dir-key-expires")) - elif line_matches_keyword("dir-signing-key", content[0]): - read_keyword_line("dir-signing-key") - self.signing_key = _get_pseudo_pgp_block(content) - elif line_matches_keyword("dir-key-crosscert", content[0]): - read_keyword_line("dir-key-crosscert") - self.crosscert = _get_pseudo_pgp_block(content) - elif line_matches_keyword("dir-key-certification", content[0]): - read_keyword_line("dir-key-certification") - self.certification = _get_pseudo_pgp_block(content) - break - elif validate: - raise ValueError("Key certificate contains unrecognized lines: %s" % content[0]) - else: - # ignore unrecognized lines if we aren't validating - self.unrecognized_lines.append(content.pop(0)) - - self.unrecognized_lines = content - if self.unrecognized_lines and validate: - raise ValueError("Unrecognized trailing data in key certificate") - - def get_unrecognized_lines(self): - """ - Returns any unrecognized lines. - - :returns: a list of unrecognized lines - """ - - return self.unrecognized_lines - diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index ad74442..989bd95 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -679,7 +679,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): **Vote Attributes:** :var str legacy_dir_key: fingerprint of and obsolete identity key - :var :class:`stem.descriptor.KeyCertificate` key_certificate: **\*** authority's key certificate + :var :class:`stem.descriptor.networkstatus.KeyCertificate` key_certificate: **\*** authority's key certificate **\*** mandatory attribute """ @@ -767,7 +767,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): self.contact = _read_keyword_line("contact", content, validate) if vote: self.legacy_dir_key = _read_keyword_line("legacy-dir-key", content, validate, True) - self.key_certificate = stem.descriptor.KeyCertificate(content.read(), validate) + self.key_certificate = KeyCertificate(content.read(), validate) else: self.vote_digest = _read_keyword_line("vote-digest", content, True, validate) self.unrecognized_lines = content.read()

1 0

[stem/master] Dropping _read_keyword_line_str() and line_matches_keyword()
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 17ce11ac3822ec811fc9b25ed28f24391ae50137 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 29 11:19:59 2012 -0700 Dropping _read_keyword_line_str() and line_matches_keyword() Dropping a couple of the descriptor helper functions that are no longer used. There's a couple more that I'd like to get rid of but they're not yet fully replaced. --- stem/descriptor/__init__.py | 36 ------------------------------------ stem/descriptor/networkstatus.py | 2 +- 2 files changed, 1 insertions(+), 37 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 362500a..ea69300 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -234,39 +234,6 @@ def _read_keyword_line(keyword, descriptor_file, validate = True, optional = Fal raise ValueError("Error parsing network status document: Expected %s, received: %s" % (keyword, line)) else: return None -def _read_keyword_line_str(keyword, lines, validate = True, optional = False): - """ - Returns the rest of the line if the first keyword matches the given keyword. If - it doesn't, a ValueError is raised if optional and validate are True, if - not, None is returned. - - Respects the opt keyword and returns the next keyword if the first is "opt". - - :param str keyword: keyword the line must begin with - :param list lines: list of strings to be read from - :param bool validate: validation is enabled - :param bool optional: if the current line must begin with the given keyword - - :returns: the text after the keyword if the keyword matches the one provided, otherwise returns None or raises an exception - - :raises: ValueError if a non-optional keyword doesn't match when validation is enabled - """ - - if not lines: - if not optional and validate: - raise ValueError("Unexpected end of document") - return - - if lines[0].startswith("opt "): - line = line[4:] - if line_matches_keyword(keyword, lines[0]): - line = lines.pop(0) - - return line[len(keyword):].strip() - elif not optional and validate: - raise ValueError("Error parsing network status document: Expected %s, received: %s" % (keyword, lines[0])) - else: return None - def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False, skip = False, end_position = None): """ Reads from the descriptor file until we get to one of the given keywords or reach the @@ -438,6 +405,3 @@ def _strptime(string, validate = True, optional = False): if validate or not optional: raise exc else: return None -def line_matches_keyword(keyword, line): - return re.search("^(opt )?" + re.escape(keyword) + "($| )", line) - diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 989bd95..7c95f96 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -55,7 +55,7 @@ import stem.exit_policy import stem.util.tor_tools from stem.descriptor import _read_until_keywords, _peek_keyword, _strptime -from stem.descriptor import _read_keyword_line, _read_keyword_line_str, _get_pseudo_pgp_block, _peek_line +from stem.descriptor import _read_keyword_line, _get_pseudo_pgp_block, _peek_line # Network status document are either a 'vote' or 'consensus', with different # mandatory fields for each. Both though require that their fields appear in a

1 0

[stem/master] Fix when router status entries have duplicate lines
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit a09b1e038fc97dada370e01ad8fd0c4f81b90c12 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Oct 4 09:39:27 2012 -0700 Fix when router status entries have duplicate lines Spotted a bug with the RouterStatusEntry where we have an undefined 'key' variable when validating that we don't have any duplicate entries. Turns out that this use case wasn't being tested so added a unit test to exercise the bug first. --- stem/descriptor/networkstatus.py | 2 +- test/unit/descriptor/networkstatus/entry.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index c27f587..fa02e44 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -1114,7 +1114,7 @@ class RouterStatusEntry(stem.descriptor.Descriptor): # most attributes can only appear at most once if validate and len(values) > 1 and keyword in ('r', 's', 'v', 'w', 'p'): - raise ValueError("Router status entries can only have a single '%s' line, got %i:\n%s" % (key, len(values), content)) + raise ValueError("Router status entries can only have a single '%s' line, got %i:\n%s" % (keyword, len(values), content)) if keyword == 'r': # "r" nickname identity digest publication IP ORPort DirPort diff --git a/test/unit/descriptor/networkstatus/entry.py b/test/unit/descriptor/networkstatus/entry.py index a0ece4d..6400f2b 100644 --- a/test/unit/descriptor/networkstatus/entry.py +++ b/test/unit/descriptor/networkstatus/entry.py @@ -100,6 +100,20 @@ class TestRouterStatusEntry(unittest.TestCase): entry = RouterStatusEntry(content) self.assertEqual("Tor 0.2.2.35", entry.version_line) + def test_duplicate_lines(self): + """ + Duplicates linesin the entry. + """ + + lines = get_router_status_entry(content = True).split("\n") + + for i in xrange(len(lines)): + content = "\n".join(lines[:i] + [lines[i]] + lines[i:]) + self.assertRaises(ValueError, RouterStatusEntry, content) + + entry = RouterStatusEntry(content, False) + self.assertEqual("caerSidi", entry.nickname) + def test_missing_r_field(self): """ Excludes fields from the 'r' line.

1 0

[stem/master] Parsing for DirectoryAuthority
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit bf192cb70a770fe2551add747bbdb6aed2f94d9d Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Oct 4 09:04:53 2012 -0700 Parsing for DirectoryAuthority Rewriting the parser for the DirectoryAuthority class. Still passes the minimal descriptor tests so next gonna add the field specific unit tests. --- stem/descriptor/networkstatus.py | 99 +++++++++++++++++++++++++++----------- 1 files changed, 71 insertions(+), 28 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 8147a26..c27f587 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -712,8 +712,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): self._unrecognized_lines = [] - #self._parse(raw_contents, validate, is_vote) - self._parse_old(raw_content, validate, is_vote) + self._parse(raw_content, validate, is_vote) def _parse(self, content, validate, is_vote): """ @@ -726,22 +725,31 @@ class DirectoryAuthority(stem.descriptor.Descriptor): :raises: ValueError if a validity check fails """ + # separate the directory authority entry from its key certificate + key_cert_content = None + + if is_vote: + key_div = content.find('\ndir-key-certificate-version') + + if key_div != -1: + key_cert_content = content[key_div + 1:] + content = content[:key_div + 1] + entries, first_keyword, _, _ = stem.descriptor._get_descriptor_components(content, validate) if validate and first_keyword != 'dir-source': raise ValueError("Authority entries are expected to start with a 'dir-source' line:\n%s" % (content)) # check that we have mandatory fields + if validate: required_fields = ["dir-source", "contact"] - if is_vote: - pass - #required_fields += ... god damnit - else: + if is_vote and not key_cert_content: + raise ValueError("Authority votes must have a key certificate:\n%s" % content) + elif not is_vote: required_fields += ["vote-digest"] - for keyword in required_fields: if not keyword in entries: raise ValueError("Authority entries must have a '%s' line:\n%s" % (keyword, content)) @@ -751,27 +759,62 @@ class DirectoryAuthority(stem.descriptor.Descriptor): line = "%s %s" % (keyword, value) # all known attributes can only appear at most once - #if validate and len(values) > 1 and keyword in ('r', 's', 'v', 'w', 'p'): - # raise ValueError("Router status entries can only have a single '%s' line, got %i:\n%s" % (key, len(values), content)) - - def _parse_old(self, raw_content, validate, is_vote): - content = StringIO(raw_content) - dir_source = _read_keyword_line("dir-source", content, validate) - self.nickname, self.fingerprint, self.hostname, self.address, self.dir_port, self.or_port = dir_source.split(" ") - self.dir_port = int(self.dir_port) - self.or_port = int(self.or_port) - - self.contact = _read_keyword_line("contact", content, validate) - if is_vote: - self.legacy_dir_key = _read_keyword_line("legacy-dir-key", content, validate, True) - self.key_certificate = KeyCertificate(content.read(), validate) - else: - self.vote_digest = _read_keyword_line("vote-digest", content, True, validate) - - remainder = content.read() + if validate and len(values) > 1 and keyword in ('dir-source', 'contact', 'legacy-dir-key', 'vote-digest'): + raise ValueError("Authority entries can only have a single '%s' line, got %i:\n%s" % (keyword, len(values), content)) + + if keyword == 'dir-source': + # "dir-source" nickname identity address IP dirport orport + + dir_source_comp = value.split(" ") + + if len(dir_source_comp) < 6: + if not validate: continue + raise ValueError("Authority entry's 'dir-source' line must have six values: %s" % line) + + if validate: + if not stem.util.tor_tools.is_valid_nickname(dir_source_comp[0]): + raise ValueError("Authority's nickname is invalid: %s" % dir_source_comp[0]) + elif not stem.util.tor_tools.is_valid_fingerprint(dir_source_comp[1]): + raise ValueError("Authority's fingerprint is invalid: %s" % dir_source_comp[1]) + elif not stem.util.connection.is_valid_ip_address(dir_source_comp[3]): + raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[3]) + elif not stem.util.connection.is_valid_port(dir_source_comp[4], allow_zero = True): + raise ValueError("Authority's DirPort is invalid: %s" % dir_source_comp[4]) + elif not stem.util.connection.is_valid_port(dir_source_comp[5]): + raise ValueError("Authority's ORPort is invalid: %s" % dir_source_comp[5]) + elif not (dir_source_comp[4].isdigit() and dir_source_comp[5].isdigit()): + continue + + self.nickname = dir_source_comp[0] + self.fingerprint = dir_source_comp[1] + self.hostname = dir_source_comp[2] + self.address = dir_source_comp[3] + self.dir_port = None if dir_source_comp[4] == '0' else int(dir_source_comp[4]) + self.or_port = int(dir_source_comp[5]) + elif keyword == 'contact': + # "contact" string + + self.contact = value + elif keyword == 'legacy-dir-key': + # "legacy-dir-key" FINGERPRINT + + if validate and not stem.util.tor_tools.is_valid_fingerprint(value): + raise ValueError("Authority has a malformed legacy directory key: %s" % line) + + self.legacy_dir_key = value + elif keyword == 'vote-digest': + # "vote-digest" digest + + # technically not a fingerprint, but has the same characteristics + if validate and not stem.util.tor_tools.is_valid_fingerprint(value): + raise ValueError("Authority has a malformed vote digest: %s" % line) + + self.vote_digest = value + else: + self._unrecognized_lines.append(line) - if remainder: - self._unrecognized_lines = remainder.split("\n") + if key_cert_content: + self.key_certificate = KeyCertificate(key_cert_content) def get_unrecognized_lines(self): """ @@ -1081,7 +1124,7 @@ class RouterStatusEntry(stem.descriptor.Descriptor): if len(r_comp) < 8: if not validate: continue - raise ValueError("Router status entry's 'r' line line must have eight values: %s" % line) + raise ValueError("Router status entry's 'r' line must have eight values: %s" % line) if validate: if not stem.util.tor_tools.is_valid_nickname(r_comp[0]):

1 0