- tor-commits - lists.torproject.org

[stem/master] Boilerplate descriptor tests for DirectoryAuthority
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 347d4c03a63b8a3f81bee83ee990e5ea4ada9125 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Oct 5 08:53:17 2012 -0700 Boilerplate descriptor tests for DirectoryAuthority Copying a few general tests from the other descriptors. I should move these into a descriptor test superclass so I don't need to keep copying them. That said, there are some minor tweaking that might prevent that... --- stem/descriptor/networkstatus.py | 2 +- .../networkstatus/directory_authority.py | 61 +++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index fa02e44..039c90c 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -684,7 +684,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): **\*** mandatory attribute """ - def __init__(self, raw_content, validate, is_vote = True): + def __init__(self, raw_content, validate = True, is_vote = False): """ Parse a directory authority entry in a v3 network status document. diff --git a/test/unit/descriptor/networkstatus/directory_authority.py b/test/unit/descriptor/networkstatus/directory_authority.py index dd4c5fa..cf16506 100644 --- a/test/unit/descriptor/networkstatus/directory_authority.py +++ b/test/unit/descriptor/networkstatus/directory_authority.py @@ -4,7 +4,8 @@ Unit tests for the DirectoryAuthority of stem.descriptor.networkstatus. import unittest -from test.mocking import get_directory_authority, get_key_certificate +from stem.descriptor.networkstatus import DirectoryAuthority +from test.mocking import get_directory_authority, get_key_certificate, AUTHORITY_HEADER class TestDirectoryAuthority(unittest.TestCase): def test_minimal_consensus_authority(self): @@ -44,4 +45,62 @@ class TestDirectoryAuthority(unittest.TestCase): self.assertEqual(None, authority.legacy_dir_key) self.assertEqual(get_key_certificate(), authority.key_certificate) self.assertEqual([], authority.get_unrecognized_lines()) + + def test_unrecognized_line(self): + """ + Includes unrecognized content in the descriptor. + """ + + authority = get_directory_authority({"pepperjack": "is oh so tasty!"}) + self.assertEquals(["pepperjack is oh so tasty!"], authority.get_unrecognized_lines()) + + def test_first_line(self): + """ + Includes a non-mandatory field before the 'dir-source' line. + """ + + content = "ho-hum 567\n" + get_directory_authority(content = True) + self.assertRaises(ValueError, DirectoryAuthority, content) + + authority = DirectoryAuthority(content, False) + self.assertEqual("turtles", authority.nickname) + self.assertEqual(["ho-hum 567"], authority.get_unrecognized_lines()) + + def test_missing_fields(self): + """ + Parse an authority where a mandatory field is missing. + """ + + for excluded_field in ("dir-source", "contact"): + content = get_directory_authority(exclude = (excluded_field,), content = True) + self.assertRaises(ValueError, DirectoryAuthority, content) + + authority = DirectoryAuthority(content, False) + + if excluded_field == "dir-source": + self.assertEqual("Mike Perry <email>", authority.contact) + else: + self.assertEqual("turtles", authority.nickname) + + def test_blank_lines(self): + """ + Includes blank lines, which should be ignored. + """ + + authority = get_directory_authority({"dir-source": AUTHORITY_HEADER[0][1] + "\n\n\n"}) + self.assertEqual("Mike Perry <email>", authority.contact) + + def test_duplicate_lines(self): + """ + Duplicates linesin the entry. + """ + + lines = get_directory_authority(content = True).split("\n") + + for i in xrange(len(lines)): + content = "\n".join(lines[:i] + [lines[i]] + lines[i:]) + self.assertRaises(ValueError, DirectoryAuthority, content) + + authority = DirectoryAuthority(content, False) + self.assertEqual("turtles", authority.nickname)

1 0

[stem/master] Adding unit test for empty dir-source values
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 5ac628e7c8bd95c6c2cade6477aceef51a941b2e Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Oct 6 12:26:10 2012 -0700 Adding unit test for empty dir-source values Just realized that the nickname and hostname fields, which lack any constraints, might also allow for empty values. I'm not really clear about this so asking for clarification... https://trac.torproject.org/7055 --- .../networkstatus/directory_authority.py | 34 ++++++++++++++++++++ 1 files changed, 34 insertions(+), 0 deletions(-) diff --git a/test/unit/descriptor/networkstatus/directory_authority.py b/test/unit/descriptor/networkstatus/directory_authority.py index cf16506..1405d12 100644 --- a/test/unit/descriptor/networkstatus/directory_authority.py +++ b/test/unit/descriptor/networkstatus/directory_authority.py @@ -4,6 +4,8 @@ Unit tests for the DirectoryAuthority of stem.descriptor.networkstatus. import unittest +import test.runner + from stem.descriptor.networkstatus import DirectoryAuthority from test.mocking import get_directory_authority, get_key_certificate, AUTHORITY_HEADER @@ -103,4 +105,36 @@ class TestDirectoryAuthority(unittest.TestCase): authority = DirectoryAuthority(content, False) self.assertEqual("turtles", authority.nickname) + + def test_empty_values(self): + """ + The 'dir-source' line has a couple string values where anything (without + spaces) would be valud. Check that we're ok with the value being an empty + string. + """ + + # TODO: Test presently fails because a missing nickname makes us think that + # a field is missing. This is technically a bug caused by us ignoring an + # idiosyncrasy with how v3 documents are formatted. With all descriptor + # types *except* v3 documents a keyword and value is split by any amount + # of whitespace. With a v3 document it must be a single space. + # + # When we have an empty nickname the value starts with a space, causing our + # keyword/value regex to gobble the extra space (making the field + # disappear). Checking with Nick before investing any further effort into + # this... + # https://trac.torproject.org/7055 + + test.runner.skip(self, "https://trac.torproject.org/7055") + return + + # drop the authority nickname + dir_source = AUTHORITY_HEADER[0][1].replace('turtles', '') + authority = get_directory_authority({"dir-source": dir_source}) + self.assertEqual('', authority.nickname) + + # drop the hostname + dir_source = AUTHORITY_HEADER[0][1].replace('no.place.com', '') + authority = get_directory_authority({"dir-source": dir_source}) + self.assertEqual('', authority.hostname)

1 0

[stem/master] Unit test to include directory authorities in document
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 72cb24dde90c928135a686707db6baa5a88d5147 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Oct 6 14:09:34 2012 -0700 Unit test to include directory authorities in document Adding a document test that includes authority entries. The especially interesting bit of this is checking that validation propagates to the KeyCertificate constructor (ie, it's checking that the 'validate' flag is being passed from the document to authority and authority to cert classes). --- stem/descriptor/networkstatus.py | 6 +++ test/mocking.py | 10 ++++- test/unit/descriptor/networkstatus/document.py | 43 ++++++++++++++++++++++- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 80eb17b..59e616a 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -835,6 +835,12 @@ class DirectoryAuthority(stem.descriptor.Descriptor): """ return self._unrecognized_lines + + def __cmp__(self, other): + if not isinstance(other, DirectoryAuthority): + return 1 + + return str(self) > str(other) class KeyCertificate(stem.descriptor.Descriptor): """ diff --git a/test/mocking.py b/test/mocking.py index 89ea2f6..2d656dd 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -591,13 +591,14 @@ def get_key_certificate(attr = None, exclude = (), content = False): else: return stem.descriptor.networkstatus.KeyCertificate(desc_content, validate = True) -def get_network_status_document(attr = None, exclude = (), routers = None, content = False): +def get_network_status_document(attr = None, exclude = (), authorities = None, routers = None, content = False): """ Provides the descriptor content for... stem.descriptor.networkstatus.NetworkStatusDocument :param dict attr: keyword/value mappings to be included in the descriptor :param list exclude: mandatory keywords to exclude from the descriptor + :param list authorities: directory authorities to include in the document :param list routers: router status entries to include in the document :param bool content: provides the str content of the descriptor rather than the class if True @@ -625,8 +626,13 @@ def get_network_status_document(attr = None, exclude = (), routers = None, conte desc_content = _get_descriptor_content(attr, exclude, NETWORK_STATUS_DOCUMENT_HEADER, NETWORK_STATUS_DOCUMENT_FOOTER) + # inject the authorities and/or routers between the header and footer + if authorities: + footer_div = desc_content.find("\ndirectory-footer") + 1 + authority_content = "\n".join([str(a) for a in authorities]) + "\n" + desc_content = desc_content[:footer_div] + authority_content + desc_content[footer_div:] + if routers: - # inject the routers between the header and footer footer_div = desc_content.find("\ndirectory-footer") + 1 router_content = "\n".join([str(r) for r in routers]) + "\n" desc_content = desc_content[:footer_div] + router_content + desc_content[footer_div:] diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py index 0d9f237..d0bb06a 100644 --- a/test/unit/descriptor/networkstatus/document.py +++ b/test/unit/descriptor/networkstatus/document.py @@ -8,8 +8,8 @@ import StringIO import stem.version from stem.descriptor import Flag -from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, RouterStatusEntry, NetworkStatusDocument, parse_file -from test.mocking import get_router_status_entry, get_network_status_document, CRYPTO_BLOB, DOC_SIG +from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, RouterStatusEntry, DirectoryAuthority, NetworkStatusDocument, parse_file +from test.mocking import get_router_status_entry, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG class TestNetworkStatusDocument(unittest.TestCase): def test_minimal_consensus(self): @@ -634,4 +634,43 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) self.assertEquals((entry3,), document.routers) + + def test_with_directory_authorities(self): + """ + Includes a couple directory authorities in the document. + """ + + for is_document_vote in (False, True): + for is_authorities_vote in (False, True): + authority1 = get_directory_authority({'contact': 'doctor jekyll'}, is_vote = is_authorities_vote) + authority2 = get_directory_authority({'contact': 'mister hyde'}, is_vote = is_authorities_vote) + + vote_status = "vote" if is_document_vote else "consensus" + content = get_network_status_document({"vote-status": vote_status}, authorities = (authority1, authority2), content = True) + + if is_document_vote == is_authorities_vote: + document = NetworkStatusDocument(content) + self.assertEquals((authority1, authority2), document.directory_authorities) + else: + # authority votes in a consensus or consensus authorities in a vote + self.assertRaises(ValueError, NetworkStatusDocument, content) + document = NetworkStatusDocument(content, validate = False) + self.assertEquals((authority1, authority2), document.directory_authorities) + + def test_authority_validation_flag_propagation(self): + """ + Includes invalid certificate content in an authority entry. This is testing + that the 'validate' flag propagages from the document to authority, and + authority to certificate classes. + """ + + # make the dir-key-published field of the certiciate be malformed + authority_content = get_directory_authority(is_vote = True, content = True) + authority_content = authority_content.replace("dir-key-published 2011", "dir-key-published 2011a") + + content = get_network_status_document({"vote-status": "vote"}, authorities = (authority_content,), content = True) + self.assertRaises(ValueError, NetworkStatusDocument, content) + + document = NetworkStatusDocument(content, validate = False) + self.assertEquals((DirectoryAuthority(authority_content, False, True),), document.directory_authorities)

1 0

[stem/master] Unit tests for DirectoryAuthority fields
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit dc89b293bb3f8a282766971597ada90723f17fa7 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Oct 6 13:51:29 2012 -0700 Unit tests for DirectoryAuthority fields Tests for the DirectoryAuthority's individual fields, and fixes for a couple issues they uncovered. --- stem/descriptor/networkstatus.py | 33 +++-- .../networkstatus/directory_authority.py | 142 ++++++++++++++++++++ 2 files changed, 164 insertions(+), 11 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 039c90c..80eb17b 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -726,14 +726,13 @@ class DirectoryAuthority(stem.descriptor.Descriptor): """ # separate the directory authority entry from its key certificate - key_cert_content = None + key_div = content.find('\ndir-key-certificate-version') - if is_vote: - key_div = content.find('\ndir-key-certificate-version') - - if key_div != -1: - key_cert_content = content[key_div + 1:] - content = content[:key_div + 1] + if key_div != -1: + key_cert_content = content[key_div + 1:] + content = content[:key_div + 1] + else: + key_cert_content = None entries, first_keyword, _, _ = stem.descriptor._get_descriptor_components(content, validate) @@ -743,16 +742,28 @@ class DirectoryAuthority(stem.descriptor.Descriptor): # check that we have mandatory fields if validate: - required_fields = ["dir-source", "contact"] + required_fields, excluded_fields = ["dir-source", "contact"], [] - if is_vote and not key_cert_content: - raise ValueError("Authority votes must have a key certificate:\n%s" % content) + if is_vote: + if not key_cert_content: + raise ValueError("Authority votes must have a key certificate:\n%s" % content) + + excluded_fields += ["vote-digest"] elif not is_vote: + if key_cert_content: + raise ValueError("Authority consensus entries shouldn't have a key certificate:\n%s" % content) + required_fields += ["vote-digest"] + excluded_fields += ["legacy-dir-key"] for keyword in required_fields: if not keyword in entries: raise ValueError("Authority entries must have a '%s' line:\n%s" % (keyword, content)) + + for keyword in entries: + if keyword in excluded_fields: + type_label = "votes" if is_vote else "consensus entries" + raise ValueError("Authority %s shouldn't have a '%s' line:\n%s" % (type_label, keyword, content)) for keyword, values in entries.items(): value, block_contents = values[0] @@ -814,7 +825,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): self._unrecognized_lines.append(line) if key_cert_content: - self.key_certificate = KeyCertificate(key_cert_content) + self.key_certificate = KeyCertificate(key_cert_content, validate) def get_unrecognized_lines(self): """ diff --git a/test/unit/descriptor/networkstatus/directory_authority.py b/test/unit/descriptor/networkstatus/directory_authority.py index 1405d12..b032b48 100644 --- a/test/unit/descriptor/networkstatus/directory_authority.py +++ b/test/unit/descriptor/networkstatus/directory_authority.py @@ -106,6 +106,25 @@ class TestDirectoryAuthority(unittest.TestCase): authority = DirectoryAuthority(content, False) self.assertEqual("turtles", authority.nickname) + def test_missing_dir_source_field(self): + """ + Excludes fields from the 'dir-source' line. + """ + + for missing_value in AUTHORITY_HEADER[0][1].split(' '): + dir_source = AUTHORITY_HEADER[0][1].replace(missing_value, '').replace(' ', ' ') + content = get_directory_authority({"dir-source": dir_source}, content = True) + self.assertRaises(ValueError, DirectoryAuthority, content) + + authority = DirectoryAuthority(content, False) + + self.assertEqual(None, authority.nickname) + self.assertEqual(None, authority.fingerprint) + self.assertEqual(None, authority.hostname) + self.assertEqual(None, authority.address) + self.assertEqual(None, authority.dir_port) + self.assertEqual(None, authority.or_port) + def test_empty_values(self): """ The 'dir-source' line has a couple string values where anything (without @@ -137,4 +156,127 @@ class TestDirectoryAuthority(unittest.TestCase): dir_source = AUTHORITY_HEADER[0][1].replace('no.place.com', '') authority = get_directory_authority({"dir-source": dir_source}) self.assertEqual('', authority.hostname) + + def test_malformed_fingerprint(self): + """ + Includes a malformed fingerprint on the 'dir-source' line. + """ + + test_values = ( + "", + "zzzzz", + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + ) + + for value in test_values: + dir_source = AUTHORITY_HEADER[0][1].replace('27B6B5996C426270A5C95488AA5BCEB6BCC86956', value) + content = get_directory_authority({"dir-source": dir_source}, content = True) + self.assertRaises(ValueError, DirectoryAuthority, content) + + authority = DirectoryAuthority(content, False) + self.assertEqual(value, authority.fingerprint) + + def test_malformed_address(self): + """ + Includes a malformed ip address on the 'dir-source' line. + """ + + test_values = ( + "", + "71.35.150.", + "71.35..29", + "71.35.150", + "71.35.150.256", + "[fd9f:2e19:3bcf::02:9970]", + ) + + for value in test_values: + dir_source = AUTHORITY_HEADER[0][1].replace('76.73.17.194', value) + content = get_directory_authority({"dir-source": dir_source}, content = True) + self.assertRaises(ValueError, DirectoryAuthority, content) + + authority = DirectoryAuthority(content, False) + self.assertEqual(value, authority.address) + + def test_malformed_port(self): + """ + Includes a malformed orport or dirport on the 'dir-source' line. + """ + + test_values = ( + "", + "-1", + "399482", + "blarg", + ) + + for value in test_values: + for include_or_port in (False, True): + for include_dir_port in (False, True): + if not include_or_port and not include_dir_port: + continue + + dir_source = AUTHORITY_HEADER[0][1] + + if include_or_port: + dir_source = dir_source.replace('9090', value) + + if include_dir_port: + dir_source = dir_source.replace('9030', value) + + content = get_directory_authority({"dir-source": dir_source}, content = True) + self.assertRaises(ValueError, DirectoryAuthority, content) + + authority = DirectoryAuthority(content, False) + + expected_value = 399482 if value == "399482" else None + actual_value = authority.or_port if include_or_port else authority.dir_port + self.assertEqual(expected_value, actual_value) + + def test_legacy_dir_key(self): + """ + Includes a 'legacy-dir-key' line with both valid and invalid content. + """ + + test_value = "65968CCB6BECB5AA88459C5A072624C6995B6B72" + authority = get_directory_authority({"legacy-dir-key": test_value}, is_vote = True) + self.assertEqual(test_value, authority.legacy_dir_key) + + # check that we'll fail if legacy-dir-key appears in a consensus + content = get_directory_authority({"legacy-dir-key": test_value}, content = True) + self.assertRaises(ValueError, DirectoryAuthority, content) + + test_values = ( + "", + "zzzzz", + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + ) + + for value in test_values: + content = get_directory_authority({"legacy-dir-key": value}, content = True) + self.assertRaises(ValueError, DirectoryAuthority, content) + + authority = DirectoryAuthority(content, False) + self.assertEqual(value, authority.legacy_dir_key) + + def test_key_certificate(self): + """ + Includes or exclude a key certificate from the directory entry. + """ + + key_cert = get_key_certificate() + + # include a key cert with a consensus + content = get_directory_authority(content = True) + "\n" + str(key_cert) + self.assertRaises(ValueError, DirectoryAuthority, content) + + authority = DirectoryAuthority(content, False) + self.assertEqual('turtles', authority.nickname) + + # exclude key cert from a vote + content = get_directory_authority(content = True, is_vote = True).replace("\n" + str(key_cert), '') + self.assertRaises(ValueError, DirectoryAuthority, content, True, True) + + authority = DirectoryAuthority(content, False, True) + self.assertEqual('turtles', authority.nickname)

1 0

[stem/master] Unit tests for minimal v2 and microdescriptor v3 router status entries
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit a1d2a0ffffd5aeec96c37fbbf1b4134c9b34b631 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Oct 7 18:18:14 2012 -0700 Unit tests for minimal v2 and microdescriptor v3 router status entries Tests to exercise basic parsing for the v2 and microdescriptor v3 RouterStatusEntry subclasses. They're both largely a subset of the v3 RouterStatusEntry so I don't plan to add additional tests for the moment. --- stem/descriptor/router_status_entry.py | 2 +- test/mocking.py | 74 ++++++++++++++++++++++++--- test/unit/descriptor/router_status_entry.py | 47 ++++++++++++++++- 3 files changed, 111 insertions(+), 12 deletions(-) diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index 50f0825..f663f07 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -285,7 +285,7 @@ class RouterStatusEntryMicroV3(RouterStatusEntry): self.digest = None - super(RouterStatusEntryMicroV3, self).__init__(content) + super(RouterStatusEntryMicroV3, self).__init__(content, validate, document) def _parse(self, entries, validate): for keyword, values in entries.items(): diff --git a/test/mocking.py b/test/mocking.py index 808cb8d..7c848ad 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -23,14 +23,24 @@ calling :func:`test.mocking.revert_mocking`. Instance Constructors get_message - stem.socket.ControlMessage get_protocolinfo_response - stem.response.protocolinfo.ProtocolInfoResponse - get_relay_server_descriptor - stem.descriptor.server_descriptor.RelayDescriptor - get_bridge_server_descriptor - stem.descriptor.server_descriptor.BridgeDescriptor - get_relay_extrainfo_descriptor - stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor - get_bridge_extrainfo_descriptor - stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor - get_router_status_entry_v3 - stem.descriptor.router_status_entry.RouterStatusEntryV3 - get_directory_authority - stem.descriptor.networkstatus.DirectoryAuthority - get_key_certificate - stem.descriptor.networkstatus.KeyCertificate - get_network_status_document - stem.descriptor.networkstatus.NetworkStatusDocument + + stem.descriptor.server_descriptor + get_relay_server_descriptor - RelayDescriptor + get_bridge_server_descriptor - BridgeDescriptor + + stem.descriptor.extrainfo_descriptor + get_relay_extrainfo_descriptor - RelayExtraInfoDescriptor + get_bridge_extrainfo_descriptor - BridgeExtraInfoDescriptor + + stem.descriptor.networkstatus + get_directory_authority - DirectoryAuthority + get_key_certificate - KeyCertificate + get_network_status_document - NetworkStatusDocument + + stem.descriptor.router_status_entry + get_router_status_entry_v2 - RouterStatusEntryV2 + get_router_status_entry_v3 - RouterStatusEntryV3 + get_router_status_entry_micro_v3 - RouterStatusEntryMicroV3 """ import inspect @@ -107,11 +117,21 @@ BRIDGE_EXTRAINFO_FOOTER = ( ("router-digest", "006FD96BA35E7785A6A3B8B75FE2E2435A13BDB4"), ) +ROUTER_STATUS_ENTRY_V2_HEADER = ( + ("r", "caerSidi p1aag7VwarGxqctS7/fS0y5FU+s oQZFLYe9e4A7bOkWKR7TaNxb0JE 2012-08-06 11:19:31 71.35.150.29 9001 0"), +) + ROUTER_STATUS_ENTRY_V3_HEADER = ( ("r", "caerSidi p1aag7VwarGxqctS7/fS0y5FU+s oQZFLYe9e4A7bOkWKR7TaNxb0JE 2012-08-06 11:19:31 71.35.150.29 9001 0"), ("s", "Fast Named Running Stable Valid"), ) +ROUTER_STATUS_ENTRY_MICRO_V3_HEADER = ( + ("r", "Konata ARIJF2zbqirB9IwsW0mQznccWww 2012-09-24 13:40:40 69.64.48.168 9001 9030"), + ("m", "aiUklwBrua82obG5AsTX+iEpkjQA2+AQHxZ7GwMfY70"), + ("s", "Fast Guard HSDir Named Running Stable V2Dir Valid"), +) + AUTHORITY_HEADER = ( ("dir-source", "turtles 27B6B5996C426270A5C95488AA5BCEB6BCC86956 no.place.com 76.73.17.194 9030 9090"), ("contact", "Mike Perry <email>"), @@ -523,6 +543,25 @@ def get_bridge_extrainfo_descriptor(attr = None, exclude = (), content = False): else: return stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(desc_content, validate = True) +def get_router_status_entry_v2(attr = None, exclude = (), content = False): + """ + Provides the descriptor content for... + stem.descriptor.router_status_entry.RouterStatusEntryV2 + + :param dict attr: keyword/value mappings to be included in the descriptor + :param list exclude: mandatory keywords to exclude from the descriptor + :param bool content: provides the str content of the descriptor rather than the class if True + + :returns: RouterStatusEntryV2 for the requested descriptor content + """ + + desc_content = _get_descriptor_content(attr, exclude, ROUTER_STATUS_ENTRY_V2_HEADER) + + if content: + return desc_content + else: + return stem.descriptor.router_status_entry.RouterStatusEntryV2(desc_content, validate = True) + def get_router_status_entry_v3(attr = None, exclude = (), content = False): """ Provides the descriptor content for... @@ -542,6 +581,25 @@ def get_router_status_entry_v3(attr = None, exclude = (), content = False): else: return stem.descriptor.router_status_entry.RouterStatusEntryV3(desc_content, validate = True) +def get_router_status_entry_micro_v3(attr = None, exclude = (), content = False): + """ + Provides the descriptor content for... + stem.descriptor.router_status_entry.RouterStatusEntryMicroV3 + + :param dict attr: keyword/value mappings to be included in the descriptor + :param list exclude: mandatory keywords to exclude from the descriptor + :param bool content: provides the str content of the descriptor rather than the class if True + + :returns: RouterStatusEntryMicroV3 for the requested descriptor content + """ + + desc_content = _get_descriptor_content(attr, exclude, ROUTER_STATUS_ENTRY_MICRO_V3_HEADER) + + if content: + return desc_content + else: + return stem.descriptor.router_status_entry.RouterStatusEntryMicroV3(desc_content, validate = True) + def get_directory_authority(attr = None, exclude = (), is_vote = False, content = False): """ Provides the descriptor content for... diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py index 6816656..d971906 100644 --- a/test/unit/descriptor/router_status_entry.py +++ b/test/unit/descriptor/router_status_entry.py @@ -9,7 +9,7 @@ from stem.descriptor import Flag from stem.descriptor.router_status_entry import RouterStatusEntryV3, _decode_fingerprint from stem.version import Version from stem.exit_policy import MicrodescriptorExitPolicy -from test.mocking import get_router_status_entry_v3, ROUTER_STATUS_ENTRY_V3_HEADER +from test.mocking import get_router_status_entry_v2, get_router_status_entry_v3, get_router_status_entry_micro_v3, ROUTER_STATUS_ENTRY_V3_HEADER class TestRouterStatusEntry(unittest.TestCase): def test_fingerprint_decoding(self): @@ -35,9 +35,29 @@ class TestRouterStatusEntry(unittest.TestCase): self.assertRaises(ValueError, _decode_fingerprint, arg, True) self.assertEqual(None, _decode_fingerprint(arg, False)) - def test_minimal(self): + def test_minimal_v2(self): """ - Parses a minimal router status entry. + Parses a minimal v2 router status entry. + """ + + entry = get_router_status_entry_v2() + + self.assertEqual(None, entry.document) + self.assertEqual("caerSidi", entry.nickname) + self.assertEqual("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", entry.fingerprint) + self.assertEqual("oQZFLYe9e4A7bOkWKR7TaNxb0JE", entry.digest) + self.assertEqual(datetime.datetime(2012, 8, 6, 11, 19, 31), entry.published) + self.assertEqual("71.35.150.29", entry.address) + self.assertEqual(9001, entry.or_port) + self.assertEqual(None, entry.dir_port) + self.assertEqual(None, entry.flags) + self.assertEqual(None, entry.version_line) + self.assertEqual(None, entry.version) + self.assertEqual([], entry.get_unrecognized_lines()) + + def test_minimal_v3(self): + """ + Parses a minimal v3 router status entry. """ entry = get_router_status_entry_v3() @@ -61,6 +81,27 @@ class TestRouterStatusEntry(unittest.TestCase): self.assertEqual(None, entry.microdescriptor_hashes) self.assertEqual([], entry.get_unrecognized_lines()) + def test_minimal_micro_v3(self): + """ + Parses a minimal microdescriptor v3 router status entry. + """ + + entry = get_router_status_entry_micro_v3() + + expected_flags = set([Flag.FAST, Flag.GUARD, Flag.HSDIR, Flag.NAMED, Flag.RUNNING, Flag.STABLE, Flag.V2DIR, Flag.VALID]) + self.assertEqual(None, entry.document) + self.assertEqual("Konata", entry.nickname) + self.assertEqual("011209176CDBAA2AC1F48C2C5B4990CE771C5B0C", entry.fingerprint) + self.assertEqual(datetime.datetime(2012, 9, 24, 13, 40, 40), entry.published) + self.assertEqual("69.64.48.168", entry.address) + self.assertEqual(9001, entry.or_port) + self.assertEqual(9030, entry.dir_port) + self.assertEqual(expected_flags, set(entry.flags)) + self.assertEqual(None, entry.version_line) + self.assertEqual(None, entry.version) + self.assertEqual("aiUklwBrua82obG5AsTX+iEpkjQA2+AQHxZ7GwMfY70", entry.digest) + self.assertEqual([], entry.get_unrecognized_lines()) + def test_missing_fields(self): """ Parses a router status entry that's missing fields.

1 0

[stem/master] Module for V2, V3 and Microdescriptor router status entries
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit b8ca825a7f6dcde9f7689bca41a0938598749b4f Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Oct 7 17:59:08 2012 -0700 Module for V2, V3 and Microdescriptor router status entries Moving the router status entries to their own module and adding classes for V2 entries and microdescriptors (both still completely untested). The abstraction is based on the ServerDescriptor breakdown but with more liberal use of helpers for the actual parsing. --- run_tests.py | 4 +- stem/descriptor/__init__.py | 1 + stem/descriptor/networkstatus.py | 328 +-------------- stem/descriptor/router_status_entry.py | 538 ++++++++++++++++++++++++ stem/descriptor/server_descriptor.py | 22 +- test/mocking.py | 15 +- test/unit/descriptor/__init__.py | 9 +- test/unit/descriptor/networkstatus/__init__.py | 2 +- test/unit/descriptor/networkstatus/document.py | 15 +- test/unit/descriptor/networkstatus/entry.py | 425 ------------------- test/unit/descriptor/router_status_entry.py | 425 +++++++++++++++++++ 11 files changed, 1009 insertions(+), 775 deletions(-) diff --git a/run_tests.py b/run_tests.py index 8f76dd3..c4db524 100755 --- a/run_tests.py +++ b/run_tests.py @@ -20,7 +20,7 @@ import test.unit.descriptor.export import test.unit.descriptor.reader import test.unit.descriptor.server_descriptor import test.unit.descriptor.extrainfo_descriptor -import test.unit.descriptor.networkstatus.entry +import test.unit.descriptor.router_status_entry import test.unit.descriptor.networkstatus.directory_authority import test.unit.descriptor.networkstatus.key_certificate import test.unit.descriptor.networkstatus.document @@ -118,7 +118,7 @@ UNIT_TESTS = ( test.unit.descriptor.reader.TestDescriptorReader, test.unit.descriptor.server_descriptor.TestServerDescriptor, test.unit.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, - test.unit.descriptor.networkstatus.entry.TestRouterStatusEntry, + test.unit.descriptor.router_status_entry.TestRouterStatusEntry, test.unit.descriptor.networkstatus.directory_authority.TestDirectoryAuthority, test.unit.descriptor.networkstatus.key_certificate.TestKeyCertificate, test.unit.descriptor.networkstatus.document.TestNetworkStatusDocument, diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index ea69300..ef3d558 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -18,6 +18,7 @@ __all__ = [ "extrainfo_descriptor", "server_descriptor", "networkstatus", + "router_status_entry", "parse_file", "Descriptor", ] diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 59e616a..38207b8 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -4,7 +4,7 @@ status documents (both votes and consensuses). The network status documents also contain a list of router descriptors, directory authorities, signatures etc. If you only need the -:class:`stem.descriptor.networkstatus.RouterStatusEntry` objects, use +:class:`stem.descriptor.router_status_entry.RouterStatusEntry` objects, use :func:`stem.descriptor.parse_file`. Other information can be accessed by directly instantiating :class:`stem.descriptor.networkstatus.NetworkStatusDocument` objects. @@ -34,14 +34,11 @@ The documents can be obtained from any of the following sources... parse_file - parses a network status file and provides a NetworkStatusDocument NetworkStatusDocument - Tor v3 network status document +- MicrodescriptorConsensus - Microdescriptor flavoured consensus documents - RouterStatusEntry - Router descriptor; contains information about a Tor relay - +- RouterMicrodescriptor - Router microdescriptor; contains information that doesn't change frequently DocumentSignature - Signature of a document by a directory authority DirectoryAuthority - Directory authority defined in a v3 network status document """ import re -import base64 import datetime try: @@ -50,6 +47,7 @@ except: from StringIO import StringIO import stem.descriptor +import stem.descriptor.router_status_entry import stem.version import stem.exit_policy import stem.util.tor_tools @@ -131,9 +129,9 @@ BANDWIDTH_WEIGHT_ENTRIES = ( def parse_file(document_file, validate = True, is_microdescriptor = False): """ - Parses a network status and iterates over the RouterStatusEntry or - RouterMicrodescriptor in it. The document that these instances reference have - an empty 'rotuers' attribute to allow for limited memory usage. + Parses a network status and iterates over the RouterStatusEntry in it. The + document that these instances reference have an empty 'rotuers' attribute to + allow for limited memory usage. :param file document_file: file with network status document content :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise @@ -159,7 +157,7 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): if not is_microdescriptor: document = NetworkStatusDocument(document_content, validate) - router_type = RouterStatusEntry + router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3 else: document = MicrodescriptorConsensus(document_content, validate) router_type = RouterMicrodescriptor @@ -221,7 +219,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): """ Version 3 network status document. This could be either a vote or consensus. - :var tuple routers: RouterStatusEntry contained in the document + :var tuple routers: RouterStatusEntryV3 contained in the document :var str version: **\*** document version :var bool is_consensus: **\*** true if the document is a consensus @@ -294,7 +292,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): self._unrecognized_lines += value def _get_router_type(self): - return RouterStatusEntry + return stem.descriptor.router_status_entry.RouterStatusEntryV3 def meets_consensus_method(self, method): """ @@ -1036,265 +1034,6 @@ class DocumentSignature(object): return 0 -class RouterStatusEntry(stem.descriptor.Descriptor): - """ - Information about an individual router stored within a network status - document. - - :var NetworkStatusDocument document: **\*** document that this descriptor came from - - :var str nickname: **\*** router's nickname - :var str fingerprint: **\*** router's fingerprint - :var str digest: **\*** router's digest - :var datetime published: **\*** router's publication - :var str address: **\*** router's IP address - :var int or_port: **\*** router's ORPort - :var int dir_port: **\*** router's DirPort - :var list flags: **\*** list of status flags - - :var stem.version.Version version: parsed version of tor, this is None if the relay's using a new versioning scheme - :var str version_line: versioning information reported by the relay - - :var int bandwidth: bandwidth claimed by the relay (in kb/s) - :var int measured: bandwith measured to be available by the relay - :var list unrecognized_bandwidth_entries: **\*** bandwidth weighting information that isn't yet recognized - - :var stem.exit_policy.MicrodescriptorExitPolicy exit_policy: router's exit policy - - :var list microdescriptor_hashes: tuples of two values, the list of consensus methods for generting a set of digests and the 'algorithm => digest' mappings - - **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined - """ - - def __init__(self, raw_contents, validate = True, document = None): - """ - Parse a router descriptor in a v3 network status document. - - :param str raw_content: router descriptor content to be parsed - :param NetworkStatusDocument document: document this descriptor came from - :param bool validate: checks the validity of the content if True, skips these checks otherwise - - :raises: ValueError if the descriptor data is invalid - """ - - super(RouterStatusEntry, self).__init__(raw_contents) - - self.document = document - - self.nickname = None - self.fingerprint = None - self.digest = None - self.published = None - self.address = None - self.or_port = None - self.dir_port = None - - self.flags = None - - self.version_line = None - self.version = None - - self.bandwidth = None - self.measured = None - self.unrecognized_bandwidth_entries = [] - - self.exit_policy = None - self.microdescriptor_hashes = None - self._unrecognized_lines = [] - - self._parse(raw_contents, validate) - - def _parse(self, content, validate): - """ - Parses the given content and applies the attributes. - - :param str content: descriptor content - :param bool validate: checks validity if True - - :raises: ValueError if a validity check fails - """ - - entries, first_keyword, _, _ = stem.descriptor._get_descriptor_components(content, validate) - - if validate and first_keyword != 'r': - raise ValueError("Router status entries are expected to start with a 'r' line:\n%s" % (content)) - - # check that we have mandatory fields - if validate: - for keyword in ('r', 's'): - if not keyword in entries: - raise ValueError("Router status entries must have a '%s' line:\n%s" % (keyword, content)) - - for keyword, values in entries.items(): - value, block_contents = values[0] - line = "%s %s" % (keyword, value) - - # most attributes can only appear at most once - if validate and len(values) > 1 and keyword in ('r', 's', 'v', 'w', 'p'): - raise ValueError("Router status entries can only have a single '%s' line, got %i:\n%s" % (keyword, len(values), content)) - - if keyword == 'r': - # "r" nickname identity digest publication IP ORPort DirPort - # r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0 - - r_comp = value.split(" ") - - if len(r_comp) < 8: - if not validate: continue - raise ValueError("Router status entry's 'r' line must have eight values: %s" % line) - - if validate: - if not stem.util.tor_tools.is_valid_nickname(r_comp[0]): - raise ValueError("Router status entry's nickname isn't valid: %s" % r_comp[0]) - elif not stem.util.connection.is_valid_ip_address(r_comp[5]): - raise ValueError("Router status entry's address isn't a valid IPv4 address: %s" % r_comp[5]) - elif not stem.util.connection.is_valid_port(r_comp[6]): - raise ValueError("Router status entry's ORPort is invalid: %s" % r_comp[6]) - elif not stem.util.connection.is_valid_port(r_comp[7], allow_zero = True): - raise ValueError("Router status entry's DirPort is invalid: %s" % r_comp[7]) - elif not (r_comp[6].isdigit() and r_comp[7].isdigit()): - continue - - self.nickname = r_comp[0] - self.fingerprint = _decode_fingerprint(r_comp[1], validate) - self.digest = r_comp[2] - self.address = r_comp[5] - self.or_port = int(r_comp[6]) - self.dir_port = None if r_comp[7] == '0' else int(r_comp[7]) - - try: - published = "%s %s" % (r_comp[3], r_comp[4]) - self.published = datetime.datetime.strptime(published, "%Y-%m-%d %H:%M:%S") - except ValueError: - if validate: - raise ValueError("Publication time time wasn't parseable: %s" % line) - elif keyword == 's': - # "s" Flags - # s Named Running Stable Valid - - if value == "": - self.flags = [] - else: - self.flags = value.split(" ") - - if validate: - for flag in self.flags: - if self.flags.count(flag) > 1: - raise ValueError("Router status entry had duplicate flags: %s" % line) - elif flag == "": - raise ValueError("Router status entry had extra whitespace on its 's' line: %s" % line) - elif keyword == 'v': - # "v" version - # v Tor 0.2.2.35 - # - # The spec says that if this starts with "Tor " then what follows is a - # tor version. If not then it has "upgraded to a more sophisticated - # protocol versioning system". - - self.version_line = value - - if value.startswith("Tor "): - try: - self.version = stem.version.Version(value[4:]) - except ValueError, exc: - if validate: - raise ValueError("Router status entry has a malformed tor version (%s): %s" % (exc, line)) - elif keyword == 'w': - # "w" "Bandwidth=" INT ["Measured=" INT] - # w Bandwidth=7980 - - w_comp = value.split(" ") - - if len(w_comp) < 1: - if not validate: continue - raise ValueError("Router status entry's 'w' line is blank: %s" % line) - elif not w_comp[0].startswith("Bandwidth="): - if not validate: continue - raise ValueError("Router status entry's 'w' line needs to start with a 'Bandwidth=' entry: %s" % line) - - for w_entry in w_comp: - if '=' in w_entry: - w_key, w_value = w_entry.split('=', 1) - else: - w_key, w_value = w_entry, None - - if w_key == "Bandwidth": - if not (w_value and w_value.isdigit()): - if not validate: continue - raise ValueError("Router status entry's 'Bandwidth=' entry needs to have a numeric value: %s" % line) - - self.bandwidth = int(w_value) - elif w_key == "Measured": - if not (w_value and w_value.isdigit()): - if not validate: continue - raise ValueError("Router status entry's 'Measured=' entry needs to have a numeric value: %s" % line) - - self.measured = int(w_value) - else: - self.unrecognized_bandwidth_entries.append(w_entry) - elif keyword == 'p': - # "p" ("accept" / "reject") PortList - # p reject 1-65535 - # p accept 80,110,143,443,993,995,6660-6669,6697,7000-7001 - - try: - self.exit_policy = stem.exit_policy.MicrodescriptorExitPolicy(value) - except ValueError, exc: - if not validate: continue - raise ValueError("Router status entry's exit policy is malformed (%s): %s" % (exc, line)) - elif keyword == 'm': - # "m" methods 1*(algorithm "=" digest) - # m 8,9,10,11,12 sha256=g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs - - m_comp = value.split(" ") - - if not (self.document and self.document.is_vote): - if not validate: continue - - vote_status = "vote" if self.document else "<undefined document>" - raise ValueError("Router status entry's 'm' line should only appear in votes (appeared in a %s): %s" % (vote_status, line)) - elif len(m_comp) < 1: - if not validate: continue - raise ValueError("Router status entry's 'm' line needs to start with a series of methods: %s" % line) - - try: - methods = [int(entry) for entry in m_comp[0].split(",")] - except ValueError: - if not validate: continue - raise ValueError("Router status entry's microdescriptor methods should be a series of comma separated integers: %s" % line) - - hashes = {} - - for entry in m_comp[1:]: - if not '=' in entry: - if not validate: continue - raise ValueError("Router status entry's can only have a series of 'algorithm=digest' mappings after the methods: %s" % line) - - hash_name, digest = entry.split('=', 1) - hashes[hash_name] = digest - - if self.microdescriptor_hashes is None: - self.microdescriptor_hashes = [] - - self.microdescriptor_hashes.append((methods, hashes)) - else: - self._unrecognized_lines.append(line) - - def get_unrecognized_lines(self): - """ - Provides any unrecognized lines. - - :returns: list of unrecognized lines - """ - - return list(self._unrecognized_lines) - - def __cmp__(self, other): - if not isinstance(other, RouterStatusEntry): - return 1 - - return str(self) > str(other) - class MicrodescriptorConsensus(NetworkStatusDocument): """ A v3 microdescriptor consensus. @@ -1326,7 +1065,7 @@ class MicrodescriptorConsensus(NetworkStatusDocument): def _validate_network_status_version(self): return self.version == "3 microdesc" -class RouterMicrodescriptor(RouterStatusEntry): +class RouterMicrodescriptor(stem.descriptor.router_status_entry.RouterStatusEntry): """ Router microdescriptor object. Parses and stores router information in a router microdescriptor from a v3 microdescriptor consensus. @@ -1452,52 +1191,3 @@ class RouterMicrodescriptor(RouterStatusEntry): return self.unrecognized_lines -def _decode_fingerprint(identity, validate): - """ - Decodes the 'identity' value found in consensuses into the more common hex - encoding of the relay's fingerprint. For example... - - :: - - >>> _decode_fingerprint('p1aag7VwarGxqctS7/fS0y5FU+s') - 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB' - - :param str identity: encoded fingerprint from the consensus - :param bool validate: checks validity if True - - :returns: str with the uppercase hex encoding of the relay's fingerprint - - :raises: ValueError if the result isn't a valid fingerprint - """ - - # trailing equal signs were stripped from the identity - missing_padding = 28 - len(identity) - identity += "=" * missing_padding - - fingerprint = "" - - try: - identity_decoded = base64.b64decode(identity) - except TypeError, exc: - if not validate: return None - raise ValueError("Unable to decode identity string '%s'" % identity) - - for char in identity_decoded: - # Individual characters are either standard ascii or hex encoded, and each - # represent two hex digits. For instnace... - # - # >>> ord('\n') - # 10 - # >>> hex(10) - # '0xa' - # >>> '0xa'[2:].zfill(2).upper() - # '0A' - - fingerprint += hex(ord(char))[2:].zfill(2).upper() - - if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): - if not validate: return None - raise ValueError("Decoded '%s' to be '%s', which isn't a valid fingerprint" % (identity, fingerprint)) - - return fingerprint - diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py new file mode 100644 index 0000000..50f0825 --- /dev/null +++ b/stem/descriptor/router_status_entry.py @@ -0,0 +1,538 @@ +""" +Parsing for router status entries, the information for individual routers +within a network status document. This information is provided from a few +sources... + +* control port via 'GETINFO ns/*' and 'GETINFO md/*' queries +* router entries in a network status document, like the cached-consensus + +**Module Overview:** + +:: + + RouterStatusEntry - Common parent for router status entries. + |- RouterStatusEntryV2 - Entry for a network status v2 document. + |- RouterStatusEntryV3 - Entry for a network status v3 document. + +- RouterStatusEntryMicroV3 - Entry for a microdescriptor flavored v3 document. +""" + +import base64 +import datetime + +import stem.descriptor +import stem.exit_policy + +class RouterStatusEntry(stem.descriptor.Descriptor): + """ + Information about an individual router stored within a network status + document. This is the common parent for concrete status entry types. + + :var NetworkStatusDocument document: **\*** document that this descriptor came from + + :var str nickname: **\*** router's nickname + :var str fingerprint: **\*** router's fingerprint + :var datetime published: **\*** router's publication + :var str address: **\*** router's IP address + :var int or_port: **\*** router's ORPort + :var int dir_port: **\*** router's DirPort + + :var list flags: **\*** list of status flags + + :var stem.version.Version version: parsed version of tor, this is None if the relay's using a new versioning scheme + :var str version_line: versioning information reported by the relay + """ + + def __init__(self, content, validate, document): + """ + Parse a router descriptor in a network status document. + + :param str content: router descriptor content to be parsed + :param NetworkStatusDocument document: document this descriptor came from + :param bool validate: checks the validity of the content if True, skips these checks otherwise + + :raises: ValueError if the descriptor data is invalid + """ + + super(RouterStatusEntry, self).__init__(content) + + self.document = document + + self.nickname = None + self.fingerprint = None + self.published = None + self.address = None + self.or_port = None + self.dir_port = None + + self.flags = None + + self.version_line = None + self.version = None + + self._unrecognized_lines = [] + + entries, first_keyword, _, _ = stem.descriptor._get_descriptor_components(content, validate) + if validate: self._check_constraints(entries, first_keyword) + self._parse(entries, validate) + + def _parse(self, entries, validate): + """ + Parses the given content and applies the attributes. + + :param dict entries: keyword => (value, pgp key) entries + :param bool validate: checks validity if True + + :raises: ValueError if a validity check fails + """ + + for keyword, values in entries.items(): + value, _ = values[0] + + if keyword == 's': + _parse_s_line(self, value, validate) + elif keyword == 'v': + _parse_v_line(self, value, validate) + else: + self._unrecognized_lines.append("%s %s" % (keyword, value)) + + def _check_constraints(self, entries, first_keyword): + """ + Does a basic check that the entries conform to this descriptor type's + constraints. + + :param dict entries: keyword => (value, pgp key) entries + :param str first_keyword: keyword of the first line + + :raises: ValueError if an issue arises in validation + """ + + for keyword in self._required_fields(): + if not keyword in entries: + raise ValueError("%s must have a '%s' line:\n%s" % (self._name(True), keyword, str(self))) + + for keyword in self._single_fields(): + if keyword in entries and len(entries[keyword]) > 1: + raise ValueError("%s can only have a single '%s' line, got %i:\n%s" % (self._name(True), keyword, len(entries[keyword]), str(self))) + + if first_keyword != 'r': + raise ValueError("%s are expected to start with a 'r' line:\n%s" % (self._name(True), str(self))) + + def _name(self, is_plural = False): + """ + Name for this descriptor type. + """ + + if is_plural: + return "Router status entries" + else: + return "Router status entry" + + def _required_fields(self): + """ + Provides lines that must appear in the descriptor. + """ + + return () + + def _single_fields(self): + """ + Provides lines that can only appear in the descriptor once. + """ + + return () + + def get_unrecognized_lines(self): + """ + Provides any unrecognized lines. + + :returns: list of unrecognized lines + """ + + return list(self._unrecognized_lines) + + def __cmp__(self, other): + if not isinstance(other, RouterStatusEntry): + return 1 + + return str(self) > str(other) + +class RouterStatusEntryV2(RouterStatusEntry): + """ + Information about an individual router stored within a version 2 network + status document. + + :var str digest: **\*** router's digest + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, content, validate = True, document = None): + self.digest = None + super(RouterStatusEntryV2, self).__init__(content, validate, document) + + def _parse(self, entries, validate): + for keyword, values in entries.items(): + value, _ = values[0] + + if keyword == 'r': + _parse_r_line(self, value, validate, True) + del entries['r'] + + RouterStatusEntry._parse(self, entries, validate) + + def _name(self, is_plural = False): + if is_plural: + return "Router status entries (v2)" + else: + return "Router status entry (v2)" + + def _required_fields(self): + return ('r') + + def _single_fields(self): + return ('r', 's', 'v') + + def __cmp__(self, other): + if not isinstance(other, RouterStatusEntryV2): + return 1 + + return str(self) > str(other) + +class RouterStatusEntryV3(RouterStatusEntry): + """ + Information about an individual router stored within a version 3 network + status document. + + :var str digest: **\*** router's digest + + :var int bandwidth: bandwidth claimed by the relay (in kb/s) + :var int measured: bandwith measured to be available by the relay + :var list unrecognized_bandwidth_entries: **\*** bandwidth weighting information that isn't yet recognized + + :var stem.exit_policy.MicrodescriptorExitPolicy exit_policy: router's exit policy + + :var list microdescriptor_hashes: tuples of two values, the list of consensus methods for generting a set of digests and the 'algorithm => digest' mappings + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, content, validate = True, document = None): + self.digest = None + + self.bandwidth = None + self.measured = None + self.unrecognized_bandwidth_entries = [] + + self.exit_policy = None + self.microdescriptor_hashes = None + + super(RouterStatusEntryV3, self).__init__(content, validate, document) + + def _parse(self, entries, validate): + for keyword, values in entries.items(): + value, _ = values[0] + + if keyword == 'r': + _parse_r_line(self, value, validate, True) + del entries['r'] + elif keyword == 'w': + _parse_w_line(self, value, validate) + del entries['w'] + elif keyword == 'p': + _parse_p_line(self, value, validate) + del entries['p'] + elif keyword == 'm': + _parse_m_line(self, value, validate) + del entries['m'] + + RouterStatusEntry._parse(self, entries, validate) + + def _name(self, is_plural = False): + if is_plural: + return "Router status entries (v3)" + else: + return "Router status entry (v3)" + + def _required_fields(self): + return ('r', 's') + + def _single_fields(self): + return ('r', 's', 'v', 'w', 'p') + + def __cmp__(self, other): + if not isinstance(other, RouterStatusEntryV3): + return 1 + + return str(self) > str(other) + +class RouterStatusEntryMicroV3(RouterStatusEntry): + """ + Information about an individual router stored within a microdescriptor + flavored network status document. + + :var int bandwidth: bandwidth claimed by the relay (in kb/s) + :var int measured: bandwith measured to be available by the relay + :var list unrecognized_bandwidth_entries: **\*** bandwidth weighting information that isn't yet recognized + + :var str digest: **\*** router's base64 encoded router microdescriptor digest + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, content, validate = True, document = None): + self.version_line = None + self.version = None + + self.digest = None + + super(RouterStatusEntryMicroV3, self).__init__(content) + + def _parse(self, entries, validate): + for keyword, values in entries.items(): + value, _ = values[0] + + if keyword == 'r': + _parse_r_line(self, value, validate, False) + del entries['r'] + elif keyword == 'w': + _parse_w_line(self, value, validate) + del entries['w'] + elif keyword == 'm': + # "m" digest + # example: m aiUklwBrua82obG5AsTX+iEpkjQA2+AQHxZ7GwMfY70 + + self.digest = value + del entries['m'] + + RouterStatusEntry._parse(self, entries, validate) + + def _name(self, is_plural = False): + if is_plural: + return "Router status entries (micro v3)" + else: + return "Router status entry (micro v3)" + + def _required_fields(self): + return ('r', 's', 'm') + + def _single_fields(self): + return ('r', 's', 'v', 'w', 'm') + + def __cmp__(self, other): + if not isinstance(other, RouterStatusEntryMicroV3): + return 1 + + return str(self) > str(other) + +def _parse_r_line(desc, value, validate, include_digest = True): + # Parses a RouterStatusEntry's 'r' line. They're very nearly identical for + # all current entry types (v2, v3, and microdescriptor v3) with one little + # wrinkle: only the microdescriptor flavor excludes a 'digest' field. + # + # For v2 and v3 router status entries: + # "r" nickname identity digest publication IP ORPort DirPort + # example: r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0 + # + # For v3 microdescriptor router status entries: + # "r" nickname identity publication IP ORPort DirPort + # example: r Konata ARIJF2zbqirB9IwsW0mQznccWww 2012-09-24 13:40:40 69.64.48.168 9001 9030 + + r_comp = value.split(" ") + + # inject a None for the digest to normalize the field positioning + if not include_digest: + r_comp.insert(2, None) + + if len(r_comp) < 8: + if not validate: return + + expected_field_count = 'eight' if include_digest else 'seven' + raise ValueError("%s 'r' line must have %s values: r %s" % (desc._name(), expected_field_count, value)) + + if validate: + if not stem.util.tor_tools.is_valid_nickname(r_comp[0]): + raise ValueError("%s nickname isn't valid: %s" % (desc._name(), r_comp[0])) + elif not stem.util.connection.is_valid_ip_address(r_comp[5]): + raise ValueError("%s address isn't a valid IPv4 address: %s" % (desc._name(), r_comp[5])) + elif not stem.util.connection.is_valid_port(r_comp[6]): + raise ValueError("%s ORPort is invalid: %s" % (desc._name(), r_comp[6])) + elif not stem.util.connection.is_valid_port(r_comp[7], allow_zero = True): + raise ValueError("%s DirPort is invalid: %s" % (desc._name(), r_comp[7])) + elif not (r_comp[6].isdigit() and r_comp[7].isdigit()): + return + + desc.nickname = r_comp[0] + desc.fingerprint = _decode_fingerprint(r_comp[1], validate) + if include_digest: desc.digest = r_comp[2] + desc.address = r_comp[5] + desc.or_port = int(r_comp[6]) + desc.dir_port = None if r_comp[7] == '0' else int(r_comp[7]) + + try: + published = "%s %s" % (r_comp[3], r_comp[4]) + desc.published = datetime.datetime.strptime(published, "%Y-%m-%d %H:%M:%S") + except ValueError: + if validate: + raise ValueError("Publication time time wasn't parseable: r %s" % value) + +def _parse_s_line(desc, value, validate): + # "s" Flags + # example: s Named Running Stable Valid + + flags = [] if value == "" else value.split(" ") + desc.flags = flags + + if validate: + for flag in flags: + if flags.count(flag) > 1: + raise ValueError("%s had duplicate flags: s %s" % (desc._name(), value)) + elif flag == "": + raise ValueError("%s had extra whitespace on its 's' line: s %s" % (desc._name(), value)) + +def _parse_v_line(desc, value, validate): + # "v" version + # example: v Tor 0.2.2.35 + # + # The spec says that if this starts with "Tor " then what follows is a + # tor version. If not then it has "upgraded to a more sophisticated + # protocol versioning system". + + desc.version_line = value + + if value.startswith("Tor "): + try: + desc.version = stem.version.Version(value[4:]) + except ValueError, exc: + if validate: + raise ValueError("%s has a malformed tor version (%s): v %s" % (desc._name(), exc, value)) + +def _parse_w_line(desc, value, validate): + # "w" "Bandwidth=" INT ["Measured=" INT] + # example: w Bandwidth=7980 + + w_comp = value.split(" ") + + if len(w_comp) < 1: + if not validate: return + raise ValueError("%s 'w' line is blank: w %s" % (desc._name(), value)) + elif not w_comp[0].startswith("Bandwidth="): + if not validate: return + raise ValueError("%s 'w' line needs to start with a 'Bandwidth=' entry: w %s" % (desc._name(), value)) + + for w_entry in w_comp: + if '=' in w_entry: + w_key, w_value = w_entry.split('=', 1) + else: + w_key, w_value = w_entry, None + + if w_key == "Bandwidth": + if not (w_value and w_value.isdigit()): + if not validate: return + raise ValueError("%s 'Bandwidth=' entry needs to have a numeric value: w %s" % (desc._name(), value)) + + desc.bandwidth = int(w_value) + elif w_key == "Measured": + if not (w_value and w_value.isdigit()): + if not validate: return + raise ValueError("%s 'Measured=' entry needs to have a numeric value: w %s" % (desc._name(), value)) + + desc.measured = int(w_value) + else: + desc.unrecognized_bandwidth_entries.append(w_entry) + +def _parse_p_line(desc, value, validate): + # "p" ("accept" / "reject") PortList + # p reject 1-65535 + # example: p accept 80,110,143,443,993,995,6660-6669,6697,7000-7001 + + try: + desc.exit_policy = stem.exit_policy.MicrodescriptorExitPolicy(value) + except ValueError, exc: + if not validate: return + raise ValueError("%s exit policy is malformed (%s): p %s" % (desc._name(), exc, value)) + +def _parse_m_line(desc, value, validate): + # "m" methods 1*(algorithm "=" digest) + # example: m 8,9,10,11,12 sha256=g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs + + m_comp = value.split(" ") + + if not (desc.document and desc.document.is_vote): + if not validate: return + + vote_status = "vote" if desc.document else "<undefined document>" + raise ValueError("%s 'm' line should only appear in votes (appeared in a %s): m %s" % (desc._name(), vote_status, value)) + elif len(m_comp) < 1: + if not validate: return + raise ValueError("%s 'm' line needs to start with a series of methods: m %s" % (desc._name(), value)) + + try: + methods = [int(entry) for entry in m_comp[0].split(",")] + except ValueError: + if not validate: return + raise ValueError("%s microdescriptor methods should be a series of comma separated integers: m %s" % (desc._name(), value)) + + hashes = {} + + for entry in m_comp[1:]: + if not '=' in entry: + if not validate: continue + raise ValueError("%s can only have a series of 'algorithm=digest' mappings after the methods: m %s" % (desc._name(), value)) + + hash_name, digest = entry.split('=', 1) + hashes[hash_name] = digest + + if desc.microdescriptor_hashes is None: + desc.microdescriptor_hashes = [] + + desc.microdescriptor_hashes.append((methods, hashes)) + +def _decode_fingerprint(identity, validate): + """ + Decodes the 'identity' value found in consensuses into the more common hex + encoding of the relay's fingerprint. For example... + + :: + + >>> _decode_fingerprint('p1aag7VwarGxqctS7/fS0y5FU+s') + 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB' + + :param str identity: encoded fingerprint from the consensus + :param bool validate: checks validity if True + + :returns: str with the uppercase hex encoding of the relay's fingerprint + + :raises: ValueError if the result isn't a valid fingerprint + """ + + # trailing equal signs were stripped from the identity + missing_padding = 28 - len(identity) + identity += "=" * missing_padding + + fingerprint = "" + + try: + identity_decoded = base64.b64decode(identity) + except TypeError: + if not validate: return None + raise ValueError("Unable to decode identity string '%s'" % identity) + + for char in identity_decoded: + # Individual characters are either standard ascii or hex encoded, and each + # represent two hex digits. For instnace... + # + # >>> ord('\n') + # 10 + # >>> hex(10) + # '0xa' + # >>> '0xa'[2:].zfill(2).upper() + # '0A' + + fingerprint += hex(ord(char))[2:].zfill(2).upper() + + if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): + if not validate: return None + raise ValueError("Decoded '%s' to be '%s', which isn't a valid fingerprint" % (identity, fingerprint)) + + return fingerprint + diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 6eb3883..0ded3b1 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -507,24 +507,20 @@ class ServerDescriptor(stem.descriptor.Descriptor): :raises: ValueError if an issue arises in validation """ - required_fields = self._required_fields() - if required_fields: - for keyword in required_fields: - if not keyword in entries: - raise ValueError("Descriptor must have a '%s' entry" % keyword) - - single_fields = self._single_fields() - if single_fields: - for keyword in self._single_fields(): - if keyword in entries and len(entries[keyword]) > 1: - raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) + for keyword in self._required_fields(): + if not keyword in entries: + raise ValueError("Descriptor must have a '%s' entry" % keyword) + + for keyword in self._single_fields(): + if keyword in entries and len(entries[keyword]) > 1: + raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) expected_first_keyword = self._first_keyword() - if expected_first_keyword and not first_keyword == expected_first_keyword: + if expected_first_keyword and first_keyword != expected_first_keyword: raise ValueError("Descriptor must start with a '%s' entry" % expected_first_keyword) expected_last_keyword = self._last_keyword() - if expected_last_keyword and not last_keyword == expected_last_keyword: + if expected_last_keyword and last_keyword != expected_last_keyword: raise ValueError("Descriptor must end with a '%s' entry" % expected_last_keyword) if not self.exit_policy: diff --git a/test/mocking.py b/test/mocking.py index 2d656dd..808cb8d 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -27,7 +27,7 @@ calling :func:`test.mocking.revert_mocking`. get_bridge_server_descriptor - stem.descriptor.server_descriptor.BridgeDescriptor get_relay_extrainfo_descriptor - stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor get_bridge_extrainfo_descriptor - stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor - get_router_status_entry - stem.descriptor.networkstatus.RouterStatusEntry + get_router_status_entry_v3 - stem.descriptor.router_status_entry.RouterStatusEntryV3 get_directory_authority - stem.descriptor.networkstatus.DirectoryAuthority get_key_certificate - stem.descriptor.networkstatus.KeyCertificate get_network_status_document - stem.descriptor.networkstatus.NetworkStatusDocument @@ -43,6 +43,7 @@ import stem.socket import stem.descriptor.server_descriptor import stem.descriptor.extrainfo_descriptor import stem.descriptor.networkstatus +import stem.descriptor.router_status_entry # Once we've mocked a function we can't rely on its __module__ or __name__ # attributes, so instead we associate a unique 'mock_id' attribute that maps @@ -106,7 +107,7 @@ BRIDGE_EXTRAINFO_FOOTER = ( ("router-digest", "006FD96BA35E7785A6A3B8B75FE2E2435A13BDB4"), ) -ROUTER_STATUS_ENTRY_HEADER = ( +ROUTER_STATUS_ENTRY_V3_HEADER = ( ("r", "caerSidi p1aag7VwarGxqctS7/fS0y5FU+s oQZFLYe9e4A7bOkWKR7TaNxb0JE 2012-08-06 11:19:31 71.35.150.29 9001 0"), ("s", "Fast Named Running Stable Valid"), ) @@ -522,24 +523,24 @@ def get_bridge_extrainfo_descriptor(attr = None, exclude = (), content = False): else: return stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(desc_content, validate = True) -def get_router_status_entry(attr = None, exclude = (), content = False): +def get_router_status_entry_v3(attr = None, exclude = (), content = False): """ Provides the descriptor content for... - stem.descriptor.networkstatus.RouterStatusEntry + stem.descriptor.router_status_entry.RouterStatusEntryV3 :param dict attr: keyword/value mappings to be included in the descriptor :param list exclude: mandatory keywords to exclude from the descriptor :param bool content: provides the str content of the descriptor rather than the class if True - :returns: RouterStatusEntry for the requested descriptor content + :returns: RouterStatusEntryV3 for the requested descriptor content """ - desc_content = _get_descriptor_content(attr, exclude, ROUTER_STATUS_ENTRY_HEADER) + desc_content = _get_descriptor_content(attr, exclude, ROUTER_STATUS_ENTRY_V3_HEADER) if content: return desc_content else: - return stem.descriptor.networkstatus.RouterStatusEntry(desc_content, validate = True) + return stem.descriptor.router_status_entry.RouterStatusEntryV3(desc_content, validate = True) def get_directory_authority(attr = None, exclude = (), is_vote = False, content = False): """ diff --git a/test/unit/descriptor/__init__.py b/test/unit/descriptor/__init__.py index 4793560..163f33b 100644 --- a/test/unit/descriptor/__init__.py +++ b/test/unit/descriptor/__init__.py @@ -2,5 +2,12 @@ Unit tests for stem.descriptor. """ -__all__ = ["export", "reader", "extrainfo_descriptor", "server_descriptor", "networkstatus"] +__all__ = [ + "export", + "extrainfo_descriptor", + "networkstatus", + "reader", + "router_status_entry", + "server_descriptor", +] diff --git a/test/unit/descriptor/networkstatus/__init__.py b/test/unit/descriptor/networkstatus/__init__.py index b2314cc..a41defe 100644 --- a/test/unit/descriptor/networkstatus/__init__.py +++ b/test/unit/descriptor/networkstatus/__init__.py @@ -2,5 +2,5 @@ Unit tests for stem.descriptor.networkstatus. """ -__all__ = ["entry", "directory_authority", "key_certificate", "document"] +__all__ = ["directory_authority", "key_certificate", "document"] diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py index d0bb06a..e92aee2 100644 --- a/test/unit/descriptor/networkstatus/document.py +++ b/test/unit/descriptor/networkstatus/document.py @@ -8,8 +8,9 @@ import StringIO import stem.version from stem.descriptor import Flag -from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, RouterStatusEntry, DirectoryAuthority, NetworkStatusDocument, parse_file -from test.mocking import get_router_status_entry, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG +from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, DirectoryAuthority, NetworkStatusDocument, parse_file +from stem.descriptor.router_status_entry import RouterStatusEntryV3 +from test.mocking import get_router_status_entry_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG class TestNetworkStatusDocument(unittest.TestCase): def test_minimal_consensus(self): @@ -81,8 +82,8 @@ class TestNetworkStatusDocument(unittest.TestCase): Try parsing a document via the parse_file() function. """ - entry1 = get_router_status_entry({'s': "Fast"}) - entry2 = get_router_status_entry({'s': "Valid"}) + entry1 = get_router_status_entry_v3({'s': "Fast"}) + entry2 = get_router_status_entry_v3({'s': "Valid"}) content = get_network_status_document(routers = (entry1, entry2), content = True) # the document that the entries refer to should actually be the minimal @@ -620,15 +621,15 @@ class TestNetworkStatusDocument(unittest.TestCase): document. """ - entry1 = get_router_status_entry({'s': "Fast"}) - entry2 = get_router_status_entry({'s': "Valid"}) + entry1 = get_router_status_entry_v3({'s': "Fast"}) + entry2 = get_router_status_entry_v3({'s': "Valid"}) document = get_network_status_document(routers = (entry1, entry2)) self.assertEquals((entry1, entry2), document.routers) # try with an invalid RouterStatusEntry - entry3 = RouterStatusEntry(get_router_status_entry({'r': "ugabuga"}, content = True), False) + entry3 = RouterStatusEntryV3(get_router_status_entry_v3({'r': "ugabuga"}, content = True), False) content = get_network_status_document(routers = (entry3,), content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) diff --git a/test/unit/descriptor/networkstatus/entry.py b/test/unit/descriptor/networkstatus/entry.py deleted file mode 100644 index 6400f2b..0000000 --- a/test/unit/descriptor/networkstatus/entry.py +++ /dev/null @@ -1,425 +0,0 @@ -""" -Unit tests for the RouterStatusEntry of stem.descriptor.networkstatus. -""" - -import datetime -import unittest - -from stem.descriptor import Flag -from stem.descriptor.networkstatus import RouterStatusEntry, _decode_fingerprint -from stem.version import Version -from stem.exit_policy import MicrodescriptorExitPolicy -from test.mocking import get_router_status_entry, ROUTER_STATUS_ENTRY_HEADER - -class TestRouterStatusEntry(unittest.TestCase): - def test_fingerprint_decoding(self): - """ - Tests for the _decode_fingerprint() helper. - """ - - # consensus identity field and fingerprint for caerSidi and Amunet1-5 - test_values = { - 'p1aag7VwarGxqctS7/fS0y5FU+s': 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB', - 'IbhGa8T+8tyy/MhxCk/qI+EI2LU': '21B8466BC4FEF2DCB2FCC8710A4FEA23E108D8B5', - '20wYcbFGwFfMktmuffYj6Z1RM9k': 'DB4C1871B146C057CC92D9AE7DF623E99D5133D9', - 'nTv9AG1cZeFW2hXiSIEAF6JLRJ4': '9D3BFD006D5C65E156DA15E248810017A24B449E', - '/UKsQiOSGPi/6es0/ha1prNTeDI': 'FD42AC42239218F8BFE9EB34FE16B5A6B3537832', - '/nHdqoKZ6bKZixxAPzYt9Qen+Is': 'FE71DDAA8299E9B2998B1C403F362DF507A7F88B', - } - - for arg, expected in test_values.items(): - self.assertEqual(expected, _decode_fingerprint(arg, True)) - - # checks with some malformed inputs - for arg in ('', '20wYcb', '20wYcb' * 30): - self.assertRaises(ValueError, _decode_fingerprint, arg, True) - self.assertEqual(None, _decode_fingerprint(arg, False)) - - def test_minimal(self): - """ - Parses a minimal router status entry. - """ - - entry = get_router_status_entry() - - expected_flags = set([Flag.FAST, Flag.NAMED, Flag.RUNNING, Flag.STABLE, Flag.VALID]) - self.assertEqual(None, entry.document) - self.assertEqual("caerSidi", entry.nickname) - self.assertEqual("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", entry.fingerprint) - self.assertEqual("oQZFLYe9e4A7bOkWKR7TaNxb0JE", entry.digest) - self.assertEqual(datetime.datetime(2012, 8, 6, 11, 19, 31), entry.published) - self.assertEqual("71.35.150.29", entry.address) - self.assertEqual(9001, entry.or_port) - self.assertEqual(None, entry.dir_port) - self.assertEqual(expected_flags, set(entry.flags)) - self.assertEqual(None, entry.version_line) - self.assertEqual(None, entry.version) - self.assertEqual(None, entry.bandwidth) - self.assertEqual(None, entry.measured) - self.assertEqual([], entry.unrecognized_bandwidth_entries) - self.assertEqual(None, entry.exit_policy) - self.assertEqual(None, entry.microdescriptor_hashes) - self.assertEqual([], entry.get_unrecognized_lines()) - - def test_missing_fields(self): - """ - Parses a router status entry that's missing fields. - """ - - content = get_router_status_entry(exclude = ('r', 's'), content = True) - self._expect_invalid_attr(content, "address") - - content = get_router_status_entry(exclude = ('r',), content = True) - self._expect_invalid_attr(content, "address") - - content = get_router_status_entry(exclude = ('s',), content = True) - self._expect_invalid_attr(content, "flags") - - def test_unrecognized_lines(self): - """ - Parses a router status entry with new keywords. - """ - - entry = get_router_status_entry({'z': 'New tor feature: sparkly unicorns!'}) - self.assertEquals(['z New tor feature: sparkly unicorns!'], entry.get_unrecognized_lines()) - - def test_proceeding_line(self): - """ - Includes content prior to the 'r' line. - """ - - content = 'z some stuff\n' + get_router_status_entry(content = True) - self._expect_invalid_attr(content, "_unrecognized_lines", ['z some stuff']) - - def test_blank_lines(self): - """ - Includes blank lines, which should be ignored. - """ - - content = get_router_status_entry(content = True) + "\n\nv Tor 0.2.2.35\n\n" - entry = RouterStatusEntry(content) - self.assertEqual("Tor 0.2.2.35", entry.version_line) - - def test_duplicate_lines(self): - """ - Duplicates linesin the entry. - """ - - lines = get_router_status_entry(content = True).split("\n") - - for i in xrange(len(lines)): - content = "\n".join(lines[:i] + [lines[i]] + lines[i:]) - self.assertRaises(ValueError, RouterStatusEntry, content) - - entry = RouterStatusEntry(content, False) - self.assertEqual("caerSidi", entry.nickname) - - def test_missing_r_field(self): - """ - Excludes fields from the 'r' line. - """ - - components = ( - ('nickname', 'caerSidi'), - ('fingerprint', 'p1aag7VwarGxqctS7/fS0y5FU+s'), - ('digest', 'oQZFLYe9e4A7bOkWKR7TaNxb0JE'), - ('published', '2012-08-06 11:19:31'), - ('address', '71.35.150.29'), - ('or_port', '9001'), - ('dir_port', '0'), - ) - - for attr, value in components: - # construct the 'r' line without this field - test_components = [comp[1] for comp in components] - test_components.remove(value) - r_line = ' '.join(test_components) - - content = get_router_status_entry({'r': r_line}, content = True) - self._expect_invalid_attr(content, attr) - - def test_malformed_nickname(self): - """ - Parses an 'r' line with a malformed nickname. - """ - - test_values = ( - "", - "saberrider2008ReallyLongNickname", # too long - "$aberrider2008", # invalid characters - ) - - for value in test_values: - r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("caerSidi", value) - content = get_router_status_entry({'r': r_line}, content = True) - - # TODO: Initial whitespace is consumed as part of the keyword/value - # divider. This is a bug in the case of V3 router status entries, but - # proper behavior for V2 router status entries and server/extrainfo - # descriptors. - # - # I'm inclined to leave this as-is for the moment since fixing it - # requires special KEYWORD_LINE handling, and the only result of this bug - # is that our validation doesn't catch the new SP restriction on V3 - # entries. - - if value == "": value = None - - self._expect_invalid_attr(content, "nickname", value) - - def test_malformed_fingerprint(self): - """ - Parses an 'r' line with a malformed fingerprint. - """ - - test_values = ( - "", - "zzzzz", - "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", - ) - - for value in test_values: - r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("p1aag7VwarGxqctS7/fS0y5FU+s", value) - content = get_router_status_entry({'r': r_line}, content = True) - self._expect_invalid_attr(content, "fingerprint") - - def test_malformed_published_date(self): - """ - Parses an 'r' line with a malformed published date. - """ - - test_values = ( - "", - "2012-08-06 11:19:", - "2012-08-06 11:19:71", - "2012-08-06 11::31", - "2012-08-06 11:79:31", - "2012-08-06 :19:31", - "2012-08-06 41:19:31", - "2012-08- 11:19:31", - "2012-08-86 11:19:31", - "2012--06 11:19:31", - "2012-38-06 11:19:31", - "-08-06 11:19:31", - "2012-08-06 11:19:31", - ) - - for value in test_values: - r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("2012-08-06 11:19:31", value) - content = get_router_status_entry({'r': r_line}, content = True) - self._expect_invalid_attr(content, "published") - - def test_malformed_address(self): - """ - Parses an 'r' line with a malformed address. - """ - - test_values = ( - "", - "71.35.150.", - "71.35..29", - "71.35.150", - "71.35.150.256", - ) - - for value in test_values: - r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("71.35.150.29", value) - content = get_router_status_entry({'r': r_line}, content = True) - self._expect_invalid_attr(content, "address", value) - - def test_malformed_port(self): - """ - Parses an 'r' line with a malformed ORPort or DirPort. - """ - - test_values = ( - "", - "-1", - "399482", - "blarg", - ) - - for value in test_values: - for include_or_port in (False, True): - for include_dir_port in (False, True): - if not include_or_port and not include_dir_port: - continue - - r_line = ROUTER_STATUS_ENTRY_HEADER[0][1] - - if include_or_port: - r_line = r_line.replace(" 9001 ", " %s " % value) - - if include_dir_port: - r_line = r_line[:-1] + value - - attr = "or_port" if include_or_port else "dir_port" - expected = int(value) if value.isdigit() else None - - content = get_router_status_entry({'r': r_line}, content = True) - self._expect_invalid_attr(content, attr, expected) - - def test_flags(self): - """ - Handles a variety of flag inputs. - """ - - test_values = { - "": [], - "Fast": [Flag.FAST], - "Fast Valid": [Flag.FAST, Flag.VALID], - "Ugabuga": ["Ugabuga"], - } - - for s_line, expected in test_values.items(): - entry = get_router_status_entry({'s': s_line}) - self.assertEquals(expected, entry.flags) - - # tries some invalid inputs - test_values = { - "Fast ": [Flag.FAST, "", "", ""], - "Fast Valid": [Flag.FAST, "", Flag.VALID], - "Fast Fast": [Flag.FAST, Flag.FAST], - } - - for s_line, expected in test_values.items(): - content = get_router_status_entry({'s': s_line}, content = True) - self._expect_invalid_attr(content, "flags", expected) - - def test_versions(self): - """ - Handles a variety of version inputs. - """ - - test_values = { - "Tor 0.2.2.35": Version("0.2.2.35"), - "Tor 0.1.2": Version("0.1.2"), - "Torr new_stuff": None, - "new_stuff and stuff": None, - } - - for v_line, expected in test_values.items(): - entry = get_router_status_entry({'v': v_line}) - self.assertEquals(expected, entry.version) - self.assertEquals(v_line, entry.version_line) - - # tries an invalid input - content = get_router_status_entry({'v': "Tor ugabuga"}, content = True) - self._expect_invalid_attr(content, "version") - - def test_bandwidth(self): - """ - Handles a variety of 'w' lines. - """ - - test_values = { - "Bandwidth=0": (0, None, []), - "Bandwidth=63138": (63138, None, []), - "Bandwidth=11111 Measured=482": (11111, 482, []), - "Bandwidth=11111 Measured=482 Blarg!": (11111, 482, ["Blarg!"]), - } - - for w_line, expected in test_values.items(): - entry = get_router_status_entry({'w': w_line}) - self.assertEquals(expected[0], entry.bandwidth) - self.assertEquals(expected[1], entry.measured) - self.assertEquals(expected[2], entry.unrecognized_bandwidth_entries) - - # tries some invalid inputs - test_values = ( - "", - "blarg", - "Bandwidth", - "Bandwidth=", - "Bandwidth:0", - "Bandwidth 0", - "Bandwidth=-10", - "Bandwidth=10 Measured", - "Bandwidth=10 Measured=", - "Bandwidth=10 Measured=-50", - ) - - for w_line in test_values: - content = get_router_status_entry({'w': w_line}, content = True) - self._expect_invalid_attr(content) - - def test_exit_policy(self): - """ - Handles a variety of 'p' lines. - """ - - test_values = { - "reject 1-65535": MicrodescriptorExitPolicy("reject 1-65535"), - "accept 80,110,143,443": MicrodescriptorExitPolicy("accept 80,110,143,443"), - } - - for p_line, expected in test_values.items(): - entry = get_router_status_entry({'p': p_line}) - self.assertEquals(expected, entry.exit_policy) - - # tries some invalid inputs - test_values = ( - "", - "blarg", - "reject -50", - "accept 80,", - ) - - for p_line in test_values: - content = get_router_status_entry({'p': p_line}, content = True) - self._expect_invalid_attr(content, "exit_policy") - - def test_microdescriptor_hashes(self): - """ - Handles a variety of 'm' lines. - """ - - test_values = { - "8,9,10,11,12": - [([8, 9, 10, 11, 12], {})], - "8,9,10,11,12 sha256=g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs": - [([8, 9, 10, 11, 12], {"sha256": "g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs"})], - "8,9,10,11,12 sha256=g1vx9si329muxV md5=3tquWIXXySNOIwRGMeAESKs/v4DWs": - [([8, 9, 10, 11, 12], {"sha256": "g1vx9si329muxV", "md5": "3tquWIXXySNOIwRGMeAESKs/v4DWs"})], - } - - # we need a document that's a vote - mock_document = lambda x: x # just need anything with a __dict__ - mock_document.__dict__["is_vote"] = True - mock_document.__dict__["is_consensus"] = False - - for m_line, expected in test_values.items(): - content = get_router_status_entry({'m': m_line}, content = True) - entry = RouterStatusEntry(content, document = mock_document) - self.assertEquals(expected, entry.microdescriptor_hashes) - - # try without a document - content = get_router_status_entry({'m': "8,9,10,11,12"}, content = True) - self._expect_invalid_attr(content, "microdescriptor_hashes") - - # tries some invalid inputs - test_values = ( - "", - "4,a,2", - "1,2,3 stuff", - ) - - for m_line in test_values: - content = get_router_status_entry({'m': m_line}, content = True) - self.assertRaises(ValueError, RouterStatusEntry, content, True, mock_document) - - def _expect_invalid_attr(self, content, attr = None, expected_value = None): - """ - Asserts that construction will fail due to content having a malformed - attribute. If an attr is provided then we check that it matches an expected - value when we're constructed without validation. - """ - - self.assertRaises(ValueError, RouterStatusEntry, content) - entry = RouterStatusEntry(content, False) - - if attr: - self.assertEquals(expected_value, getattr(entry, attr)) - else: - self.assertEquals("caerSidi", entry.nickname) - diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py new file mode 100644 index 0000000..6816656 --- /dev/null +++ b/test/unit/descriptor/router_status_entry.py @@ -0,0 +1,425 @@ +""" +Unit tests for stem.descriptor.router_status_entry. +""" + +import datetime +import unittest + +from stem.descriptor import Flag +from stem.descriptor.router_status_entry import RouterStatusEntryV3, _decode_fingerprint +from stem.version import Version +from stem.exit_policy import MicrodescriptorExitPolicy +from test.mocking import get_router_status_entry_v3, ROUTER_STATUS_ENTRY_V3_HEADER + +class TestRouterStatusEntry(unittest.TestCase): + def test_fingerprint_decoding(self): + """ + Tests for the _decode_fingerprint() helper. + """ + + # consensus identity field and fingerprint for caerSidi and Amunet1-5 + test_values = { + 'p1aag7VwarGxqctS7/fS0y5FU+s': 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB', + 'IbhGa8T+8tyy/MhxCk/qI+EI2LU': '21B8466BC4FEF2DCB2FCC8710A4FEA23E108D8B5', + '20wYcbFGwFfMktmuffYj6Z1RM9k': 'DB4C1871B146C057CC92D9AE7DF623E99D5133D9', + 'nTv9AG1cZeFW2hXiSIEAF6JLRJ4': '9D3BFD006D5C65E156DA15E248810017A24B449E', + '/UKsQiOSGPi/6es0/ha1prNTeDI': 'FD42AC42239218F8BFE9EB34FE16B5A6B3537832', + '/nHdqoKZ6bKZixxAPzYt9Qen+Is': 'FE71DDAA8299E9B2998B1C403F362DF507A7F88B', + } + + for arg, expected in test_values.items(): + self.assertEqual(expected, _decode_fingerprint(arg, True)) + + # checks with some malformed inputs + for arg in ('', '20wYcb', '20wYcb' * 30): + self.assertRaises(ValueError, _decode_fingerprint, arg, True) + self.assertEqual(None, _decode_fingerprint(arg, False)) + + def test_minimal(self): + """ + Parses a minimal router status entry. + """ + + entry = get_router_status_entry_v3() + + expected_flags = set([Flag.FAST, Flag.NAMED, Flag.RUNNING, Flag.STABLE, Flag.VALID]) + self.assertEqual(None, entry.document) + self.assertEqual("caerSidi", entry.nickname) + self.assertEqual("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", entry.fingerprint) + self.assertEqual("oQZFLYe9e4A7bOkWKR7TaNxb0JE", entry.digest) + self.assertEqual(datetime.datetime(2012, 8, 6, 11, 19, 31), entry.published) + self.assertEqual("71.35.150.29", entry.address) + self.assertEqual(9001, entry.or_port) + self.assertEqual(None, entry.dir_port) + self.assertEqual(expected_flags, set(entry.flags)) + self.assertEqual(None, entry.version_line) + self.assertEqual(None, entry.version) + self.assertEqual(None, entry.bandwidth) + self.assertEqual(None, entry.measured) + self.assertEqual([], entry.unrecognized_bandwidth_entries) + self.assertEqual(None, entry.exit_policy) + self.assertEqual(None, entry.microdescriptor_hashes) + self.assertEqual([], entry.get_unrecognized_lines()) + + def test_missing_fields(self): + """ + Parses a router status entry that's missing fields. + """ + + content = get_router_status_entry_v3(exclude = ('r', 's'), content = True) + self._expect_invalid_attr(content, "address") + + content = get_router_status_entry_v3(exclude = ('r',), content = True) + self._expect_invalid_attr(content, "address") + + content = get_router_status_entry_v3(exclude = ('s',), content = True) + self._expect_invalid_attr(content, "flags") + + def test_unrecognized_lines(self): + """ + Parses a router status entry with new keywords. + """ + + entry = get_router_status_entry_v3({'z': 'New tor feature: sparkly unicorns!'}) + self.assertEquals(['z New tor feature: sparkly unicorns!'], entry.get_unrecognized_lines()) + + def test_proceeding_line(self): + """ + Includes content prior to the 'r' line. + """ + + content = 'z some stuff\n' + get_router_status_entry_v3(content = True) + self._expect_invalid_attr(content, "_unrecognized_lines", ['z some stuff']) + + def test_blank_lines(self): + """ + Includes blank lines, which should be ignored. + """ + + content = get_router_status_entry_v3(content = True) + "\n\nv Tor 0.2.2.35\n\n" + entry = RouterStatusEntryV3(content) + self.assertEqual("Tor 0.2.2.35", entry.version_line) + + def test_duplicate_lines(self): + """ + Duplicates linesin the entry. + """ + + lines = get_router_status_entry_v3(content = True).split("\n") + + for i in xrange(len(lines)): + content = "\n".join(lines[:i] + [lines[i]] + lines[i:]) + self.assertRaises(ValueError, RouterStatusEntryV3, content) + + entry = RouterStatusEntryV3(content, False) + self.assertEqual("caerSidi", entry.nickname) + + def test_missing_r_field(self): + """ + Excludes fields from the 'r' line. + """ + + components = ( + ('nickname', 'caerSidi'), + ('fingerprint', 'p1aag7VwarGxqctS7/fS0y5FU+s'), + ('digest', 'oQZFLYe9e4A7bOkWKR7TaNxb0JE'), + ('published', '2012-08-06 11:19:31'), + ('address', '71.35.150.29'), + ('or_port', '9001'), + ('dir_port', '0'), + ) + + for attr, value in components: + # construct the 'r' line without this field + test_components = [comp[1] for comp in components] + test_components.remove(value) + r_line = ' '.join(test_components) + + content = get_router_status_entry_v3({'r': r_line}, content = True) + self._expect_invalid_attr(content, attr) + + def test_malformed_nickname(self): + """ + Parses an 'r' line with a malformed nickname. + """ + + test_values = ( + "", + "saberrider2008ReallyLongNickname", # too long + "$aberrider2008", # invalid characters + ) + + for value in test_values: + r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1].replace("caerSidi", value) + content = get_router_status_entry_v3({'r': r_line}, content = True) + + # TODO: Initial whitespace is consumed as part of the keyword/value + # divider. This is a bug in the case of V3 router status entries, but + # proper behavior for V2 router status entries and server/extrainfo + # descriptors. + # + # I'm inclined to leave this as-is for the moment since fixing it + # requires special KEYWORD_LINE handling, and the only result of this bug + # is that our validation doesn't catch the new SP restriction on V3 + # entries. + + if value == "": value = None + + self._expect_invalid_attr(content, "nickname", value) + + def test_malformed_fingerprint(self): + """ + Parses an 'r' line with a malformed fingerprint. + """ + + test_values = ( + "", + "zzzzz", + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + ) + + for value in test_values: + r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1].replace("p1aag7VwarGxqctS7/fS0y5FU+s", value) + content = get_router_status_entry_v3({'r': r_line}, content = True) + self._expect_invalid_attr(content, "fingerprint") + + def test_malformed_published_date(self): + """ + Parses an 'r' line with a malformed published date. + """ + + test_values = ( + "", + "2012-08-06 11:19:", + "2012-08-06 11:19:71", + "2012-08-06 11::31", + "2012-08-06 11:79:31", + "2012-08-06 :19:31", + "2012-08-06 41:19:31", + "2012-08- 11:19:31", + "2012-08-86 11:19:31", + "2012--06 11:19:31", + "2012-38-06 11:19:31", + "-08-06 11:19:31", + "2012-08-06 11:19:31", + ) + + for value in test_values: + r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1].replace("2012-08-06 11:19:31", value) + content = get_router_status_entry_v3({'r': r_line}, content = True) + self._expect_invalid_attr(content, "published") + + def test_malformed_address(self): + """ + Parses an 'r' line with a malformed address. + """ + + test_values = ( + "", + "71.35.150.", + "71.35..29", + "71.35.150", + "71.35.150.256", + ) + + for value in test_values: + r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1].replace("71.35.150.29", value) + content = get_router_status_entry_v3({'r': r_line}, content = True) + self._expect_invalid_attr(content, "address", value) + + def test_malformed_port(self): + """ + Parses an 'r' line with a malformed ORPort or DirPort. + """ + + test_values = ( + "", + "-1", + "399482", + "blarg", + ) + + for value in test_values: + for include_or_port in (False, True): + for include_dir_port in (False, True): + if not include_or_port and not include_dir_port: + continue + + r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1] + + if include_or_port: + r_line = r_line.replace(" 9001 ", " %s " % value) + + if include_dir_port: + r_line = r_line[:-1] + value + + attr = "or_port" if include_or_port else "dir_port" + expected = int(value) if value.isdigit() else None + + content = get_router_status_entry_v3({'r': r_line}, content = True) + self._expect_invalid_attr(content, attr, expected) + + def test_flags(self): + """ + Handles a variety of flag inputs. + """ + + test_values = { + "": [], + "Fast": [Flag.FAST], + "Fast Valid": [Flag.FAST, Flag.VALID], + "Ugabuga": ["Ugabuga"], + } + + for s_line, expected in test_values.items(): + entry = get_router_status_entry_v3({'s': s_line}) + self.assertEquals(expected, entry.flags) + + # tries some invalid inputs + test_values = { + "Fast ": [Flag.FAST, "", "", ""], + "Fast Valid": [Flag.FAST, "", Flag.VALID], + "Fast Fast": [Flag.FAST, Flag.FAST], + } + + for s_line, expected in test_values.items(): + content = get_router_status_entry_v3({'s': s_line}, content = True) + self._expect_invalid_attr(content, "flags", expected) + + def test_versions(self): + """ + Handles a variety of version inputs. + """ + + test_values = { + "Tor 0.2.2.35": Version("0.2.2.35"), + "Tor 0.1.2": Version("0.1.2"), + "Torr new_stuff": None, + "new_stuff and stuff": None, + } + + for v_line, expected in test_values.items(): + entry = get_router_status_entry_v3({'v': v_line}) + self.assertEquals(expected, entry.version) + self.assertEquals(v_line, entry.version_line) + + # tries an invalid input + content = get_router_status_entry_v3({'v': "Tor ugabuga"}, content = True) + self._expect_invalid_attr(content, "version") + + def test_bandwidth(self): + """ + Handles a variety of 'w' lines. + """ + + test_values = { + "Bandwidth=0": (0, None, []), + "Bandwidth=63138": (63138, None, []), + "Bandwidth=11111 Measured=482": (11111, 482, []), + "Bandwidth=11111 Measured=482 Blarg!": (11111, 482, ["Blarg!"]), + } + + for w_line, expected in test_values.items(): + entry = get_router_status_entry_v3({'w': w_line}) + self.assertEquals(expected[0], entry.bandwidth) + self.assertEquals(expected[1], entry.measured) + self.assertEquals(expected[2], entry.unrecognized_bandwidth_entries) + + # tries some invalid inputs + test_values = ( + "", + "blarg", + "Bandwidth", + "Bandwidth=", + "Bandwidth:0", + "Bandwidth 0", + "Bandwidth=-10", + "Bandwidth=10 Measured", + "Bandwidth=10 Measured=", + "Bandwidth=10 Measured=-50", + ) + + for w_line in test_values: + content = get_router_status_entry_v3({'w': w_line}, content = True) + self._expect_invalid_attr(content) + + def test_exit_policy(self): + """ + Handles a variety of 'p' lines. + """ + + test_values = { + "reject 1-65535": MicrodescriptorExitPolicy("reject 1-65535"), + "accept 80,110,143,443": MicrodescriptorExitPolicy("accept 80,110,143,443"), + } + + for p_line, expected in test_values.items(): + entry = get_router_status_entry_v3({'p': p_line}) + self.assertEquals(expected, entry.exit_policy) + + # tries some invalid inputs + test_values = ( + "", + "blarg", + "reject -50", + "accept 80,", + ) + + for p_line in test_values: + content = get_router_status_entry_v3({'p': p_line}, content = True) + self._expect_invalid_attr(content, "exit_policy") + + def test_microdescriptor_hashes(self): + """ + Handles a variety of 'm' lines. + """ + + test_values = { + "8,9,10,11,12": + [([8, 9, 10, 11, 12], {})], + "8,9,10,11,12 sha256=g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs": + [([8, 9, 10, 11, 12], {"sha256": "g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs"})], + "8,9,10,11,12 sha256=g1vx9si329muxV md5=3tquWIXXySNOIwRGMeAESKs/v4DWs": + [([8, 9, 10, 11, 12], {"sha256": "g1vx9si329muxV", "md5": "3tquWIXXySNOIwRGMeAESKs/v4DWs"})], + } + + # we need a document that's a vote + mock_document = lambda x: x # just need anything with a __dict__ + mock_document.__dict__["is_vote"] = True + mock_document.__dict__["is_consensus"] = False + + for m_line, expected in test_values.items(): + content = get_router_status_entry_v3({'m': m_line}, content = True) + entry = RouterStatusEntryV3(content, document = mock_document) + self.assertEquals(expected, entry.microdescriptor_hashes) + + # try without a document + content = get_router_status_entry_v3({'m': "8,9,10,11,12"}, content = True) + self._expect_invalid_attr(content, "microdescriptor_hashes") + + # tries some invalid inputs + test_values = ( + "", + "4,a,2", + "1,2,3 stuff", + ) + + for m_line in test_values: + content = get_router_status_entry_v3({'m': m_line}, content = True) + self.assertRaises(ValueError, RouterStatusEntryV3, content, True, mock_document) + + def _expect_invalid_attr(self, content, attr = None, expected_value = None): + """ + Asserts that construction will fail due to content having a malformed + attribute. If an attr is provided then we check that it matches an expected + value when we're constructed without validation. + """ + + self.assertRaises(ValueError, RouterStatusEntryV3, content) + entry = RouterStatusEntryV3(content, False) + + if attr: + self.assertEquals(expected_value, getattr(entry, attr)) + else: + self.assertEquals("caerSidi", entry.nickname) +

1 0

[stem/master] Supporting microdescriptor flavored consensuses
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 4216b5f1d5762d229945306508ea078c9fd1902c Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Oct 7 18:47:27 2012 -0700 Supporting microdescriptor flavored consensuses Adding support for microdescriptor flavored consensuses into the NetworkStatusDocument class. It made sense to have a separate class for it, but on the other hand it *is* still a v3 consensus and the only impact the flavor has is alternate router status entries so just blending a 'flavor' and 'is_microdescriptor' attribute in. --- stem/descriptor/networkstatus.py | 208 ++++-------------------- test/unit/descriptor/networkstatus/document.py | 72 ++++++++- 2 files changed, 100 insertions(+), 180 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 38207b8..d9aab4f 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -156,11 +156,9 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): document_content = "".join(header + footer) if not is_microdescriptor: - document = NetworkStatusDocument(document_content, validate) router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3 else: - document = MicrodescriptorConsensus(document_content, validate) - router_type = RouterMicrodescriptor + router_type = stem.descriptor.router_status_entry.RouterStatusEntryMicroV3 desc_iterator = _get_entries( document_file, @@ -169,7 +167,7 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): entry_keyword = ROUTERS_START, start_position = routers_start, end_position = routers_end, - extra_args = (document,), + extra_args = (NetworkStatusDocument(document_content, validate),), ) for desc in desc_iterator: @@ -221,9 +219,11 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :var tuple routers: RouterStatusEntryV3 contained in the document - :var str version: **\*** document version + :var int version: **\*** document version + :var str version_flavor: **\*** flavor associated with the document (such as 'microdesc') :var bool is_consensus: **\*** true if the document is a consensus :var bool is_vote: **\*** true if the document is a vote + :var bool is_microdescriptor: **\*** true if this is a microdescriptor flavored document, false otherwise :var datetime valid_after: **\*** time when the consensus became valid :var datetime fresh_until: **\*** time when the next consensus should be produced :var datetime valid_until: **\*** time when this consensus becomes obsolete @@ -262,6 +262,14 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): document_file = StringIO(raw_content) self._header = _DocumentHeader(document_file, validate, default_params) + self._unrecognized_lines = [] + + # merge header attributes into us + for attr, value in vars(self._header).items(): + if attr != "_unrecognized_lines": + setattr(self, attr, value) + else: + self._unrecognized_lines += value self.directory_authorities = tuple(_get_entries( document_file, @@ -272,28 +280,29 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): extra_args = (self._header.is_vote,), )) + if not self._header.is_microdescriptor: + router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3 + else: + router_type = stem.descriptor.router_status_entry.RouterStatusEntryMicroV3 + self.routers = tuple(_get_entries( document_file, validate, - entry_class = self._get_router_type(), + entry_class = router_type, entry_keyword = ROUTERS_START, section_end_keywords = FOOTER_START, extra_args = (self,), )) self._footer = _DocumentFooter(document_file, validate, self._header) - self._unrecognized_lines = [] - # copy the header and footer attributes into us - for attr, value in vars(self._header).items() + vars(self._footer).items(): + # merge header attributes into us + for attr, value in vars(self._footer).items(): if attr != "_unrecognized_lines": setattr(self, attr, value) else: self._unrecognized_lines += value - def _get_router_type(self): - return stem.descriptor.router_status_entry.RouterStatusEntryV3 - def meets_consensus_method(self, method): """ Checks if we meet the given consensus-method. This works for both votes and @@ -319,8 +328,10 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): class _DocumentHeader(object): def __init__(self, document_file, validate, default_params): self.version = None + self.version_flavor = None self.is_consensus = True self.is_vote = False + self.is_microdescriptor = False self.consensus_methods = [] self.published = None self.consensus_method = None @@ -362,12 +373,20 @@ class _DocumentHeader(object): if keyword == 'network-status-version': # "network-status-version" version - self.version = value + if ' ' in value: + version, flavor = value.split(' ', 1) + else: + version, flavor = value, None + + if not version.isdigit(): + if not validate: continue + raise ValueError("Network status document has a non-numeric version: %s" % line) - # TODO: Obviously not right when we extend this to parse v2 documents, - # but we'll cross that bridge when we come to it. + self.version = int(version) + self.version_flavor = flavor + self.is_microdescriptor = flavor == 'microdesc' - if validate and self.version != "3": + if validate and self.version != 3: raise ValueError("Expected a version 3 network status documents, got version '%s' instead" % self.version) elif keyword == 'vote-status': # "vote-status" type @@ -1034,160 +1053,3 @@ class DocumentSignature(object): return 0 -class MicrodescriptorConsensus(NetworkStatusDocument): - """ - A v3 microdescriptor consensus. - - :var str version: **\*** a document format version. For v3 microdescriptor consensuses this is "3 microdesc" - :var bool is_consensus: **\*** true if the document is a consensus - :var bool is_vote: **\*** true if the document is a vote - :var int consensus_method: **~** consensus method used to generate a consensus - :var datetime valid_after: **\*** time when the consensus becomes valid - :var datetime fresh_until: **\*** time until when the consensus is considered to be fresh - :var datetime valid_until: **\*** time until when the consensus is valid - :var int vote_delay: **\*** number of seconds allowed for collecting votes from all authorities - :var int dist_delay: number of seconds allowed for collecting signatures from all authorities - :var list client_versions: list of recommended Tor client versions - :var list server_versions: list of recommended Tor server versions - :var list known_flags: **\*** list of known router flags - :var list params: dict of parameter(str) => value(int) mappings - :var list directory_authorities: **\*** list of DirectoryAuthority objects that have generated this document - :var dict bandwidth_weights: **~** dict of weight(str) => value(int) mappings - :var list signatures: **\*** list of signatures this document has - - | **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined - | **~** attribute appears only in consensuses - """ - - def _get_router_type(self): - return RouterMicrodescriptor - - def _validate_network_status_version(self): - return self.version == "3 microdesc" - -class RouterMicrodescriptor(stem.descriptor.router_status_entry.RouterStatusEntry): - """ - Router microdescriptor object. Parses and stores router information in a router - microdescriptor from a v3 microdescriptor consensus. - - :var MicrodescriptorConsensus document: **\*** document this descriptor came from - - :var str nickname: **\*** router's nickname - :var str fingerprint: **\*** router's fingerprint - :var datetime published: **\*** router's publication - :var str ip: **\*** router's IP address - :var int or_port: **\*** router's ORPort - :var int dir_port: **\*** router's DirPort - - :var list flags: **\*** list of status flags - - :var :class:`stem.version.Version`,str version: Version of the Tor protocol this router is running - - :var int bandwidth: router's claimed bandwidth - :var int measured_bandwidth: router's measured bandwidth - - :var str digest: base64 of the hash of the router's microdescriptor with trailing =s omitted - - | **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined - """ - - def __init__(self, raw_contents, validate, document): - """ - Parse a router descriptor in a v3 microdescriptor consensus and provide a new - RouterMicrodescriptor object. - - :param str raw_content: router descriptor content to be parsed - :param MicrodescriptorConsensus document: document this descriptor came from - :param bool validate: whether the router descriptor should be validated - - :raises: ValueError if the descriptor data is invalid - """ - - super(RouterMicrodescriptor, self).__init__(raw_contents, validate, document) - - self.document = document - - def _parse(self, raw_content, validate): - """ - :param dict raw_content: router descriptor contents to be parsed - :param bool validate: checks the validity of descriptor content if True - - :raises: ValueError if an error occures in validation - """ - - content = StringIO(raw_content) - seen_keywords = set() - peek_check_kw = lambda keyword: keyword == _peek_keyword(content) - - r = _read_keyword_line("r", content, validate) - # r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0 - # "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort SP DirPort NL - if r: - seen_keywords.add("r") - values = r.split(" ") - self.nickname, self.fingerprint = values[0], _decode_fingerprint(values[1], validate) - self.published = _strptime(" ".join((values[2], values[3])), validate) - self.ip, self.or_port, self.dir_port = values[4], int(values[5]), int(values[6]) - if self.dir_port == 0: self.dir_port = None - elif validate: raise ValueError("Invalid router descriptor: empty 'r' line") - - while _peek_line(content): - if peek_check_kw("s"): - if "s" in seen_keywords: raise ValueError("Invalid router descriptor: 's' line appears twice") - line = _read_keyword_line("s", content, validate) - if not line: continue - seen_keywords.add("s") - # s Named Running Stable Valid - #A series of space-separated status flags, in *lexical order* - self.flags = line.split(" ") - - elif peek_check_kw("v"): - if "v" in seen_keywords: raise ValueError("Invalid router descriptor: 'v' line appears twice") - line = _read_keyword_line("v", content, validate, True) - seen_keywords.add("v") - # v Tor 0.2.2.35 - if line: - if line.startswith("Tor "): - self.version = stem.version.Version(line[4:]) - else: - self.version = line - elif validate: raise ValueError("Invalid router descriptor: empty 'v' line" ) - - elif peek_check_kw("w"): - if "w" in seen_keywords: raise ValueError("Invalid router descriptor: 'w' line appears twice") - w = _read_keyword_line("w", content, validate, True) - # "w" SP "Bandwidth=" INT [SP "Measured=" INT] NL - seen_keywords.add("w") - if w: - values = w.split(" ") - if len(values) <= 2 and len(values) > 0: - key, value = values[0].split("=") - if key == "Bandwidth": self.bandwidth = int(value) - elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Bandwidth, read " + key) - - if len(values) == 2: - key, value = values[1].split("=") - if key == "Measured": self.measured_bandwidth = int(value) - elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Measured, read " + key) - elif validate: raise ValueError("Router descriptor contains invalid 'w' line") - elif validate: raise ValueError("Router descriptor contains empty 'w' line") - - elif peek_check_kw("m"): - # microdescriptor hashes - self.digest = _read_keyword_line("m", content, validate, True) - - elif validate: - raise ValueError("Router descriptor contains unrecognized trailing lines: %s" % content.readline()) - - else: - self.unrecognized_lines.append(content.readline()) # ignore unrecognized lines if we aren't validating - - def get_unrecognized_lines(self): - """ - Returns any unrecognized lines. - - :returns: a list of unrecognized lines - """ - - return self.unrecognized_lines - diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py index e92aee2..8009fbb 100644 --- a/test/unit/descriptor/networkstatus/document.py +++ b/test/unit/descriptor/networkstatus/document.py @@ -9,8 +9,8 @@ import StringIO import stem.version from stem.descriptor import Flag from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, DirectoryAuthority, NetworkStatusDocument, parse_file -from stem.descriptor.router_status_entry import RouterStatusEntryV3 -from test.mocking import get_router_status_entry_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG +from stem.descriptor.router_status_entry import RouterStatusEntryV3, RouterStatusEntryMicroV3 +from test.mocking import get_router_status_entry_v3, get_router_status_entry_micro_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG class TestNetworkStatusDocument(unittest.TestCase): def test_minimal_consensus(self): @@ -25,9 +25,11 @@ class TestNetworkStatusDocument(unittest.TestCase): Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID] self.assertEqual((), document.routers) - self.assertEqual("3", document.version) + self.assertEqual(3, document.version) + self.assertEqual(None, document.version_flavor) self.assertEqual(True, document.is_consensus) self.assertEqual(False, document.is_vote) + self.assertEqual(False, document.is_microdescriptor) self.assertEqual(9, document.consensus_method) self.assertEqual([], document.consensus_methods) self.assertEqual(None, document.published) @@ -57,7 +59,7 @@ class TestNetworkStatusDocument(unittest.TestCase): Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID] self.assertEqual((), document.routers) - self.assertEqual("3", document.version) + self.assertEqual(3, document.version) self.assertEqual(False, document.is_consensus) self.assertEqual(True, document.is_vote) self.assertEqual(None, document.consensus_method) @@ -178,13 +180,22 @@ class TestNetworkStatusDocument(unittest.TestCase): """ document = get_network_status_document({"network-status-version": "3"}) - self.assertEquals("3", document.version) + self.assertEquals(3, document.version) + self.assertEquals(None, document.version_flavor) + self.assertEquals(False, document.is_microdescriptor) + + document = get_network_status_document({"network-status-version": "3 microdesc"}) + self.assertEquals(3, document.version) + self.assertEquals('microdesc', document.version_flavor) + self.assertEquals(True, document.is_microdescriptor) content = get_network_status_document({"network-status-version": "4"}, content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) - self.assertEquals("4", document.version) + self.assertEquals(4, document.version) + self.assertEquals(None, document.version_flavor) + self.assertEquals(False, document.is_microdescriptor) def test_vote_status(self): """ @@ -616,7 +627,7 @@ class TestNetworkStatusDocument(unittest.TestCase): def test_with_router_status_entries(self): """ - Includes a router status entry within the document. This isn't to test the + Includes router status entries within the document. This isn't to test the RouterStatusEntry parsing but rather the inclusion of it within the document. """ @@ -635,6 +646,53 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertRaises(ValueError, NetworkStatusDocument, content) document = NetworkStatusDocument(content, False) self.assertEquals((entry3,), document.routers) + + # try including with a microdescriptor consensus + + content = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry1, entry2), content = True) + self.assertRaises(ValueError, NetworkStatusDocument, content) + + expected_routers = ( + RouterStatusEntryMicroV3(str(entry1), False), + RouterStatusEntryMicroV3(str(entry2), False), + ) + + document = NetworkStatusDocument(content, False) + self.assertEquals(expected_routers, document.routers) + + def test_with_microdescriptor_router_status_entries(self): + """ + Includes microdescriptor flavored router status entries within the + document. + """ + + entry1 = get_router_status_entry_micro_v3({'s': "Fast"}) + entry2 = get_router_status_entry_micro_v3({'s': "Valid"}) + document = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry1, entry2)) + + self.assertEquals((entry1, entry2), document.routers) + + # try with an invalid RouterStatusEntry + + entry3 = RouterStatusEntryMicroV3(get_router_status_entry_micro_v3({'r': "ugabuga"}, content = True), False) + content = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry3,), content = True) + + self.assertRaises(ValueError, NetworkStatusDocument, content) + document = NetworkStatusDocument(content, False) + self.assertEquals((entry3,), document.routers) + + # try including microdescriptor entries in a normal consensus + + content = get_network_status_document(routers = (entry1, entry2), content = True) + self.assertRaises(ValueError, NetworkStatusDocument, content) + + expected_routers = ( + RouterStatusEntryV3(str(entry1), False), + RouterStatusEntryV3(str(entry2), False), + ) + + document = NetworkStatusDocument(content, False) + self.assertEquals(expected_routers, document.routers) def test_with_directory_authorities(self): """

1 0

[stem/master] Dropping unused descriptor helpers
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 60868f5f161dc0db59f4c19019b75c29f417b37c Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Oct 7 18:53:18 2012 -0700 Dropping unused descriptor helpers Removing the helper functions for the prior parsers of network status documents. --- stem/descriptor/__init__.py | 73 -------------------------------------- stem/descriptor/networkstatus.py | 13 +++---- 2 files changed, 5 insertions(+), 81 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index ef3d558..28aeed1 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -169,72 +169,6 @@ class Descriptor(object): def __str__(self): return self._raw_contents -def _peek_line(descriptor_file): - """ - Returns the line at the current offset of descriptor_file. - - :param file descriptor_file: file with the descriptor content - - :returns: line at the current offset of descriptor_file - """ - - last_position = descriptor_file.tell() - line = descriptor_file.readline() - descriptor_file.seek(last_position) - - return line - -def _peek_keyword(descriptor_file): - """ - Returns the keyword at the current offset of descriptor_file. Respects the - "opt" keyword and returns the next keyword instead. - - :param file descriptor_file: file with the descriptor content - - :returns: keyword at the current offset of descriptor_file - """ - - line = _peek_line(descriptor_file) - - if line.startswith("opt "): - line = line[4:] - if not line: return None - - return line.split(" ", 1)[0].rstrip("\n") - -def _read_keyword_line(keyword, descriptor_file, validate = True, optional = False): - """ - Returns the rest of the line if the first keyword matches the given keyword. If - it doesn't, a ValueError is raised if optional and validate are True, if - not, None is returned. - - Respects the opt keyword and returns the next keyword if the first is "opt". - - :param str keyword: keyword the line must begin with - :param bool descriptor_file: file/file-like object containing descriptor data - :param bool validate: validation is enabled - :param bool optional: if the current line must begin with the given keyword - - :returns: the text after the keyword if the keyword matches the one provided, otherwise returns None or raises an exception - - :raises: ValueError if a non-optional keyword doesn't match when validation is enabled - """ - - line = _peek_line(descriptor_file) - if not line: - if not optional and validate: - raise ValueError("Unexpected end of document") - return None - - if line.startswith("opt "): - line = line[4:] - if re.match("^" + re.escape(keyword) + "($| )", line): - descriptor_file.readline() - return line[len(keyword):].strip() - elif not optional and validate: - raise ValueError("Error parsing network status document: Expected %s, received: %s" % (keyword, line)) - else: return None - def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False, skip = False, end_position = None): """ Reads from the descriptor file until we get to one of the given keywords or reach the @@ -399,10 +333,3 @@ def _get_descriptor_components(raw_contents, validate, extra_keywords = ()): return entries, first_keyword, last_keyword, extra_entries -def _strptime(string, validate = True, optional = False): - try: - return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") - except ValueError, exc: - if validate or not optional: raise exc - else: return None - diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index d9aab4f..dea16f4 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -52,9 +52,6 @@ import stem.version import stem.exit_policy import stem.util.tor_tools -from stem.descriptor import _read_until_keywords, _peek_keyword, _strptime -from stem.descriptor import _read_keyword_line, _get_pseudo_pgp_block, _peek_line - # Network status document are either a 'vote' or 'consensus', with different # mandatory fields for each. Both though require that their fields appear in a # specific order. This is an ordered listing of the following... @@ -146,10 +143,10 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): # getting the document without the routers section - header = _read_until_keywords((ROUTERS_START, FOOTER_START), document_file) + header = stem.descriptor._read_until_keywords((ROUTERS_START, FOOTER_START), document_file) routers_start = document_file.tell() - _read_until_keywords(FOOTER_START, document_file, skip = True) + stem.descriptor._read_until_keywords(FOOTER_START, document_file, skip = True) routers_end = document_file.tell() footer = document_file.readlines() @@ -203,14 +200,14 @@ def _get_entries(document_file, validate, entry_class, entry_keyword, start_posi if end_position is None: if section_end_keywords: - _read_until_keywords(section_end_keywords, document_file, skip = True) + stem.descriptor._read_until_keywords(section_end_keywords, document_file, skip = True) end_position = document_file.tell() else: raise ValueError("Either a end_position or section_end_keywords must be provided") document_file.seek(start_position) while document_file.tell() < end_position: - desc_content = "".join(_read_until_keywords(entry_keyword, document_file, ignore_first = True, end_position = end_position)) + desc_content = "".join(stem.descriptor._read_until_keywords(entry_keyword, document_file, ignore_first = True, end_position = end_position)) yield entry_class(desc_content, validate, *extra_args) class NetworkStatusDocument(stem.descriptor.Descriptor): @@ -347,7 +344,7 @@ class _DocumentHeader(object): self._unrecognized_lines = [] - content = "".join(_read_until_keywords((AUTH_START, ROUTERS_START, FOOTER_START), document_file)) + content = "".join(stem.descriptor._read_until_keywords((AUTH_START, ROUTERS_START, FOOTER_START), document_file)) entries = stem.descriptor._get_descriptor_components(content, validate)[0] self._parse(entries, validate)

1 0

[stem/master] Dropping test for empty dir-source fields
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 98d8e4685bab0dd107f5dd6f185492d6f3a5c136 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 8 08:36:47 2012 -0700 Dropping test for empty dir-source fields Nick replied on 'https://trac.torproject.org/7055' saying that the fields can't be empty strings, so dropping the test. Also adding a validation check that the hostname isn't an empty string. --- stem/descriptor/networkstatus.py | 3 ++ .../networkstatus/directory_authority.py | 32 -------------------- 2 files changed, 3 insertions(+), 32 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index dea16f4..a2c13a2 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -801,6 +801,9 @@ class DirectoryAuthority(stem.descriptor.Descriptor): raise ValueError("Authority's nickname is invalid: %s" % dir_source_comp[0]) elif not stem.util.tor_tools.is_valid_fingerprint(dir_source_comp[1]): raise ValueError("Authority's fingerprint is invalid: %s" % dir_source_comp[1]) + elif not dir_source_comp[2]: + # https://trac.torproject.org/7055 + raise ValueError("Authority's hostname can't be blank: %s" % line) elif not stem.util.connection.is_valid_ip_address(dir_source_comp[3]): raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[3]) elif not stem.util.connection.is_valid_port(dir_source_comp[4], allow_zero = True): diff --git a/test/unit/descriptor/networkstatus/directory_authority.py b/test/unit/descriptor/networkstatus/directory_authority.py index b032b48..f01604f 100644 --- a/test/unit/descriptor/networkstatus/directory_authority.py +++ b/test/unit/descriptor/networkstatus/directory_authority.py @@ -125,38 +125,6 @@ class TestDirectoryAuthority(unittest.TestCase): self.assertEqual(None, authority.dir_port) self.assertEqual(None, authority.or_port) - def test_empty_values(self): - """ - The 'dir-source' line has a couple string values where anything (without - spaces) would be valud. Check that we're ok with the value being an empty - string. - """ - - # TODO: Test presently fails because a missing nickname makes us think that - # a field is missing. This is technically a bug caused by us ignoring an - # idiosyncrasy with how v3 documents are formatted. With all descriptor - # types *except* v3 documents a keyword and value is split by any amount - # of whitespace. With a v3 document it must be a single space. - # - # When we have an empty nickname the value starts with a space, causing our - # keyword/value regex to gobble the extra space (making the field - # disappear). Checking with Nick before investing any further effort into - # this... - # https://trac.torproject.org/7055 - - test.runner.skip(self, "https://trac.torproject.org/7055") - return - - # drop the authority nickname - dir_source = AUTHORITY_HEADER[0][1].replace('turtles', '') - authority = get_directory_authority({"dir-source": dir_source}) - self.assertEqual('', authority.nickname) - - # drop the hostname - dir_source = AUTHORITY_HEADER[0][1].replace('no.place.com', '') - authority = get_directory_authority({"dir-source": dir_source}) - self.assertEqual('', authority.hostname) - def test_malformed_fingerprint(self): """ Includes a malformed fingerprint on the 'dir-source' line.

1 0

[stem/master] Rewriting network status module pydocs
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 3ddd3c55da9b859dcce218283ee91aab251bf230 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 8 09:27:57 2012 -0700 Rewriting network status module pydocs Revising the header documentation for the network status document module. The bit about v2 support is presently a lie, and I need to include the examples in the tests. Otherwise, this module is almost done! --- stem/descriptor/networkstatus.py | 78 +++++++++++++++++++++++-------------- 1 files changed, 48 insertions(+), 30 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 91c50e6..f6116c5 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -1,55 +1,73 @@ """ -Parsing for Tor network status documents. Currently supports parsing v3 network -status documents (both votes and consensuses). - -The network status documents also contain a list of router descriptors, -directory authorities, signatures etc. If you only need the -:class:`stem.descriptor.router_status_entry.RouterStatusEntry` objects, use -:func:`stem.descriptor.parse_file`. Other information can be accessed by -directly instantiating :class:`stem.descriptor.networkstatus.NetworkStatusDocument` -objects. - -The documents can be obtained from any of the following sources... +Parsing for Tor network status documents. This supports both the v2 and v3 +dir-spec. Documents can be obtained from a few sources... * the 'cached-consensus' file in tor's data directory * tor metrics, at https://metrics.torproject.org/data.html * directory authorities and mirrors via their DirPort +... and contain the following sections... + +* document header +* list of :class:`stem.descriptor.networkstatus.DirectoryAuthority` +* list of :class:`stem.descriptor.router_status_entry.RouterStatusEntry` +* document footer + +Of these, the router status entry section can be quite large (on the order of +hundreds of kilobytes). As such we provide a couple methods of reading network +status documents... + +* :class:`stem.descriptor.networkstatus.NetworkStatusDocument` constructor + +If read time and memory aren't a concern then you can simply use the document +constructor. Router entries are assigned to its 'routers' attribute... + :: - import stem.descriptor.networkstatus + from stem.descriptor.networkstatus import NetworkStatusDocument - nsdoc_file = open("/home/neena/.tor/cached-consensus") - try: - consensus = stem.descriptor.networkstatus.parse_file(nsdoc_file) - except ValueError: - print "Invalid cached-consensus file" + with open('.tor/cached-consensus', 'r') as consensus_file: + # Reads the full consensus into memory twice (both for the parsed and + # unparsed contents). + + consensus = NetworkStatusDocument(consensus_file.read()) + + for router in consensus.routers: + print router.nickname + +* :func:`stem.descriptor.parse_file` + +Alternatively, the parse_file() function provides an iterator for a document's +routers. Those routers refer to a 'thin' document, which doesn't have a +'routers' attribute. This allows for lower memory usage and upfront runtime. + +:: + + from stem.descriptor.networkstatus import parse_file - print "Consensus was valid between %s and %s" % (str(consensus.valid_after), str(consensus.valid_until)) + with open('.tor/cached-consensus', 'r') as consensus_file: + # Processes the routers as we read them in. The routers refer to a document + # with an unset 'routers' attribute. + + for router in parse_file(consensus_file): + print router.nickname **Module Overview:** :: - parse_file - parses a network status file and provides a NetworkStatusDocument - NetworkStatusDocument - Tor v3 network status document - +- MicrodescriptorConsensus - Microdescriptor flavoured consensus documents - DocumentSignature - Signature of a document by a directory authority - DirectoryAuthority - Directory authority defined in a v3 network status document + parse_file - parses a network status file, providing an iterator for its routers + NetworkStatusDocument - Version 3 network status document. + DocumentSignature - Signature of a document by a directory authority. + DirectoryAuthority - Directory authority as defined in a v3 network status document. """ -import re import datetime - -try: - from cStringIO import StringIO -except: - from StringIO import StringIO +from StringIO import StringIO import stem.descriptor import stem.descriptor.router_status_entry import stem.version -import stem.exit_policy import stem.util.tor_tools # Network status document are either a 'vote' or 'consensus', with different

1 0