October 2012 - tor-commits - lists.torproject.org

[stem/master] Simplifying _get_descriptor_components() usage
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 27356d41908c70fdec23d26d4f9933f8fefb71f9 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 8 08:50:13 2012 -0700 Simplifying _get_descriptor_components() usage The _get_descriptor_components() provided the first and last keywords since those are often needed for validation. However, this is pointless now that we're using an ordered dictionary (we can simply check the key listing). --- stem/descriptor/__init__.py | 19 +++++++------------ stem/descriptor/extrainfo_descriptor.py | 7 +++---- stem/descriptor/networkstatus.py | 14 +++++++------- stem/descriptor/router_status_entry.py | 9 ++++----- stem/descriptor/server_descriptor.py | 12 +++++------- 5 files changed, 26 insertions(+), 35 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 28aeed1..f0ab82d 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -275,17 +275,12 @@ def _get_descriptor_components(raw_contents, validate, extra_keywords = ()): :param list extra_keywords: entity keywords to put into a separate listing with ordering intact :returns: - tuple with the following attributes... - - * **entries (collections.OrderedDict)** - keyword => (value, pgp key) entries - * **first_keyword (str)** - keyword of the first line - * **last_keyword (str)** - keyword of the last line - * **extra_entries (list)** - lines containing entries matching extra_keywords + collections.OrderedDict with the 'keyword => (value, pgp key) entries' + mappings. If a extra_keywords was provided then this instead provides a two + value tuple, the second being a list of those entries. """ entries = collections.OrderedDict() - first_keyword = None - last_keyword = None extra_entries = [] # entries with a keyword in extra_keywords remaining_lines = raw_contents.split("\n") @@ -317,9 +312,6 @@ def _get_descriptor_components(raw_contents, validate, extra_keywords = ()): keyword, value = line_match.groups() if value is None: value = '' - if not first_keyword: first_keyword = keyword - last_keyword = keyword - try: block_contents = _get_pseudo_pgp_block(remaining_lines) except ValueError, exc: @@ -331,5 +323,8 @@ def _get_descriptor_components(raw_contents, validate, extra_keywords = ()): else: entries.setdefault(keyword, []).append((value, block_contents)) - return entries, first_keyword, last_keyword, extra_entries + if extra_keywords: + return entries, extra_entries + else: + return entries diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 8d181ae..d66ea87 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -359,8 +359,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self._unrecognized_lines = [] - entries, first_keyword, last_keyword, _ = \ - stem.descriptor._get_descriptor_components(raw_contents, validate, ()) + entries = stem.descriptor._get_descriptor_components(raw_contents, validate) if validate: for keyword in self._required_fields(): @@ -372,11 +371,11 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("The '%s' entry can only appear once in an extra-info descriptor" % keyword) expected_first_keyword = self._first_keyword() - if expected_first_keyword and not first_keyword == expected_first_keyword: + if expected_first_keyword and expected_first_keyword != entries.keys()[0]: raise ValueError("Extra-info descriptor must start with a '%s' entry" % expected_first_keyword) expected_last_keyword = self._last_keyword() - if expected_last_keyword and not last_keyword == expected_last_keyword: + if expected_last_keyword and expected_last_keyword != entries.keys()[-1]: raise ValueError("Descriptor must end with a '%s' entry" % expected_last_keyword) self._parse(entries, validate) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index a2c13a2..91c50e6 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -345,7 +345,7 @@ class _DocumentHeader(object): self._unrecognized_lines = [] content = "".join(stem.descriptor._read_until_keywords((AUTH_START, ROUTERS_START, FOOTER_START), document_file)) - entries = stem.descriptor._get_descriptor_components(content, validate)[0] + entries = stem.descriptor._get_descriptor_components(content, validate) self._parse(entries, validate) # doing this validation afterward so we know our 'is_consensus' and @@ -538,7 +538,7 @@ class _DocumentFooter(object): elif not content and not header.meets_consensus_method(9): return # footer is optional and there's nothing to parse - entries = stem.descriptor._get_descriptor_components(content, validate)[0] + entries = stem.descriptor._get_descriptor_components(content, validate) self._parse(entries, validate, header) if validate: @@ -748,9 +748,9 @@ class DirectoryAuthority(stem.descriptor.Descriptor): else: key_cert_content = None - entries, first_keyword, _, _ = stem.descriptor._get_descriptor_components(content, validate) + entries = stem.descriptor._get_descriptor_components(content, validate) - if validate and first_keyword != 'dir-source': + if validate and 'dir-source' != entries.keys()[0]: raise ValueError("Authority entries are expected to start with a 'dir-source' line:\n%s" % (content)) # check that we have mandatory fields @@ -905,12 +905,12 @@ class KeyCertificate(stem.descriptor.Descriptor): :raises: ValueError if a validity check fails """ - entries, first_keyword, last_keyword, _ = stem.descriptor._get_descriptor_components(content, validate) + entries = stem.descriptor._get_descriptor_components(content, validate) if validate: - if first_keyword != 'dir-key-certificate-version': + if 'dir-key-certificate-version' != entries.keys()[0]: raise ValueError("Key certificates must start with a 'dir-key-certificate-version' line:\n%s" % (content)) - elif last_keyword != 'dir-key-certification': + elif 'dir-key-certification' != entries.keys()[-1]: raise ValueError("Key certificates must end with a 'dir-key-certification' line:\n%s" % (content)) # check that we have mandatory fields and that our known fields only diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index f663f07..d94b2e8 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -71,8 +71,8 @@ class RouterStatusEntry(stem.descriptor.Descriptor): self._unrecognized_lines = [] - entries, first_keyword, _, _ = stem.descriptor._get_descriptor_components(content, validate) - if validate: self._check_constraints(entries, first_keyword) + entries = stem.descriptor._get_descriptor_components(content, validate) + if validate: self._check_constraints(entries) self._parse(entries, validate) def _parse(self, entries, validate): @@ -95,13 +95,12 @@ class RouterStatusEntry(stem.descriptor.Descriptor): else: self._unrecognized_lines.append("%s %s" % (keyword, value)) - def _check_constraints(self, entries, first_keyword): + def _check_constraints(self, entries): """ Does a basic check that the entries conform to this descriptor type's constraints. :param dict entries: keyword => (value, pgp key) entries - :param str first_keyword: keyword of the first line :raises: ValueError if an issue arises in validation """ @@ -114,7 +113,7 @@ class RouterStatusEntry(stem.descriptor.Descriptor): if keyword in entries and len(entries[keyword]) > 1: raise ValueError("%s can only have a single '%s' line, got %i:\n%s" % (self._name(True), keyword, len(entries[keyword]), str(self))) - if first_keyword != 'r': + if 'r' != entries.keys()[0]: raise ValueError("%s are expected to start with a 'r' line:\n%s" % (self._name(True), str(self))) def _name(self, is_plural = False): diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 0ded3b1..5e5e04a 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -240,12 +240,12 @@ class ServerDescriptor(stem.descriptor.Descriptor): # influences the resulting exit policy, but for everything else the order # does not matter so breaking it into key / value pairs. - entries, first_keyword, last_keyword, policy = \ + entries, policy = \ stem.descriptor._get_descriptor_components(raw_contents, validate, ("accept", "reject")) self.exit_policy = stem.exit_policy.ExitPolicy(*policy) self._parse(entries, validate) - if validate: self._check_constraints(entries, first_keyword, last_keyword) + if validate: self._check_constraints(entries) def digest(self): """ @@ -495,14 +495,12 @@ class ServerDescriptor(stem.descriptor.Descriptor): if self.uptime < 0 and self.tor_version >= stem.version.Version("0.1.2.7"): raise ValueError("Descriptor for version '%s' had a negative uptime value: %i" % (self.tor_version, self.uptime)) - def _check_constraints(self, entries, first_keyword, last_keyword): + def _check_constraints(self, entries): """ Does a basic check that the entries conform to this descriptor type's constraints. :param dict entries: keyword => (value, pgp key) entries - :param str first_keyword: keyword of the first line - :param str last_keyword: keyword of the last line :raises: ValueError if an issue arises in validation """ @@ -516,11 +514,11 @@ class ServerDescriptor(stem.descriptor.Descriptor): raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) expected_first_keyword = self._first_keyword() - if expected_first_keyword and first_keyword != expected_first_keyword: + if expected_first_keyword and expected_first_keyword != entries.keys()[0]: raise ValueError("Descriptor must start with a '%s' entry" % expected_first_keyword) expected_last_keyword = self._last_keyword() - if expected_last_keyword and last_keyword != expected_last_keyword: + if expected_last_keyword and expected_last_keyword != entries.keys()[-1]: raise ValueError("Descriptor must end with a '%s' entry" % expected_last_keyword) if not self.exit_policy:

1 0

[stem/master] Adding tests for the network status document examples
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 78e09a971b7c242dc1d0a6b95718b78edd5cb32b Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Oct 9 07:56:33 2012 -0700 Adding tests for the network status document examples Including unit tests for the header pydoc examples to check that they're runnable. I also changed the first example to show that the consensus file doesn't need to remain open when using that method. --- stem/descriptor/networkstatus.py | 21 ++++++++-------- test/mocking.py | 4 +++ test/unit/descriptor/networkstatus/document.py | 30 +++++++++++++++++++++++- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index f6116c5..ae0f0c3 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -26,14 +26,15 @@ constructor. Router entries are assigned to its 'routers' attribute... from stem.descriptor.networkstatus import NetworkStatusDocument - with open('.tor/cached-consensus', 'r') as consensus_file: - # Reads the full consensus into memory twice (both for the parsed and - # unparsed contents). - - consensus = NetworkStatusDocument(consensus_file.read()) - - for router in consensus.routers: - print router.nickname + # Reads the full consensus into memory twice (both for the parsed and + # unparsed contents). + + consensus_file = open('.tor/cached-consensus', 'r') + consensus = NetworkStatusDocument(consensus_file.read()) + consensus_file.close() + + for router in consensus.routers: + print router.nickname * :func:`stem.descriptor.parse_file` @@ -378,7 +379,7 @@ class _DocumentHeader(object): def _parse(self, entries, validate): for keyword, values in entries.items(): - value, block_contents = values[0] + value, _ = values[0] line = "%s %s" % (keyword, value) # all known header fields can only appear once except @@ -798,7 +799,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): raise ValueError("Authority %s shouldn't have a '%s' line:\n%s" % (type_label, keyword, content)) for keyword, values in entries.items(): - value, block_contents = values[0] + value, _ = values[0] line = "%s %s" % (keyword, value) # all known attributes can only appear at most once diff --git a/test/mocking.py b/test/mocking.py index 7c848ad..ae74166 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -10,6 +10,7 @@ calling :func:`test.mocking.revert_mocking`. revert_mocking - reverts any changes made by the mock function get_real_function - provides the non-mocked version of a function get_all_combinations - provides all combinations of attributes + support_with - makes object be compatable for use via the 'with' keyword Mocking Functions no_op - does nothing @@ -216,10 +217,13 @@ def support_with(obj): does nothing. :param object obj: object to support the 'with' keyword + + :returns: input object """ obj.__dict__["__enter__"] = return_value(obj) obj.__dict__["__exit__"] = no_op() + return obj def mock(target, mock_call, target_module=None): """ diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py index 8009fbb..246f7aa 100644 --- a/test/unit/descriptor/networkstatus/document.py +++ b/test/unit/descriptor/networkstatus/document.py @@ -2,6 +2,8 @@ Unit tests for the NetworkStatusDocument of stem.descriptor.networkstatus. """ +from __future__ import with_statement + import datetime import unittest import StringIO @@ -10,7 +12,7 @@ import stem.version from stem.descriptor import Flag from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, DirectoryAuthority, NetworkStatusDocument, parse_file from stem.descriptor.router_status_entry import RouterStatusEntryV3, RouterStatusEntryMicroV3 -from test.mocking import get_router_status_entry_v3, get_router_status_entry_micro_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG +from test.mocking import support_with, get_router_status_entry_v3, get_router_status_entry_micro_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG class TestNetworkStatusDocument(unittest.TestCase): def test_minimal_consensus(self): @@ -79,6 +81,32 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertEqual([DOC_SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines()) + def test_examples(self): + """ + Run something similar to the examples in the header pydocs. + """ + + # makes a consensus with a couple routers, both with the same nickname + + entry1 = get_router_status_entry_v3({'s': "Fast"}) + entry2 = get_router_status_entry_v3({'s': "Valid"}) + content = get_network_status_document(routers = (entry1, entry2), content = True) + + # first example: parsing via the NetworkStatusDocument constructor + + consensus_file = StringIO.StringIO(content) + consensus = NetworkStatusDocument(consensus_file.read()) + consensus_file.close() + + for router in consensus.routers: + self.assertEqual('caerSidi', router.nickname) + + # second example: using parse_file + + with support_with(StringIO.StringIO(content)) as consensus_file: + for router in parse_file(consensus_file): + self.assertEqual('caerSidi', router.nickname) + def test_parse_file(self): """ Try parsing a document via the parse_file() function.

1 0

[stem/master] Addressing issues spotted by integ tests
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit c20cfdcd6d87a04d23c508b8b333086c77000d77 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Oct 9 09:15:41 2012 -0700 Addressing issues spotted by integ tests Enough of this unit testing, time to run our new parser against actual network status content. Unsurprisingly this ran into a couple issues... - Microdescriptors have an extra field on their 'directory-signature' lines. This is undocumented so it'll also need a spec fix... https://trac.torproject.org/7072 - Our parser for 'directory-signature' was only reading the first one, rather than iterating over all entries. Most of the rest of the changes are just revising the integ tests that Ravi wrote to accomidate changes I've made to the classes. --- stem/descriptor/networkstatus.py | 35 ++++++++++++++++----- test/integ/descriptor/networkstatus.py | 16 +++++---- test/mocking.py | 8 ++++- test/unit/descriptor/networkstatus/document.py | 38 +++++++++++++++++++++++- 4 files changed, 79 insertions(+), 18 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index ae0f0c3..ce3a00c 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -593,12 +593,25 @@ class _DocumentFooter(object): raise ValueError("A network status document's 'bandwidth-weights' entries should be '%s', got '%s'" % (expected_label, actual_label)) elif keyword == "directory-signature": - if not " " in value or not block_contents: - if not validate: continue - raise ValueError("Authority signatures in a network status document are expected to be of the form 'directory-signature FINGERPRINT KEY_DIGEST\\nSIGNATURE', got:\n%s" % line) - - fingerprint, key_digest = value.split(" ", 1) - self.signatures.append(DocumentSignature(fingerprint, key_digest, block_contents, validate)) + for sig_value, block_contents in values: + if not header.is_microdescriptor: + expected_spaces = 1 + format_label = 'directory-signature FINGERPRINT KEY_DIGEST' + else: + expected_spaces = 2 + format_label = 'directory-signature METHOD FINGERPRINT KEY_DIGEST' + + if sig_value.count(" ") != expected_spaces or not block_contents: + if not validate: continue + raise ValueError("Authority signatures in a network status document are expected to be of the form '%s\\nSIGNATURE', got:\n%s\n%s" % (format_label, sig_value, block_contents)) + + if not header.is_microdescriptor: + method = None + fingerprint, key_digest = sig_value.split(" ", 1) + else: + method, fingerprint, key_digest = sig_value.split(" ", 2) + + self.signatures.append(DocumentSignature(method, fingerprint, key_digest, block_contents, validate)) def _check_for_missing_and_disallowed_fields(header, entries, fields): """ @@ -1033,12 +1046,11 @@ class KeyCertificate(stem.descriptor.Descriptor): return str(self) > str(other) -# TODO: microdescriptors have a slightly different format (including a -# 'method') - should probably be a subclass class DocumentSignature(object): """ Directory signature of a v3 network status document. + :var str method: method used to make the signature, this only appears in microdescriptor consensuses :var str identity: fingerprint of the authority that made the signature :var str key_digest: digest of the signing key :var str signature: document signature @@ -1047,7 +1059,7 @@ class DocumentSignature(object): :raises: ValueError if a validity check fails """ - def __init__(self, identity, key_digest, signature, validate = True): + def __init__(self, method, identity, key_digest, signature, validate = True): # Checking that these attributes are valid. Technically the key # digest isn't a fingerprint, but it has the same characteristics. @@ -1058,6 +1070,11 @@ class DocumentSignature(object): if not stem.util.tor_tools.is_valid_fingerprint(key_digest): raise ValueError("Malformed key digest (%s) in the document signature" % (key_digest)) + # TODO: The method field is undocumented so I'm just guessing how we should + # handle it. Ticket for clarification... + # https://trac.torproject.org/7072 + + self.method = method self.identity = identity self.key_digest = key_digest self.signature = signature diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index aa91ab3..31f2c73 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -82,11 +82,12 @@ class TestNetworkStatus(unittest.TestCase): descriptor_path = test.integ.descriptor.get_resource("cached-consensus") descriptor_file = file(descriptor_path) - desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) + desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read(), default_params = False) router1 = desc.routers[0] descriptor_file.close() - self.assertEquals("3", desc.version) + self.assertEquals(3, desc.version) + self.assertEquals(None, desc.version_flavor) self.assertEquals(True, desc.is_consensus) self.assertEquals(False, desc.is_vote) self.assertEquals([], desc.consensus_methods) @@ -123,8 +124,8 @@ class TestNetworkStatus(unittest.TestCase): self.assertEquals(8, len(desc.directory_authorities)) self.assertEquals("tor26", desc.directory_authorities[0].nickname) self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.directory_authorities[0].fingerprint) + self.assertEquals("86.59.21.38", desc.directory_authorities[0].hostname) self.assertEquals("86.59.21.38", desc.directory_authorities[0].address) - self.assertEquals("86.59.21.38", desc.directory_authorities[0].ip) self.assertEquals(80, desc.directory_authorities[0].dir_port) self.assertEquals(443, desc.directory_authorities[0].or_port) self.assertEquals("Peter Palfrader", desc.directory_authorities[0].contact) @@ -175,11 +176,12 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= descriptor_path = test.integ.descriptor.get_resource("vote") descriptor_file = file(descriptor_path) - desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) + desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read(), default_params = False) router1 = desc.routers[0] descriptor_file.close() - self.assertEquals("3", desc.version) + self.assertEquals(3, desc.version) + self.assertEquals(None, desc.version_flavor) self.assertEquals(False, desc.is_consensus) self.assertEquals(True, desc.is_vote) self.assertEquals(range(1, 13), desc.consensus_methods) @@ -207,8 +209,8 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals(1, len(desc.directory_authorities)) self.assertEquals("turtles", desc.directory_authorities[0].nickname) self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_authorities[0].fingerprint) + self.assertEquals("76.73.17.194", desc.directory_authorities[0].hostname) self.assertEquals("76.73.17.194", desc.directory_authorities[0].address) - self.assertEquals("76.73.17.194", desc.directory_authorities[0].ip) self.assertEquals(9030, desc.directory_authorities[0].dir_port) self.assertEquals(9090, desc.directory_authorities[0].or_port) self.assertEquals("Mike Perry <email>", desc.directory_authorities[0].contact) @@ -245,7 +247,7 @@ KG2OUeQUNoCck4nDpsZwFqPlrWCHcHfTV2iDYFV1HQWDTtZz/qf+GtB8NXsq+I1w brADmvReM2BD6p/13h0QURCI5hq7ZYlIKcKrBa0jn1d9cduULl7vgKsRCJDls/ID emBZ6pUxMpBmV0v+PrA3v9w4DlE7GHAq61FF/zju2kpqj6MInbEvI/E+e438sWsL -----END SIGNATURE-----""" - self.assertEquals("3", desc.directory_authorities[0].key_certificate.key_certificate_version) + self.assertEquals(3, desc.directory_authorities[0].key_certificate.version) self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_authorities[0].key_certificate.fingerprint) self.assertEquals(_strptime("2011-11-28 21:51:04"), desc.directory_authorities[0].key_certificate.published) self.assertEquals(_strptime("2012-11-28 21:51:04"), desc.directory_authorities[0].key_certificate.expires) diff --git a/test/mocking.py b/test/mocking.py index ae74166..ada34aa 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -75,6 +75,7 @@ WPi4Fl2qryzTb3QO5r5x7T8OsG2IBUET1bLQzmtbC560SYR49IvVAgMBAAE= """ DOC_SIG = stem.descriptor.networkstatus.DocumentSignature( + None, "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", "BF112F1C6D5543CFD0A32215ACABD4197B5279AD", "-----BEGIN SIGNATURE-----%s-----END SIGNATURE-----" % CRYPTO_BLOB) @@ -671,7 +672,7 @@ def get_network_status_document(attr = None, exclude = (), authorities = None, r if attr is None: attr = {} - # add defaults only found in a vote or consensus + # add defaults only found in a vote, consensus, or microdescriptor if attr.get("vote-status") == "vote": extra_defaults = { @@ -683,6 +684,11 @@ def get_network_status_document(attr = None, exclude = (), authorities = None, r "consensus-method": "9", } + if "microdesc" in attr.get("network-status-version", ""): + extra_defaults.update({ + "directory-signature": "sha256 " + NETWORK_STATUS_DOCUMENT_FOOTER[2][1], + }) + for k, v in extra_defaults.items(): if not (k in attr or (exclude and k in exclude)): attr[k] = v diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py index 246f7aa..7539b63 100644 --- a/test/unit/descriptor/networkstatus/document.py +++ b/test/unit/descriptor/networkstatus/document.py @@ -12,7 +12,7 @@ import stem.version from stem.descriptor import Flag from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, DirectoryAuthority, NetworkStatusDocument, parse_file from stem.descriptor.router_status_entry import RouterStatusEntryV3, RouterStatusEntryMicroV3 -from test.mocking import support_with, get_router_status_entry_v3, get_router_status_entry_micro_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG +from test.mocking import support_with, get_router_status_entry_v3, get_router_status_entry_micro_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG, NETWORK_STATUS_DOCUMENT_FOOTER class TestNetworkStatusDocument(unittest.TestCase): def test_minimal_consensus(self): @@ -633,6 +633,42 @@ class TestNetworkStatusDocument(unittest.TestCase): document = NetworkStatusDocument(content, False) self.assertEquals(expected, document.bandwidth_weights) + def test_microdescriptor_signature(self): + """ + The 'directory-signature' lines for normal and microdescriptor conensuses + differ slightly in their format. + """ + + # including a microdescriptor flavored 'directory-signature' line should work + + document = get_network_status_document({"network-status-version": "3 microdesc"}) + self.assertEqual('sha256', document.signatures[0].method) + + # include a standard 'directory-signature' line in a microdescriptor + # consensus + + content = get_network_status_document({ + "network-status-version": "3 microdesc", + "directory-signature": NETWORK_STATUS_DOCUMENT_FOOTER[2][1], + }, content = True) + + self.assertRaises(ValueError, NetworkStatusDocument, content) + + document = NetworkStatusDocument(content, validate = False) + self.assertEqual([], document.signatures) + + # includes a microdescriptor flavored 'directory-signature' line in a + # normal consensus + + content = get_network_status_document({ + "directory-signature": "sha256 " + NETWORK_STATUS_DOCUMENT_FOOTER[2][1], + }, content = True) + + self.assertRaises(ValueError, NetworkStatusDocument, content) + + document = NetworkStatusDocument(content, validate = False) + self.assertEqual([], document.signatures) + def test_malformed_signature(self): """ Provides malformed or missing content in the 'directory-signature' line.

1 0

[stem/master] Replacing _strptime() helper with normal datetimes
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 9170c0ff411bca266bd90af4bf423c5c0decb44e Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Oct 10 09:35:59 2012 -0700 Replacing _strptime() helper with normal datetimes I kinda liked the idea of the _strptime() helper and briefly moved it to the runner so I could use it in other tests, but on reflection it doesn't really help readability nor is it appreciably shorter. Swapping its usage to normal datetimes for uniformity. --- test/integ/descriptor/networkstatus.py | 29 +++++++++++++---------------- 1 files changed, 13 insertions(+), 16 deletions(-) diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index a4d098e..378e76d 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -14,9 +14,6 @@ import stem.descriptor import stem.descriptor.networkstatus import test.integ.descriptor -def _strptime(string): - return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") - class TestNetworkStatus(unittest.TestCase): def test_cached_consensus(self): """ @@ -106,7 +103,7 @@ class TestNetworkStatus(unittest.TestCase): self.assertEquals("sumkledi", router.nickname) self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router.fingerprint) self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router.digest) - self.assertEquals(_strptime("2012-07-12 04:01:55"), router.published) + self.assertEquals(datetime.datetime(2012, 7, 12, 4, 1, 55), router.published) self.assertEquals("178.218.213.229", router.address) self.assertEquals(80, router.or_port) self.assertEquals(None, router.dir_port) @@ -152,9 +149,9 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals([], document.consensus_methods) self.assertEquals(None, document.published) self.assertEquals(12, document.consensus_method) - self.assertEquals(_strptime("2012-07-12 10:00:00"), document.valid_after) - self.assertEquals(_strptime("2012-07-12 11:00:00"), document.fresh_until) - self.assertEquals(_strptime("2012-07-12 13:00:00"), document.valid_until) + self.assertEquals(datetime.datetime(2012, 7, 12, 10, 0, 0), document.valid_after) + self.assertEquals(datetime.datetime(2012, 7, 12, 11, 0, 0), document.fresh_until) + self.assertEquals(datetime.datetime(2012, 7, 12, 13, 0, 0), document.valid_until) self.assertEquals(300, document.vote_delay) self.assertEquals(300, document.dist_delay) self.assertEquals(expected_versions, document.client_versions) @@ -168,7 +165,7 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals("sumkledi", router.nickname) self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router.fingerprint) self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router.digest) - self.assertEquals(_strptime("2012-07-12 04:01:55"), router.published) + self.assertEquals(datetime.datetime(2012, 7, 12, 4, 1, 55), router.published) self.assertEquals("178.218.213.229", router.address) self.assertEquals(80, router.or_port) self.assertEquals(None, router.dir_port) @@ -207,7 +204,7 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals("sumkledi", router.nickname) self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router.fingerprint) self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router.digest) - self.assertEquals(_strptime("2012-07-11 04:22:53"), router.published) + self.assertEquals(datetime.datetime(2012, 7, 11, 4, 22, 53), router.published) self.assertEquals("178.218.213.229", router.address) self.assertEquals(80, router.or_port) self.assertEquals(None, router.dir_port) @@ -271,11 +268,11 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEquals(False, document.is_consensus) self.assertEquals(True, document.is_vote) self.assertEquals(range(1, 13), document.consensus_methods) - self.assertEquals(_strptime("2012-07-11 23:50:01"), document.published) + self.assertEquals(datetime.datetime(2012, 7, 11, 23, 50, 1), document.published) self.assertEquals(None, document.consensus_method) - self.assertEquals(_strptime("2012-07-12 00:00:00"), document.valid_after) - self.assertEquals(_strptime("2012-07-12 01:00:00"), document.fresh_until) - self.assertEquals(_strptime("2012-07-12 03:00:00"), document.valid_until) + self.assertEquals(datetime.datetime(2012, 7, 12, 0, 0, 0), document.valid_after) + self.assertEquals(datetime.datetime(2012, 7, 12, 1, 0, 0), document.fresh_until) + self.assertEquals(datetime.datetime(2012, 7, 12, 3, 0, 0), document.valid_until) self.assertEquals(300, document.vote_delay) self.assertEquals(300, document.dist_delay) self.assertEquals([], document.client_versions) @@ -288,7 +285,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEquals("sumkledi", router.nickname) self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router.fingerprint) self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router.digest) - self.assertEquals(_strptime("2012-07-11 04:22:53"), router.published) + self.assertEquals(datetime.datetime(2012, 7, 11, 4, 22, 53), router.published) self.assertEquals("178.218.213.229", router.address) self.assertEquals(80, router.or_port) self.assertEquals(None, router.dir_port) @@ -306,8 +303,8 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEquals(3, authority.key_certificate.version) self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", authority.key_certificate.fingerprint) - self.assertEquals(_strptime("2011-11-28 21:51:04"), authority.key_certificate.published) - self.assertEquals(_strptime("2012-11-28 21:51:04"), authority.key_certificate.expires) + self.assertEquals(datetime.datetime(2011, 11, 28, 21, 51, 4), authority.key_certificate.published) + self.assertEquals(datetime.datetime(2012, 11, 28, 21, 51, 4), authority.key_certificate.expires) self.assertEquals(expected_identity_key, authority.key_certificate.identity_key) self.assertEquals(expected_signing_key, authority.key_certificate.signing_key) self.assertEquals(expected_key_crosscert, authority.key_certificate.crosscert)

1 0

[stem/master] Refactoring network status integ tests
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit b0d24bee1d2a00f20aba8953a15df1e006860b33 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Oct 10 09:15:13 2012 -0700 Refactoring network status integ tests General refactoring for the network integ tests, mostly moving things around and renaming to make things more readable. The only functional difference is that a couple tests were using the 'assert' keyword rather than testing assertions. This was a bug - it meant that we'd skip those tests unless running in assert mode and even then it would cause exceptions rather than testing failures. --- test/integ/descriptor/networkstatus.py | 345 +++++++++++++++++--------------- 1 files changed, 188 insertions(+), 157 deletions(-) diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index 31f2c73..a4d098e 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -9,7 +9,6 @@ import resource import datetime import unittest -import stem.exit_policy import stem.version import stem.descriptor import stem.descriptor.networkstatus @@ -27,43 +26,81 @@ class TestNetworkStatus(unittest.TestCase): # lengthy test and uneffected by targets, so only run once if test.runner.only_run_once(self, "test_cached_consensus"): return - descriptor_path = test.runner.get_runner().get_test_dir("cached-consensus") + consensus_path = test.runner.get_runner().get_test_dir("cached-consensus") - if not os.path.exists(descriptor_path): + if not os.path.exists(consensus_path): test.runner.skip(self, "(no cached-consensus)") - - if stem.util.system.is_windows(): - # might hog memory and hang the system - # and we aren't checking for memory usage in windows, so, skip. + return + elif stem.util.system.is_windows(): + # Unable to check memory usage on windows, so can't prevent hanging the + # system if things go bad. + test.runner.skip(self, "(unavailable on windows)") + return count = 0 - with open(descriptor_path) as descriptor_file: - for desc in stem.descriptor.networkstatus.parse_file(descriptor_file): + with open(consensus_path) as descriptor_file: + for router in stem.descriptor.networkstatus.parse_file(descriptor_file): + count += 1 + + # We should have constant memory usage. Fail if we're using over 200 MB. if resource.getrusage(resource.RUSAGE_SELF).ru_maxrss > 200000: - # if we're using > 200 MB we should fail self.fail() - assert desc.nickname # check that the router has a nickname + + # check if there's any unknown flags + # TODO: this should be a 'new capability' check later rather than + # failing the tests + for flag in router.flags: + if not flag in stem.descriptor.Flag: + raise ValueError("Unrecognized flag type: %s, found on relay %s (%s)" % (flag, roouter.fingerprint, router.nickname)) + + # Sanity test that there's at least a hundred relays. If that's not the + # case then this probably isn't a real, complete tor consensus. + + self.assertTrue(count > 100) + + def test_cached_microdesc_consensus(self): + """ + Parses the cached-microdesc-consensus file in our data directory. + """ + + # lengthy test and uneffected by targets, so only run once + if test.runner.only_run_once(self, "test_cached_microdesc_consensus"): return + + consensus_path = test.runner.get_runner().get_test_dir("cached-microdesc-consensus") + + if not os.path.exists(consensus_path): + test.runner.skip(self, "(no cached-microdesc-consensus)") + elif stem.util.system.is_windows(): + test.runner.skip(self, "(unavailable on windows)") + return + + count = 0 + with open(consensus_path) as descriptor_file: + for router in stem.descriptor.networkstatus.parse_file(descriptor_file, is_microdescriptor = True): count += 1 + if resource.getrusage(resource.RUSAGE_SELF).ru_maxrss > 200000: + self.fail() + # check if there's any unknown flags - for flag in desc.flags: + # TODO: this should be a 'new capability' check later rather than + # failing the tests + for flag in router.flags: if not flag in stem.descriptor.Flag: - # TODO: this should be a special 'new capability' check later - # rather than failing the tests - raise ValueError("Unrecognized flag type: %s, found on relay %s (%s)" % (flag, desc.fingerprint, desc.nickname)) + raise ValueError("Unrecognized flag type: %s, found on microdescriptor relay %s (%s)" % (flag, roouter.fingerprint, router.nickname)) - assert count > 100 # sanity check - assuming atleast 100 relays in the Tor network + self.assertTrue(count > 100) def test_metrics_consensus(self): """ Checks if consensus documents from Metrics are parsed properly. """ - descriptor_path = test.integ.descriptor.get_resource("metrics_consensus") + consensus_path = test.integ.descriptor.get_resource("metrics_consensus") - with file(descriptor_path) as descriptor_file: - descriptors = stem.descriptor.parse_file(descriptor_path, descriptor_file) + with open(consensus_path) as descriptor_file: + descriptors = stem.descriptor.parse_file(consensus_path, descriptor_file) router = next(descriptors) self.assertEquals("sumkledi", router.nickname) @@ -79,85 +116,92 @@ class TestNetworkStatus(unittest.TestCase): Checks that consensus documents are properly parsed. """ - descriptor_path = test.integ.descriptor.get_resource("cached-consensus") - - descriptor_file = file(descriptor_path) - desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read(), default_params = False) - router1 = desc.routers[0] - descriptor_file.close() - - self.assertEquals(3, desc.version) - self.assertEquals(None, desc.version_flavor) - self.assertEquals(True, desc.is_consensus) - self.assertEquals(False, desc.is_vote) - self.assertEquals([], desc.consensus_methods) - self.assertEquals(None, desc.published) - self.assertEquals(12, desc.consensus_method) - self.assertEquals(_strptime("2012-07-12 10:00:00"), desc.valid_after) - self.assertEquals(_strptime("2012-07-12 11:00:00"), desc.fresh_until) - self.assertEquals(_strptime("2012-07-12 13:00:00"), desc.valid_until) - self.assertEquals(300, desc.vote_delay) - self.assertEquals(300, desc.dist_delay) - expected_client_versions = [stem.version.Version(version_string) for version_string in ["0.2.2.35", - "0.2.2.36", "0.2.2.37", "0.2.3.10-alpha", "0.2.3.11-alpha", "0.2.3.12-alpha", - "0.2.3.13-alpha", "0.2.3.14-alpha", "0.2.3.15-alpha", "0.2.3.16-alpha", "0.2.3.17-beta", - "0.2.3.18-rc", "0.2.3.19-rc"]] - expected_server_versions = [stem.version.Version(version_string) for version_string in ["0.2.2.35", - "0.2.2.36", "0.2.2.37", "0.2.3.10-alpha", "0.2.3.11-alpha", "0.2.3.12-alpha", - "0.2.3.13-alpha", "0.2.3.14-alpha", "0.2.3.15-alpha", "0.2.3.16-alpha", "0.2.3.17-beta", - "0.2.3.18-rc", "0.2.3.19-rc"]] - self.assertEquals(expected_client_versions, desc.client_versions) - self.assertEquals(expected_server_versions, desc.server_versions) - self.assertEquals(set(desc.known_flags), set(["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", "Named", "Running", "Stable", "Unnamed", "V2Dir", "Valid"])) - expected_params = {"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1} - self.assertEquals(expected_params, desc.params) + # the document's expected client and server versions are the same + expected_versions = [stem.version.Version(v) for v in ( + "0.2.2.35", "0.2.2.36", "0.2.2.37", "0.2.3.10-alpha", + "0.2.3.11-alpha", "0.2.3.12-alpha", "0.2.3.13-alpha", "0.2.3.14-alpha", + "0.2.3.15-alpha", "0.2.3.16-alpha", "0.2.3.17-beta", "0.2.3.18-rc", + "0.2.3.19-rc", + )] - self.assertEquals("sumkledi", router1.nickname) - self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router1.fingerprint) - self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router1.digest) - self.assertEquals(_strptime("2012-07-12 04:01:55"), router1.published) - self.assertEquals("178.218.213.229", router1.address) - self.assertEquals(80, router1.or_port) - self.assertEquals(None, router1.dir_port) - self.assertEquals(set(["Exit", "Fast", "Named", "Running", "Valid"]), set(router1.flags)) + expected_flags = set(["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", + "Named", "Running", "Stable", "Unnamed", "V2Dir", "Valid"]) - self.assertEquals(8, len(desc.directory_authorities)) - self.assertEquals("tor26", desc.directory_authorities[0].nickname) - self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.directory_authorities[0].fingerprint) - self.assertEquals("86.59.21.38", desc.directory_authorities[0].hostname) - self.assertEquals("86.59.21.38", desc.directory_authorities[0].address) - self.assertEquals(80, desc.directory_authorities[0].dir_port) - self.assertEquals(443, desc.directory_authorities[0].or_port) - self.assertEquals("Peter Palfrader", desc.directory_authorities[0].contact) - self.assertEquals(None, desc.directory_authorities[0].legacy_dir_key) - self.assertEquals(None, desc.directory_authorities[0].key_certificate) - self.assertEquals("0B6D1E9A300B895AA2D0B427F92917B6995C3C1C", desc.directory_authorities[0].vote_digest) expected_bandwidth_weights = { - "Wbd": 3335, "Wbe": 0, "Wbg": 3536, "Wbm": 10000, "Wdb": 10000, "Web": 10000, - "Wed": 3329, "Wee": 10000, "Weg": 3329, "Wem": 10000, "Wgb": 10000, "Wgd": 3335, - "Wgg": 6464, "Wgm": 6464, "Wmb": 10000, "Wmd": 3335, "Wme": 0, "Wmg": 3536, "Wmm": 10000 - } - self.assertEquals(expected_bandwidth_weights, desc.bandwidth_weights) + "Wbd": 3335, "Wbe": 0, "Wbg": 3536, "Wbm": 10000, "Wdb": 10000, + "Web": 10000, "Wed": 3329, "Wee": 10000, "Weg": 3329, "Wem": 10000, + "Wgb": 10000, "Wgd": 3335, "Wgg": 6464, "Wgm": 6464, "Wmb": 10000, + "Wmd": 3335, "Wme": 0, "Wmg": 3536, "Wmm": 10000 + } expected_signature = """-----BEGIN SIGNATURE----- HFXB4497LzESysYJ/4jJY83E5vLjhv+igIxD9LU6lf6ftkGeF+lNmIAIEKaMts8H mfWcW0b+jsrXcJoCxV5IrwCDF3u1aC3diwZY6yiG186pwWbOwE41188XI2DeYPwE I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= -----END SIGNATURE-----""" - self.assertEquals(8, len(desc.signatures)) - self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.signatures[0].identity) - self.assertEquals("BF112F1C6D5543CFD0A32215ACABD4197B5279AD", desc.signatures[0].key_digest) - self.assertEquals(expected_signature, desc.signatures[0].signature) + + consensus_path = test.integ.descriptor.get_resource("cached-consensus") + + with open(consensus_path) as descriptor_file: + document = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read(), default_params = False) + + self.assertEquals(3, document.version) + self.assertEquals(None, document.version_flavor) + self.assertEquals(True, document.is_consensus) + self.assertEquals(False, document.is_vote) + self.assertEquals([], document.consensus_methods) + self.assertEquals(None, document.published) + self.assertEquals(12, document.consensus_method) + self.assertEquals(_strptime("2012-07-12 10:00:00"), document.valid_after) + self.assertEquals(_strptime("2012-07-12 11:00:00"), document.fresh_until) + self.assertEquals(_strptime("2012-07-12 13:00:00"), document.valid_until) + self.assertEquals(300, document.vote_delay) + self.assertEquals(300, document.dist_delay) + self.assertEquals(expected_versions, document.client_versions) + self.assertEquals(expected_versions, document.server_versions) + self.assertEquals(expected_flags, set(document.known_flags)) + self.assertEquals({"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1}, document.params) + self.assertEquals(expected_bandwidth_weights, document.bandwidth_weights) + self.assertEquals([], document.get_unrecognized_lines()) + + router = document.routers[0] + self.assertEquals("sumkledi", router.nickname) + self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router.fingerprint) + self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router.digest) + self.assertEquals(_strptime("2012-07-12 04:01:55"), router.published) + self.assertEquals("178.218.213.229", router.address) + self.assertEquals(80, router.or_port) + self.assertEquals(None, router.dir_port) + self.assertEquals(set(["Exit", "Fast", "Named", "Running", "Valid"]), set(router.flags)) + + authority = document.directory_authorities[0] + self.assertEquals(8, len(document.directory_authorities)) + self.assertEquals("tor26", authority.nickname) + self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", authority.fingerprint) + self.assertEquals("86.59.21.38", authority.hostname) + self.assertEquals("86.59.21.38", authority.address) + self.assertEquals(80, authority.dir_port) + self.assertEquals(443, authority.or_port) + self.assertEquals("Peter Palfrader", authority.contact) + self.assertEquals(None, authority.legacy_dir_key) + self.assertEquals(None, authority.key_certificate) + self.assertEquals("0B6D1E9A300B895AA2D0B427F92917B6995C3C1C", authority.vote_digest) + + signature = document.signatures[0] + self.assertEquals(8, len(document.signatures)) + self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", signature.identity) + self.assertEquals("BF112F1C6D5543CFD0A32215ACABD4197B5279AD", signature.key_digest) + self.assertEquals(expected_signature, signature.signature) def test_metrics_vote(self): """ Checks if vote documents from Metrics are parsed properly. """ - descriptor_path = test.integ.descriptor.get_resource("metrics_vote") + vote_path = test.integ.descriptor.get_resource("metrics_vote") - with file(descriptor_path) as descriptor_file: - descriptors = stem.descriptor.parse_file(descriptor_path, descriptor_file) + with open(vote_path) as descriptor_file: + descriptors = stem.descriptor.parse_file(vote_path, descriptor_file) router = next(descriptors) self.assertEquals("sumkledi", router.nickname) @@ -173,48 +217,8 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= Checks that vote documents are properly parsed. """ - descriptor_path = test.integ.descriptor.get_resource("vote") - - descriptor_file = file(descriptor_path) - desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read(), default_params = False) - router1 = desc.routers[0] - descriptor_file.close() - - self.assertEquals(3, desc.version) - self.assertEquals(None, desc.version_flavor) - self.assertEquals(False, desc.is_consensus) - self.assertEquals(True, desc.is_vote) - self.assertEquals(range(1, 13), desc.consensus_methods) - self.assertEquals(_strptime("2012-07-11 23:50:01"), desc.published) - self.assertEquals(None, desc.consensus_method) - self.assertEquals(_strptime("2012-07-12 00:00:00"), desc.valid_after) - self.assertEquals(_strptime("2012-07-12 01:00:00"), desc.fresh_until) - self.assertEquals(_strptime("2012-07-12 03:00:00"), desc.valid_until) - self.assertEquals(300, desc.vote_delay) - self.assertEquals(300, desc.dist_delay) - self.assertEquals([], desc.client_versions) - self.assertEquals([], desc.server_versions) - self.assertEquals(set(desc.known_flags), set(["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", "Running", "Stable", "V2Dir", "Valid"])) - expected_params = {"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1} - self.assertEquals(expected_params, desc.params) - - self.assertEquals("sumkledi", router1.nickname) - self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router1.fingerprint) - self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router1.digest) - self.assertEquals(_strptime("2012-07-11 04:22:53"), router1.published) - self.assertEquals("178.218.213.229", router1.address) - self.assertEquals(80, router1.or_port) - self.assertEquals(None, router1.dir_port) - - self.assertEquals(1, len(desc.directory_authorities)) - self.assertEquals("turtles", desc.directory_authorities[0].nickname) - self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_authorities[0].fingerprint) - self.assertEquals("76.73.17.194", desc.directory_authorities[0].hostname) - self.assertEquals("76.73.17.194", desc.directory_authorities[0].address) - self.assertEquals(9030, desc.directory_authorities[0].dir_port) - self.assertEquals(9090, desc.directory_authorities[0].or_port) - self.assertEquals("Mike Perry <email>", desc.directory_authorities[0].contact) - self.assertEquals(None, desc.directory_authorities[0].legacy_dir_key) + expected_flags = set(["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", + "Running", "Stable", "V2Dir", "Valid"]) expected_identity_key = """-----BEGIN RSA PUBLIC KEY----- MIIBigKCAYEA6uSmsoxj2MiJ3qyZq0qYXlRoG8o82SNqg+22m+t1c7MlQOZWPJYn @@ -227,16 +231,19 @@ Ef22ZHeiVMMKmpV9TtFyiFqvlI6GpQn3mNbsQqF1y3XCA3Q4vlRAkpgJVUSvTxFP 2bNDobOyVCpCM/rwxU1+RCNY5MFJ/+oktUY+0ydvTen3gFdZdgNqCYjKPLfBNm9m RGL7jZunMUNvAgMBAAE= -----END RSA PUBLIC KEY-----""" + expected_signing_key = """-----BEGIN RSA PUBLIC KEY----- MIGJAoGBAJ5itcJRYNEM3Qf1OVWLRkwjqf84oXPc2ZusaJ5zOe7TVvBMra9GNyc0 NM9y6zVkHCAePAjr4KbW/8P1olA6FUE2LV9bozaU1jFf6K8B2OELKs5FUEW+n+ic GM0x6MhngyXonWOcKt5Gj+mAu5lrno9tpNbPkz2Utr/Pi0nsDhWlAgMBAAE= -----END RSA PUBLIC KEY-----""" + expected_key_crosscert = """-----BEGIN ID SIGNATURE----- RHYImGTwg36wmEdAn7qaRg2sAfql7ZCtPIL/O3lU5OIdXXp0tNn/K00Bamqohjk+ Tz4FKsKXGDlbGv67PQcZPOK6NF0GRkNh4pk89prrDO4XwtEn7rkHHdBH6/qQ7IRG GdDZHtZ1a69oFZvPWD3hUaB50xeIe7GoKdKIfdNNJ+8= -----END ID SIGNATURE-----""" + expected_key_certification = """-----BEGIN SIGNATURE----- fasWOGyUZ3iMCYpDfJ+0JcMiTH25sXPWzvlHorEOyOMbaMqRYpZU4GHzt1jLgdl6 AAoR6KdamsLg5VE8xzst48a4UFuzHFlklZ5O8om2rcvDd5DhSnWWYZnYJecqB+bo @@ -247,45 +254,69 @@ KG2OUeQUNoCck4nDpsZwFqPlrWCHcHfTV2iDYFV1HQWDTtZz/qf+GtB8NXsq+I1w brADmvReM2BD6p/13h0QURCI5hq7ZYlIKcKrBa0jn1d9cduULl7vgKsRCJDls/ID emBZ6pUxMpBmV0v+PrA3v9w4DlE7GHAq61FF/zju2kpqj6MInbEvI/E+e438sWsL -----END SIGNATURE-----""" - self.assertEquals(3, desc.directory_authorities[0].key_certificate.version) - self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_authorities[0].key_certificate.fingerprint) - self.assertEquals(_strptime("2011-11-28 21:51:04"), desc.directory_authorities[0].key_certificate.published) - self.assertEquals(_strptime("2012-11-28 21:51:04"), desc.directory_authorities[0].key_certificate.expires) - self.assertEquals(expected_identity_key, desc.directory_authorities[0].key_certificate.identity_key) - self.assertEquals(expected_signing_key, desc.directory_authorities[0].key_certificate.signing_key) - self.assertEquals(expected_key_crosscert, desc.directory_authorities[0].key_certificate.crosscert) - self.assertEquals(expected_key_certification, desc.directory_authorities[0].key_certificate.certification) - self.assertEquals(None, desc.directory_authorities[0].vote_digest) - self.assertEquals({}, desc.bandwidth_weights) expected_signature = """-----BEGIN SIGNATURE----- fskXN84wB3mXfo+yKGSt0AcDaaPuU3NwMR3ROxWgLN0KjAaVi2eV9PkPCsQkcgw3 JZ/1HL9sHyZfo6bwaC6YSM9PNiiY6L7rnGpS7UkHiFI+M96VCMorvjm5YPs3FioJ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= -----END SIGNATURE-----""" - self.assertEquals(1, len(desc.signatures)) - self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.signatures[0].identity) - self.assertEquals("D5C30C15BB3F1DA27669C2D88439939E8F418FCF", desc.signatures[0].key_digest) - self.assertEquals(expected_signature, desc.signatures[0].signature) - - def test_cached_microdesc_consensus(self): - """ - Parses the cached-microdesc-consensus file in our data directory. - """ - - # lengthy test and uneffected by targets, so only run once - if test.runner.only_run_once(self, "test_cached_microdesc_consensus"): return - - descriptor_path = test.runner.get_runner().get_test_dir("cached-microdesc-consensus") - if not os.path.exists(descriptor_path): - test.runner.skip(self, "(no cached-microdesc-consensus)") + vote_path = test.integ.descriptor.get_resource("vote") - count = 0 - with open(descriptor_path) as descriptor_file: - for desc in stem.descriptor.networkstatus.parse_file(descriptor_file, True, is_microdescriptor = True): - assert desc.nickname # check that the router has a nickname - count += 1 - - assert count > 100 # sanity check - assuming atleast 100 relays in the consensus + with open(vote_path) as descriptor_file: + document = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read(), default_params = False) + + self.assertEquals(3, document.version) + self.assertEquals(None, document.version_flavor) + self.assertEquals(False, document.is_consensus) + self.assertEquals(True, document.is_vote) + self.assertEquals(range(1, 13), document.consensus_methods) + self.assertEquals(_strptime("2012-07-11 23:50:01"), document.published) + self.assertEquals(None, document.consensus_method) + self.assertEquals(_strptime("2012-07-12 00:00:00"), document.valid_after) + self.assertEquals(_strptime("2012-07-12 01:00:00"), document.fresh_until) + self.assertEquals(_strptime("2012-07-12 03:00:00"), document.valid_until) + self.assertEquals(300, document.vote_delay) + self.assertEquals(300, document.dist_delay) + self.assertEquals([], document.client_versions) + self.assertEquals([], document.server_versions) + self.assertEquals(expected_flags, set(document.known_flags)) + self.assertEquals({"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1}, document.params) + self.assertEquals({}, document.bandwidth_weights) + + router = document.routers[0] + self.assertEquals("sumkledi", router.nickname) + self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router.fingerprint) + self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router.digest) + self.assertEquals(_strptime("2012-07-11 04:22:53"), router.published) + self.assertEquals("178.218.213.229", router.address) + self.assertEquals(80, router.or_port) + self.assertEquals(None, router.dir_port) + + authority = document.directory_authorities[0] + self.assertEquals(1, len(document.directory_authorities)) + self.assertEquals("turtles", authority.nickname) + self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", authority.fingerprint) + self.assertEquals("76.73.17.194", authority.hostname) + self.assertEquals("76.73.17.194", authority.address) + self.assertEquals(9030, authority.dir_port) + self.assertEquals(9090, authority.or_port) + self.assertEquals("Mike Perry <email>", authority.contact) + self.assertEquals(None, authority.legacy_dir_key) + + self.assertEquals(3, authority.key_certificate.version) + self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", authority.key_certificate.fingerprint) + self.assertEquals(_strptime("2011-11-28 21:51:04"), authority.key_certificate.published) + self.assertEquals(_strptime("2012-11-28 21:51:04"), authority.key_certificate.expires) + self.assertEquals(expected_identity_key, authority.key_certificate.identity_key) + self.assertEquals(expected_signing_key, authority.key_certificate.signing_key) + self.assertEquals(expected_key_crosscert, authority.key_certificate.crosscert) + self.assertEquals(expected_key_certification, authority.key_certificate.certification) + self.assertEquals(None, authority.vote_digest) + + signature = document.signatures[0] + self.assertEquals(1, len(document.signatures)) + self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", signature.identity) + self.assertEquals("D5C30C15BB3F1DA27669C2D88439939E8F418FCF", signature.key_digest) + self.assertEquals(expected_signature, signature.signature)

1 0

[stem/master] Reordering document's integ test attribute checks
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 925e8a3e0232f734c1067e88ac8898e446dee35e Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Oct 10 19:15:29 2012 -0700 Reordering document's integ test attribute checks Changing the order of the attribute assertions to match the classes, and adding a few checks that were missing. --- test/integ/descriptor/networkstatus.py | 46 +++++++++++++++++++------------ 1 files changed, 28 insertions(+), 18 deletions(-) diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index 378e76d..b6526d6 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -49,7 +49,7 @@ class TestNetworkStatus(unittest.TestCase): # failing the tests for flag in router.flags: if not flag in stem.descriptor.Flag: - raise ValueError("Unrecognized flag type: %s, found on relay %s (%s)" % (flag, roouter.fingerprint, router.nickname)) + raise ValueError("Unrecognized flag type: %s, found on relay %s (%s)" % (flag, router.fingerprint, router.nickname)) # Sanity test that there's at least a hundred relays. If that's not the # case then this probably isn't a real, complete tor consensus. @@ -85,7 +85,7 @@ class TestNetworkStatus(unittest.TestCase): # failing the tests for flag in router.flags: if not flag in stem.descriptor.Flag: - raise ValueError("Unrecognized flag type: %s, found on microdescriptor relay %s (%s)" % (flag, roouter.fingerprint, router.nickname)) + raise ValueError("Unrecognized flag type: %s, found on microdescriptor relay %s (%s)" % (flag, router.fingerprint, router.nickname)) self.assertTrue(count > 100) @@ -146,9 +146,7 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals(None, document.version_flavor) self.assertEquals(True, document.is_consensus) self.assertEquals(False, document.is_vote) - self.assertEquals([], document.consensus_methods) - self.assertEquals(None, document.published) - self.assertEquals(12, document.consensus_method) + self.assertEquals(False, document.is_microdescriptor) self.assertEquals(datetime.datetime(2012, 7, 12, 10, 0, 0), document.valid_after) self.assertEquals(datetime.datetime(2012, 7, 12, 11, 0, 0), document.fresh_until) self.assertEquals(datetime.datetime(2012, 7, 12, 13, 0, 0), document.valid_until) @@ -158,7 +156,11 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals(expected_versions, document.server_versions) self.assertEquals(expected_flags, set(document.known_flags)) self.assertEquals({"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1}, document.params) + + self.assertEquals(12, document.consensus_method) self.assertEquals(expected_bandwidth_weights, document.bandwidth_weights) + self.assertEquals([], document.consensus_methods) + self.assertEquals(None, document.published) self.assertEquals([], document.get_unrecognized_lines()) router = document.routers[0] @@ -180,12 +182,13 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals(80, authority.dir_port) self.assertEquals(443, authority.or_port) self.assertEquals("Peter Palfrader", authority.contact) + self.assertEquals("0B6D1E9A300B895AA2D0B427F92917B6995C3C1C", authority.vote_digest) self.assertEquals(None, authority.legacy_dir_key) self.assertEquals(None, authority.key_certificate) - self.assertEquals("0B6D1E9A300B895AA2D0B427F92917B6995C3C1C", authority.vote_digest) signature = document.signatures[0] self.assertEquals(8, len(document.signatures)) + self.assertEquals(None, signature.method) self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", signature.identity) self.assertEquals("BF112F1C6D5543CFD0A32215ACABD4197B5279AD", signature.key_digest) self.assertEquals(expected_signature, signature.signature) @@ -267,9 +270,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEquals(None, document.version_flavor) self.assertEquals(False, document.is_consensus) self.assertEquals(True, document.is_vote) - self.assertEquals(range(1, 13), document.consensus_methods) - self.assertEquals(datetime.datetime(2012, 7, 11, 23, 50, 1), document.published) - self.assertEquals(None, document.consensus_method) + self.assertEquals(False, document.is_microdescriptor) self.assertEquals(datetime.datetime(2012, 7, 12, 0, 0, 0), document.valid_after) self.assertEquals(datetime.datetime(2012, 7, 12, 1, 0, 0), document.fresh_until) self.assertEquals(datetime.datetime(2012, 7, 12, 3, 0, 0), document.valid_until) @@ -279,7 +280,12 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEquals([], document.server_versions) self.assertEquals(expected_flags, set(document.known_flags)) self.assertEquals({"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1}, document.params) + + self.assertEquals(None, document.consensus_method) self.assertEquals({}, document.bandwidth_weights) + self.assertEquals(range(1, 13), document.consensus_methods) + self.assertEquals(datetime.datetime(2012, 7, 11, 23, 50, 1), document.published) + self.assertEquals([], document.get_unrecognized_lines()) router = document.routers[0] self.assertEquals("sumkledi", router.nickname) @@ -299,20 +305,24 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEquals(9030, authority.dir_port) self.assertEquals(9090, authority.or_port) self.assertEquals("Mike Perry <email>", authority.contact) + self.assertEquals(None, authority.vote_digest) self.assertEquals(None, authority.legacy_dir_key) - self.assertEquals(3, authority.key_certificate.version) - self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", authority.key_certificate.fingerprint) - self.assertEquals(datetime.datetime(2011, 11, 28, 21, 51, 4), authority.key_certificate.published) - self.assertEquals(datetime.datetime(2012, 11, 28, 21, 51, 4), authority.key_certificate.expires) - self.assertEquals(expected_identity_key, authority.key_certificate.identity_key) - self.assertEquals(expected_signing_key, authority.key_certificate.signing_key) - self.assertEquals(expected_key_crosscert, authority.key_certificate.crosscert) - self.assertEquals(expected_key_certification, authority.key_certificate.certification) - self.assertEquals(None, authority.vote_digest) + certificate = authority.key_certificate + self.assertEquals(3, certificate.version) + self.assertEquals(None, certificate.address) + self.assertEquals(None, certificate.dir_port) + self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", certificate.fingerprint) + self.assertEquals(expected_identity_key, certificate.identity_key) + self.assertEquals(datetime.datetime(2011, 11, 28, 21, 51, 4), certificate.published) + self.assertEquals(datetime.datetime(2012, 11, 28, 21, 51, 4), certificate.expires) + self.assertEquals(expected_signing_key, certificate.signing_key) + self.assertEquals(expected_key_crosscert, certificate.crosscert) + self.assertEquals(expected_key_certification, certificate.certification) signature = document.signatures[0] self.assertEquals(1, len(document.signatures)) + self.assertEquals(None, signature.method) self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", signature.identity) self.assertEquals("D5C30C15BB3F1DA27669C2D88439939E8F418FCF", signature.key_digest) self.assertEquals(expected_signature, signature.signature)

1 0

[stem/master] Renaming NetworkStatusDocument to NetworkStatusDocumentV3
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 0793113b37afa737862440363983105e820640bd Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Oct 11 07:53:41 2012 -0700 Renaming NetworkStatusDocument to NetworkStatusDocumentV3 Changing the name to make room for v2 documents and future versions. --- run_tests.py | 4 +- stem/descriptor/networkstatus.py | 21 +- test/integ/descriptor/networkstatus.py | 4 +- test/mocking.py | 14 +- test/unit/descriptor/networkstatus/__init__.py | 2 +- test/unit/descriptor/networkstatus/document.py | 799 --------------------- test/unit/descriptor/networkstatus/document_v3.py | 799 +++++++++++++++++++++ 7 files changed, 822 insertions(+), 821 deletions(-) diff --git a/run_tests.py b/run_tests.py index c4db524..2475254 100755 --- a/run_tests.py +++ b/run_tests.py @@ -23,7 +23,7 @@ import test.unit.descriptor.extrainfo_descriptor import test.unit.descriptor.router_status_entry import test.unit.descriptor.networkstatus.directory_authority import test.unit.descriptor.networkstatus.key_certificate -import test.unit.descriptor.networkstatus.document +import test.unit.descriptor.networkstatus.document_v3 import test.unit.response.control_line import test.unit.response.control_message import test.unit.response.getinfo @@ -121,7 +121,7 @@ UNIT_TESTS = ( test.unit.descriptor.router_status_entry.TestRouterStatusEntry, test.unit.descriptor.networkstatus.directory_authority.TestDirectoryAuthority, test.unit.descriptor.networkstatus.key_certificate.TestKeyCertificate, - test.unit.descriptor.networkstatus.document.TestNetworkStatusDocument, + test.unit.descriptor.networkstatus.document_v3.TestNetworkStatusDocument, test.unit.exit_policy.rule.TestExitPolicyRule, test.unit.exit_policy.policy.TestExitPolicy, test.unit.version.TestVersion, diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index ce3a00c..2f6972f 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -17,20 +17,20 @@ Of these, the router status entry section can be quite large (on the order of hundreds of kilobytes). As such we provide a couple methods of reading network status documents... -* :class:`stem.descriptor.networkstatus.NetworkStatusDocument` constructor +* :class:`stem.descriptor.networkstatus.NetworkStatusDocumentV3` constructor If read time and memory aren't a concern then you can simply use the document constructor. Router entries are assigned to its 'routers' attribute... :: - from stem.descriptor.networkstatus import NetworkStatusDocument + from stem.descriptor.networkstatus import NetworkStatusDocumentV3 # Reads the full consensus into memory twice (both for the parsed and # unparsed contents). consensus_file = open('.tor/cached-consensus', 'r') - consensus = NetworkStatusDocument(consensus_file.read()) + consensus = NetworkStatusDocumentV3(consensus_file.read()) consensus_file.close() for router in consensus.routers: @@ -58,7 +58,7 @@ routers. Those routers refer to a 'thin' document, which doesn't have a :: parse_file - parses a network status file, providing an iterator for its routers - NetworkStatusDocument - Version 3 network status document. + NetworkStatusDocumentV3 - Version 3 network status document. DocumentSignature - Signature of a document by a directory authority. DirectoryAuthority - Directory authority as defined in a v3 network status document. """ @@ -153,7 +153,7 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise :param bool is_microdescriptor: True if this is for a microdescriptor consensus, False otherwise - :returns: :class:`stem.descriptor.networkstatus.NetworkStatusDocument` object + :returns: :class:`stem.descriptor.networkstatus.NetworkStatusDocumentV3` object :raises: * ValueError if the contents is malformed and validate is True @@ -183,7 +183,7 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): entry_keyword = ROUTERS_START, start_position = routers_start, end_position = routers_end, - extra_args = (NetworkStatusDocument(document_content, validate),), + extra_args = (NetworkStatusDocumentV3(document_content, validate),), ) for desc in desc_iterator: @@ -229,7 +229,7 @@ def _get_entries(document_file, validate, entry_class, entry_keyword, start_posi desc_content = "".join(stem.descriptor._read_until_keywords(entry_keyword, document_file, ignore_first = True, end_position = end_position)) yield entry_class(desc_content, validate, *extra_args) -class NetworkStatusDocument(stem.descriptor.Descriptor): +class NetworkStatusDocumentV3(stem.descriptor.Descriptor): """ Version 3 network status document. This could be either a vote or consensus. @@ -265,7 +265,8 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): def __init__(self, raw_content, validate = True, default_params = True): """ - Parse a v3 network status document and provide a new NetworkStatusDocument object. + Parse a v3 network status document and provide a new + NetworkStatusDocumentV3 object. :param str raw_content: raw network status document data :param bool validate: True if the document is to be validated, False otherwise @@ -274,7 +275,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :raises: ValueError if the document is invalid """ - super(NetworkStatusDocument, self).__init__(raw_content) + super(NetworkStatusDocumentV3, self).__init__(raw_content) document_file = StringIO(raw_content) self._header = _DocumentHeader(document_file, validate, default_params) @@ -336,7 +337,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): return list(self._unrecognized_lines) def __cmp__(self, other): - if not isinstance(other, NetworkStatusDocument): + if not isinstance(other, NetworkStatusDocumentV3): return 1 return str(self) > str(other) diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index b6526d6..e174f25 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -140,7 +140,7 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= consensus_path = test.integ.descriptor.get_resource("cached-consensus") with open(consensus_path) as descriptor_file: - document = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read(), default_params = False) + document = stem.descriptor.networkstatus.NetworkStatusDocumentV3(descriptor_file.read(), default_params = False) self.assertEquals(3, document.version) self.assertEquals(None, document.version_flavor) @@ -264,7 +264,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= vote_path = test.integ.descriptor.get_resource("vote") with open(vote_path) as descriptor_file: - document = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read(), default_params = False) + document = stem.descriptor.networkstatus.NetworkStatusDocumentV3(descriptor_file.read(), default_params = False) self.assertEquals(3, document.version) self.assertEquals(None, document.version_flavor) diff --git a/test/mocking.py b/test/mocking.py index ada34aa..ab31eef 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -34,9 +34,9 @@ calling :func:`test.mocking.revert_mocking`. get_bridge_extrainfo_descriptor - BridgeExtraInfoDescriptor stem.descriptor.networkstatus - get_directory_authority - DirectoryAuthority - get_key_certificate - KeyCertificate - get_network_status_document - NetworkStatusDocument + get_directory_authority - DirectoryAuthority + get_key_certificate - KeyCertificate + get_network_status_document_v3 - NetworkStatusDocumentV3 stem.descriptor.router_status_entry get_router_status_entry_v2 - RouterStatusEntryV2 @@ -655,10 +655,10 @@ def get_key_certificate(attr = None, exclude = (), content = False): else: return stem.descriptor.networkstatus.KeyCertificate(desc_content, validate = True) -def get_network_status_document(attr = None, exclude = (), authorities = None, routers = None, content = False): +def get_network_status_document_v3(attr = None, exclude = (), authorities = None, routers = None, content = False): """ Provides the descriptor content for... - stem.descriptor.networkstatus.NetworkStatusDocument + stem.descriptor.networkstatus.NetworkStatusDocumentV3 :param dict attr: keyword/value mappings to be included in the descriptor :param list exclude: mandatory keywords to exclude from the descriptor @@ -666,7 +666,7 @@ def get_network_status_document(attr = None, exclude = (), authorities = None, r :param list routers: router status entries to include in the document :param bool content: provides the str content of the descriptor rather than the class if True - :returns: NetworkStatusDocument for the requested descriptor content + :returns: NetworkStatusDocumentV3 for the requested descriptor content """ if attr is None: @@ -709,5 +709,5 @@ def get_network_status_document(attr = None, exclude = (), authorities = None, r if content: return desc_content else: - return stem.descriptor.networkstatus.NetworkStatusDocument(desc_content, validate = True) + return stem.descriptor.networkstatus.NetworkStatusDocumentV3(desc_content, validate = True) diff --git a/test/unit/descriptor/networkstatus/__init__.py b/test/unit/descriptor/networkstatus/__init__.py index a41defe..75c27b7 100644 --- a/test/unit/descriptor/networkstatus/__init__.py +++ b/test/unit/descriptor/networkstatus/__init__.py @@ -2,5 +2,5 @@ Unit tests for stem.descriptor.networkstatus. """ -__all__ = ["directory_authority", "key_certificate", "document"] +__all__ = ["directory_authority", "key_certificate", "document_v3"] diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py deleted file mode 100644 index 7539b63..0000000 --- a/test/unit/descriptor/networkstatus/document.py +++ /dev/null @@ -1,799 +0,0 @@ -""" -Unit tests for the NetworkStatusDocument of stem.descriptor.networkstatus. -""" - -from __future__ import with_statement - -import datetime -import unittest -import StringIO - -import stem.version -from stem.descriptor import Flag -from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, DirectoryAuthority, NetworkStatusDocument, parse_file -from stem.descriptor.router_status_entry import RouterStatusEntryV3, RouterStatusEntryMicroV3 -from test.mocking import support_with, get_router_status_entry_v3, get_router_status_entry_micro_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG, NETWORK_STATUS_DOCUMENT_FOOTER - -class TestNetworkStatusDocument(unittest.TestCase): - def test_minimal_consensus(self): - """ - Parses a minimal network status document. - """ - - document = get_network_status_document() - - expected_known_flags = [Flag.AUTHORITY, Flag.BADEXIT, Flag.EXIT, - Flag.FAST, Flag.GUARD, Flag.HSDIR, Flag.NAMED, Flag.RUNNING, - Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID] - - self.assertEqual((), document.routers) - self.assertEqual(3, document.version) - self.assertEqual(None, document.version_flavor) - self.assertEqual(True, document.is_consensus) - self.assertEqual(False, document.is_vote) - self.assertEqual(False, document.is_microdescriptor) - self.assertEqual(9, document.consensus_method) - self.assertEqual([], document.consensus_methods) - self.assertEqual(None, document.published) - self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.valid_after) - self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.fresh_until) - self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.valid_until) - self.assertEqual(300, document.vote_delay) - self.assertEqual(300, document.dist_delay) - self.assertEqual([], document.client_versions) - self.assertEqual([], document.server_versions) - self.assertEqual(expected_known_flags, document.known_flags) - self.assertEqual(DEFAULT_PARAMS, document.params) - self.assertEqual((), document.directory_authorities) - self.assertEqual({}, document.bandwidth_weights) - self.assertEqual([DOC_SIG], document.signatures) - self.assertEqual([], document.get_unrecognized_lines()) - - def test_minimal_vote(self): - """ - Parses a minimal network status document. - """ - - document = get_network_status_document({"vote-status": "vote"}) - - expected_known_flags = [Flag.AUTHORITY, Flag.BADEXIT, Flag.EXIT, - Flag.FAST, Flag.GUARD, Flag.HSDIR, Flag.NAMED, Flag.RUNNING, - Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID] - - self.assertEqual((), document.routers) - self.assertEqual(3, document.version) - self.assertEqual(False, document.is_consensus) - self.assertEqual(True, document.is_vote) - self.assertEqual(None, document.consensus_method) - self.assertEqual([1, 9], document.consensus_methods) - self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.published) - self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.valid_after) - self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.fresh_until) - self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.valid_until) - self.assertEqual(300, document.vote_delay) - self.assertEqual(300, document.dist_delay) - self.assertEqual([], document.client_versions) - self.assertEqual([], document.server_versions) - self.assertEqual(expected_known_flags, document.known_flags) - self.assertEqual(DEFAULT_PARAMS, document.params) - self.assertEqual((), document.directory_authorities) - self.assertEqual({}, document.bandwidth_weights) - self.assertEqual([DOC_SIG], document.signatures) - self.assertEqual([], document.get_unrecognized_lines()) - - def test_examples(self): - """ - Run something similar to the examples in the header pydocs. - """ - - # makes a consensus with a couple routers, both with the same nickname - - entry1 = get_router_status_entry_v3({'s': "Fast"}) - entry2 = get_router_status_entry_v3({'s': "Valid"}) - content = get_network_status_document(routers = (entry1, entry2), content = True) - - # first example: parsing via the NetworkStatusDocument constructor - - consensus_file = StringIO.StringIO(content) - consensus = NetworkStatusDocument(consensus_file.read()) - consensus_file.close() - - for router in consensus.routers: - self.assertEqual('caerSidi', router.nickname) - - # second example: using parse_file - - with support_with(StringIO.StringIO(content)) as consensus_file: - for router in parse_file(consensus_file): - self.assertEqual('caerSidi', router.nickname) - - def test_parse_file(self): - """ - Try parsing a document via the parse_file() function. - """ - - entry1 = get_router_status_entry_v3({'s': "Fast"}) - entry2 = get_router_status_entry_v3({'s': "Valid"}) - content = get_network_status_document(routers = (entry1, entry2), content = True) - - # the document that the entries refer to should actually be the minimal - # descriptor (ie, without the entries) - - expected_document = get_network_status_document() - - descriptor_file = StringIO.StringIO(content) - entries = list(parse_file(descriptor_file)) - - self.assertEquals(entry1, entries[0]) - self.assertEquals(entry2, entries[1]) - self.assertEquals(expected_document, entries[0].document) - - def test_missing_fields(self): - """ - Excludes mandatory fields from both a vote and consensus document. - """ - - for is_consensus in (True, False): - attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"} - is_vote = not is_consensus - - for entries in (HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS): - for field, in_votes, in_consensus, is_mandatory in entries: - if is_mandatory and ((is_consensus and in_consensus) or (is_vote and in_votes)): - content = get_network_status_document(attr, exclude = (field,), content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - NetworkStatusDocument(content, False) # constructs without validation - - def test_unrecognized_line(self): - """ - Includes unrecognized content in the document. - """ - - document = get_network_status_document({"pepperjack": "is oh so tasty!"}) - self.assertEquals(["pepperjack is oh so tasty!"], document.get_unrecognized_lines()) - - def test_misordered_fields(self): - """ - Rearranges our descriptor fields. - """ - - for is_consensus in (True, False): - attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"} - lines = get_network_status_document(attr, content = True).split("\n") - - for i in xrange(len(lines) - 1): - # once we reach the crypto blob we're done since swapping those won't - # be detected - if lines[i].startswith(CRYPTO_BLOB[1:10]): break - - # swaps this line with the one after it - test_lines = list(lines) - test_lines[i], test_lines[i + 1] = test_lines[i + 1], test_lines[i] - - content = "\n".join(test_lines) - self.assertRaises(ValueError, NetworkStatusDocument, content) - NetworkStatusDocument(content, False) # constructs without validation - - def test_duplicate_fields(self): - """ - Almost all fields can only appear once. Checking that duplicates cause - validation errors. - """ - - for is_consensus in (True, False): - attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"} - lines = get_network_status_document(attr, content = True).split("\n") - - for i in xrange(len(lines)): - # Stop when we hit the 'directory-signature' for a couple reasons... - # - that is the one field that can validly appear multiple times - # - after it is a crypto blob, which won't trigger this kind of - # validation failure - - test_lines = list(lines) - if test_lines[i].startswith("directory-signature "): - break - - # duplicates the line - test_lines.insert(i, test_lines[i]) - - content = "\n".join(test_lines) - self.assertRaises(ValueError, NetworkStatusDocument, content) - NetworkStatusDocument(content, False) # constructs without validation - - def test_version(self): - """ - Parses the network-status-version field, including trying to handle a - different document version with the v3 parser. - """ - - document = get_network_status_document({"network-status-version": "3"}) - self.assertEquals(3, document.version) - self.assertEquals(None, document.version_flavor) - self.assertEquals(False, document.is_microdescriptor) - - document = get_network_status_document({"network-status-version": "3 microdesc"}) - self.assertEquals(3, document.version) - self.assertEquals('microdesc', document.version_flavor) - self.assertEquals(True, document.is_microdescriptor) - - content = get_network_status_document({"network-status-version": "4"}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(4, document.version) - self.assertEquals(None, document.version_flavor) - self.assertEquals(False, document.is_microdescriptor) - - def test_vote_status(self): - """ - Parses the vote-status field. - """ - - document = get_network_status_document({"vote-status": "vote"}) - self.assertEquals(False, document.is_consensus) - self.assertEquals(True, document.is_vote) - - content = get_network_status_document({"vote-status": "consensus"}, content = True) - document = NetworkStatusDocument(content) - self.assertEquals(True, document.is_consensus) - self.assertEquals(False, document.is_vote) - - test_values = ( - "", - " ", - "votee", - ) - - for test_value in test_values: - content = get_network_status_document({"vote-status": test_value}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(True, document.is_consensus) - self.assertEquals(False, document.is_vote) - - def test_consensus_methods(self): - """ - Parses the consensus-methods field. - """ - - document = get_network_status_document({"vote-status": "vote", "consensus-methods": "12 3 1 780"}) - self.assertEquals([12, 3, 1, 780], document.consensus_methods) - - # check that we default to including consensus-method 1 - content = get_network_status_document({"vote-status": "vote"}, ("consensus-methods",), content = True) - document = NetworkStatusDocument(content, False) - self.assertEquals([1], document.consensus_methods) - self.assertEquals(None, document.consensus_method) - - test_values = ( - ("", []), - (" ", []), - ("1 2 3 a 5", [1, 2, 3, 5]), - ("1 2 3 4.0 5", [1, 2, 3, 5]), - ("2 3 4", [2, 3, 4]), # spec says version one must be included - ) - - for test_value, expected_consensus_methods in test_values: - content = get_network_status_document({"vote-status": "vote", "consensus-methods": test_value}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(expected_consensus_methods, document.consensus_methods) - - def test_consensus_method(self): - """ - Parses the consensus-method field. - """ - - document = get_network_status_document({"consensus-method": "12"}) - self.assertEquals(12, document.consensus_method) - - # check that we default to being consensus-method 1 - content = get_network_status_document(exclude = ("consensus-method",), content = True) - document = NetworkStatusDocument(content, False) - self.assertEquals(1, document.consensus_method) - self.assertEquals([], document.consensus_methods) - - test_values = ( - "", - " ", - "a", - "1 2", - "2.0", - ) - - for test_value in test_values: - content = get_network_status_document({"consensus-method": test_value}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(1, document.consensus_method) - - def test_time_fields(self): - """ - Parses invalid published, valid-after, fresh-until, and valid-until fields. - All are simply datetime values. - """ - - expected = datetime.datetime(2012, 9, 2, 22, 0, 0) - test_value = "2012-09-02 22:00:00" - - document = get_network_status_document({ - "vote-status": "vote", - "published": test_value, - "valid-after": test_value, - "fresh-until": test_value, - "valid-until": test_value, - }) - - self.assertEquals(expected, document.published) - self.assertEquals(expected, document.valid_after) - self.assertEquals(expected, document.fresh_until) - self.assertEquals(expected, document.valid_until) - - test_values = ( - "", - " ", - "2012-12-12", - "2012-12-12 01:01:", - "2012-12-12 01:a1:01", - ) - - for field in ('published', 'valid-after', 'fresh-until', 'valid-until'): - attr = field.replace('-', '_') - - for test_value in test_values: - content = get_network_status_document({"vote-status": "vote", field: test_value}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(None, getattr(document, attr)) - - def test_voting_delay(self): - """ - Parses the voting-delay field. - """ - - document = get_network_status_document({"voting-delay": "12 345"}) - self.assertEquals(12, document.vote_delay) - self.assertEquals(345, document.dist_delay) - - test_values = ( - "", - " ", - "1 a", - "1\t2", - "1 2.0", - ) - - for test_value in test_values: - content = get_network_status_document({"voting-delay": test_value}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(None, document.vote_delay) - self.assertEquals(None, document.dist_delay) - - def test_version_lists(self): - """ - Parses client-versions and server-versions fields. Both are comma separated - lists of tor versions. - """ - - expected = [stem.version.Version("1.2.3.4"), stem.version.Version("56.789.12.34-alpha")] - test_value = "1.2.3.4,56.789.12.34-alpha" - - document = get_network_status_document({"client-versions": test_value, "server-versions": test_value}) - self.assertEquals(expected, document.client_versions) - self.assertEquals(expected, document.server_versions) - - test_values = ( - ("", []), - (" ", []), - ("1.2.3.4,", [stem.version.Version("1.2.3.4")]), - ("1.2.3.4,1.2.3.a", [stem.version.Version("1.2.3.4")]), - ) - - for field in ('client-versions', 'server-versions'): - attr = field.replace('-', '_') - - for test_value, expected_value in test_values: - content = get_network_status_document({field: test_value}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(expected_value, getattr(document, attr)) - - def test_known_flags(self): - """ - Parses some known-flag entries. Just exercising the field, there's not much - to test here. - """ - - test_values = ( - ("", []), - (" ", []), - ("BadExit", [Flag.BADEXIT]), - ("BadExit ", [Flag.BADEXIT]), - ("BadExit ", [Flag.BADEXIT]), - ("BadExit Fast", [Flag.BADEXIT, Flag.FAST]), - ("BadExit Unrecognized Fast", [Flag.BADEXIT, "Unrecognized", Flag.FAST]), - ) - - for test_value, expected_value in test_values: - document = get_network_status_document({"known-flags": test_value}) - self.assertEquals(expected_value, document.known_flags) - - def test_params(self): - """ - General testing for the 'params' line, exercising the happy cases. - """ - - document = get_network_status_document({"params": "CircuitPriorityHalflifeMsec=30000 bwauthpid=1 unrecognized=-122"}) - self.assertEquals(30000, document.params["CircuitPriorityHalflifeMsec"]) - self.assertEquals(1, document.params["bwauthpid"]) - self.assertEquals(-122, document.params["unrecognized"]) - - # empty params line - content = get_network_status_document({"params": ""}, content = True) - document = NetworkStatusDocument(content, default_params = True) - self.assertEquals(DEFAULT_PARAMS, document.params) - - content = get_network_status_document({"params": ""}, content = True) - document = NetworkStatusDocument(content, default_params = False) - self.assertEquals({}, document.params) - - def test_params_malformed(self): - """ - Parses a 'params' line with malformed content. - """ - - test_values = ( - "foo=", - "foo=abc", - "foo=+123", - "foo=12\tbar=12", - ) - - for test_value in test_values: - content = get_network_status_document({"params": test_value}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(DEFAULT_PARAMS, document.params) - - def test_params_range(self): - """ - Check both the furthest valid 'params' values and values that are out of - bounds. - """ - - test_values = ( - ("foo=2147483648", {"foo": 2147483648}, False), - ("foo=-2147483649", {"foo": -2147483649}, False), - ("foo=2147483647", {"foo": 2147483647}, True), - ("foo=-2147483648", {"foo": -2147483648}, True), - - # param with special range constraints - ("circwindow=99", {"circwindow": 99}, False), - ("circwindow=1001", {"circwindow": 1001}, False), - ("circwindow=500", {"circwindow": 500}, True), - - # param that relies on another param for its constraints - ("cbtclosequantile=79 cbtquantile=80", {"cbtclosequantile": 79, "cbtquantile": 80}, False), - ("cbtclosequantile=80 cbtquantile=80", {"cbtclosequantile": 80, "cbtquantile": 80}, True), - ) - - for test_value, expected_value, is_ok in test_values: - content = get_network_status_document({"params": test_value}, content = True) - - if is_ok: - document = NetworkStatusDocument(content, default_params = False) - else: - self.assertRaises(ValueError, NetworkStatusDocument, content) - document = NetworkStatusDocument(content, False, default_params = False) - - self.assertEquals(expected_value, document.params) - - def test_params_misordered(self): - """ - Check that the 'params' line is rejected if out of order. - """ - - content = get_network_status_document({"params": "unrecognized=-122 bwauthpid=1"}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False, default_params = False) - self.assertEquals({"unrecognized": -122, "bwauthpid": 1}, document.params) - - def test_footer_consensus_method_requirement(self): - """ - Check that validation will notice if a footer appears before it was - introduced. - """ - - content = get_network_status_document({"consensus-method": "8"}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEqual([DOC_SIG], document.signatures) - self.assertEqual([], document.get_unrecognized_lines()) - - # excludes a footer from a version that shouldn't have it - - document = get_network_status_document({"consensus-method": "8"}, ("directory-footer", "directory-signature")) - self.assertEqual([], document.signatures) - self.assertEqual([], document.get_unrecognized_lines()) - - def test_footer_with_value(self): - """ - Tries to parse a descriptor with content on the 'directory-footer' line. - """ - - content = get_network_status_document({"directory-footer": "blarg"}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEqual([DOC_SIG], document.signatures) - self.assertEqual([], document.get_unrecognized_lines()) - - def test_bandwidth_wights_ok(self): - """ - Parses a properly formed 'bandwidth-wights' line. Negative bandwidth - weights might or might not be valid. The spec doesn't say, so making sure - that we accept them. - """ - - weight_entries, expected = [], {} - - for i in xrange(len(BANDWIDTH_WEIGHT_ENTRIES)): - key, value = BANDWIDTH_WEIGHT_ENTRIES[i], i - 5 - - weight_entries.append("%s=%i" % (key, value)) - expected[key] = value - - document = get_network_status_document({"bandwidth-weights": " ".join(weight_entries)}) - self.assertEquals(expected, document.bandwidth_weights) - - def test_bandwidth_wights_malformed(self): - """ - Provides malformed content in the 'bandwidth-wights' line. - """ - - test_values = ( - "Wbe", - "Wbe=", - "Wbe=a", - "Wbe=+7", - ) - - base_weight_entry = " ".join(["%s=5" % e for e in BANDWIDTH_WEIGHT_ENTRIES]) - expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES if e != "Wbe"]) - - for test_value in test_values: - weight_entry = base_weight_entry.replace("Wbe=5", test_value) - content = get_network_status_document({"bandwidth-weights": weight_entry}, content = True) - - self.assertRaises(ValueError, NetworkStatusDocument, content) - document = NetworkStatusDocument(content, False) - self.assertEquals(expected, document.bandwidth_weights) - - def test_bandwidth_wights_misordered(self): - """ - Check that the 'bandwidth-wights' line is rejected if out of order. - """ - - weight_entry = " ".join(["%s=5" % e for e in reversed(BANDWIDTH_WEIGHT_ENTRIES)]) - expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES]) - - content = get_network_status_document({"bandwidth-weights": weight_entry}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(expected, document.bandwidth_weights) - - def test_bandwidth_wights_in_vote(self): - """ - Tries adding a 'bandwidth-wights' line to a vote. - """ - - weight_entry = " ".join(["%s=5" % e for e in BANDWIDTH_WEIGHT_ENTRIES]) - expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES]) - - content = get_network_status_document({"vote-status": "vote", "bandwidth-weights": weight_entry}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(expected, document.bandwidth_weights) - - def test_bandwidth_wights_omissions(self): - """ - Leaves entries out of the 'bandwidth-wights' line. - """ - - # try parsing an empty value - - content = get_network_status_document({"bandwidth-weights": ""}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals({}, document.bandwidth_weights) - - # drop individual values - - for missing_entry in BANDWIDTH_WEIGHT_ENTRIES: - weight_entries = ["%s=5" % e for e in BANDWIDTH_WEIGHT_ENTRIES if e != missing_entry] - expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES if e != missing_entry]) - - content = get_network_status_document({"bandwidth-weights": " ".join(weight_entries)}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, False) - self.assertEquals(expected, document.bandwidth_weights) - - def test_microdescriptor_signature(self): - """ - The 'directory-signature' lines for normal and microdescriptor conensuses - differ slightly in their format. - """ - - # including a microdescriptor flavored 'directory-signature' line should work - - document = get_network_status_document({"network-status-version": "3 microdesc"}) - self.assertEqual('sha256', document.signatures[0].method) - - # include a standard 'directory-signature' line in a microdescriptor - # consensus - - content = get_network_status_document({ - "network-status-version": "3 microdesc", - "directory-signature": NETWORK_STATUS_DOCUMENT_FOOTER[2][1], - }, content = True) - - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, validate = False) - self.assertEqual([], document.signatures) - - # includes a microdescriptor flavored 'directory-signature' line in a - # normal consensus - - content = get_network_status_document({ - "directory-signature": "sha256 " + NETWORK_STATUS_DOCUMENT_FOOTER[2][1], - }, content = True) - - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, validate = False) - self.assertEqual([], document.signatures) - - def test_malformed_signature(self): - """ - Provides malformed or missing content in the 'directory-signature' line. - """ - - test_values = ( - "", - "\n", - "blarg", - ) - - for test_value in test_values: - for test_attr in xrange(3): - attrs = [DOC_SIG.identity, DOC_SIG.key_digest, DOC_SIG.signature] - attrs[test_attr] = test_value - - content = get_network_status_document({"directory-signature": "%s %s\n%s" % tuple(attrs)}, content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - NetworkStatusDocument(content, False) # checks that it's still parseable without validation - - def test_with_router_status_entries(self): - """ - Includes router status entries within the document. This isn't to test the - RouterStatusEntry parsing but rather the inclusion of it within the - document. - """ - - entry1 = get_router_status_entry_v3({'s': "Fast"}) - entry2 = get_router_status_entry_v3({'s': "Valid"}) - document = get_network_status_document(routers = (entry1, entry2)) - - self.assertEquals((entry1, entry2), document.routers) - - # try with an invalid RouterStatusEntry - - entry3 = RouterStatusEntryV3(get_router_status_entry_v3({'r': "ugabuga"}, content = True), False) - content = get_network_status_document(routers = (entry3,), content = True) - - self.assertRaises(ValueError, NetworkStatusDocument, content) - document = NetworkStatusDocument(content, False) - self.assertEquals((entry3,), document.routers) - - # try including with a microdescriptor consensus - - content = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry1, entry2), content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - expected_routers = ( - RouterStatusEntryMicroV3(str(entry1), False), - RouterStatusEntryMicroV3(str(entry2), False), - ) - - document = NetworkStatusDocument(content, False) - self.assertEquals(expected_routers, document.routers) - - def test_with_microdescriptor_router_status_entries(self): - """ - Includes microdescriptor flavored router status entries within the - document. - """ - - entry1 = get_router_status_entry_micro_v3({'s': "Fast"}) - entry2 = get_router_status_entry_micro_v3({'s': "Valid"}) - document = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry1, entry2)) - - self.assertEquals((entry1, entry2), document.routers) - - # try with an invalid RouterStatusEntry - - entry3 = RouterStatusEntryMicroV3(get_router_status_entry_micro_v3({'r': "ugabuga"}, content = True), False) - content = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry3,), content = True) - - self.assertRaises(ValueError, NetworkStatusDocument, content) - document = NetworkStatusDocument(content, False) - self.assertEquals((entry3,), document.routers) - - # try including microdescriptor entries in a normal consensus - - content = get_network_status_document(routers = (entry1, entry2), content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - expected_routers = ( - RouterStatusEntryV3(str(entry1), False), - RouterStatusEntryV3(str(entry2), False), - ) - - document = NetworkStatusDocument(content, False) - self.assertEquals(expected_routers, document.routers) - - def test_with_directory_authorities(self): - """ - Includes a couple directory authorities in the document. - """ - - for is_document_vote in (False, True): - for is_authorities_vote in (False, True): - authority1 = get_directory_authority({'contact': 'doctor jekyll'}, is_vote = is_authorities_vote) - authority2 = get_directory_authority({'contact': 'mister hyde'}, is_vote = is_authorities_vote) - - vote_status = "vote" if is_document_vote else "consensus" - content = get_network_status_document({"vote-status": vote_status}, authorities = (authority1, authority2), content = True) - - if is_document_vote == is_authorities_vote: - document = NetworkStatusDocument(content) - self.assertEquals((authority1, authority2), document.directory_authorities) - else: - # authority votes in a consensus or consensus authorities in a vote - self.assertRaises(ValueError, NetworkStatusDocument, content) - document = NetworkStatusDocument(content, validate = False) - self.assertEquals((authority1, authority2), document.directory_authorities) - - def test_authority_validation_flag_propagation(self): - """ - Includes invalid certificate content in an authority entry. This is testing - that the 'validate' flag propagages from the document to authority, and - authority to certificate classes. - """ - - # make the dir-key-published field of the certiciate be malformed - authority_content = get_directory_authority(is_vote = True, content = True) - authority_content = authority_content.replace("dir-key-published 2011", "dir-key-published 2011a") - - content = get_network_status_document({"vote-status": "vote"}, authorities = (authority_content,), content = True) - self.assertRaises(ValueError, NetworkStatusDocument, content) - - document = NetworkStatusDocument(content, validate = False) - self.assertEquals((DirectoryAuthority(authority_content, False, True),), document.directory_authorities) - diff --git a/test/unit/descriptor/networkstatus/document_v3.py b/test/unit/descriptor/networkstatus/document_v3.py new file mode 100644 index 0000000..47ce4a9 --- /dev/null +++ b/test/unit/descriptor/networkstatus/document_v3.py @@ -0,0 +1,799 @@ +""" +Unit tests for the NetworkStatusDocumentV3 of stem.descriptor.networkstatus. +""" + +from __future__ import with_statement + +import datetime +import unittest +import StringIO + +import stem.version +from stem.descriptor import Flag +from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, DirectoryAuthority, NetworkStatusDocumentV3, parse_file +from stem.descriptor.router_status_entry import RouterStatusEntryV3, RouterStatusEntryMicroV3 +from test.mocking import support_with, get_router_status_entry_v3, get_router_status_entry_micro_v3, get_directory_authority, get_network_status_document_v3, CRYPTO_BLOB, DOC_SIG, NETWORK_STATUS_DOCUMENT_FOOTER + +class TestNetworkStatusDocument(unittest.TestCase): + def test_minimal_consensus(self): + """ + Parses a minimal network status document. + """ + + document = get_network_status_document_v3() + + expected_known_flags = [Flag.AUTHORITY, Flag.BADEXIT, Flag.EXIT, + Flag.FAST, Flag.GUARD, Flag.HSDIR, Flag.NAMED, Flag.RUNNING, + Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID] + + self.assertEqual((), document.routers) + self.assertEqual(3, document.version) + self.assertEqual(None, document.version_flavor) + self.assertEqual(True, document.is_consensus) + self.assertEqual(False, document.is_vote) + self.assertEqual(False, document.is_microdescriptor) + self.assertEqual(9, document.consensus_method) + self.assertEqual([], document.consensus_methods) + self.assertEqual(None, document.published) + self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.valid_after) + self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.fresh_until) + self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.valid_until) + self.assertEqual(300, document.vote_delay) + self.assertEqual(300, document.dist_delay) + self.assertEqual([], document.client_versions) + self.assertEqual([], document.server_versions) + self.assertEqual(expected_known_flags, document.known_flags) + self.assertEqual(DEFAULT_PARAMS, document.params) + self.assertEqual((), document.directory_authorities) + self.assertEqual({}, document.bandwidth_weights) + self.assertEqual([DOC_SIG], document.signatures) + self.assertEqual([], document.get_unrecognized_lines()) + + def test_minimal_vote(self): + """ + Parses a minimal network status document. + """ + + document = get_network_status_document_v3({"vote-status": "vote"}) + + expected_known_flags = [Flag.AUTHORITY, Flag.BADEXIT, Flag.EXIT, + Flag.FAST, Flag.GUARD, Flag.HSDIR, Flag.NAMED, Flag.RUNNING, + Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID] + + self.assertEqual((), document.routers) + self.assertEqual(3, document.version) + self.assertEqual(False, document.is_consensus) + self.assertEqual(True, document.is_vote) + self.assertEqual(None, document.consensus_method) + self.assertEqual([1, 9], document.consensus_methods) + self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.published) + self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.valid_after) + self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.fresh_until) + self.assertEqual(datetime.datetime(2012, 9, 2, 22, 0, 0), document.valid_until) + self.assertEqual(300, document.vote_delay) + self.assertEqual(300, document.dist_delay) + self.assertEqual([], document.client_versions) + self.assertEqual([], document.server_versions) + self.assertEqual(expected_known_flags, document.known_flags) + self.assertEqual(DEFAULT_PARAMS, document.params) + self.assertEqual((), document.directory_authorities) + self.assertEqual({}, document.bandwidth_weights) + self.assertEqual([DOC_SIG], document.signatures) + self.assertEqual([], document.get_unrecognized_lines()) + + def test_examples(self): + """ + Run something similar to the examples in the header pydocs. + """ + + # makes a consensus with a couple routers, both with the same nickname + + entry1 = get_router_status_entry_v3({'s': "Fast"}) + entry2 = get_router_status_entry_v3({'s': "Valid"}) + content = get_network_status_document_v3(routers = (entry1, entry2), content = True) + + # first example: parsing via the NetworkStatusDocumentV3 constructor + + consensus_file = StringIO.StringIO(content) + consensus = NetworkStatusDocumentV3(consensus_file.read()) + consensus_file.close() + + for router in consensus.routers: + self.assertEqual('caerSidi', router.nickname) + + # second example: using parse_file + + with support_with(StringIO.StringIO(content)) as consensus_file: + for router in parse_file(consensus_file): + self.assertEqual('caerSidi', router.nickname) + + def test_parse_file(self): + """ + Try parsing a document via the parse_file() function. + """ + + entry1 = get_router_status_entry_v3({'s': "Fast"}) + entry2 = get_router_status_entry_v3({'s': "Valid"}) + content = get_network_status_document_v3(routers = (entry1, entry2), content = True) + + # the document that the entries refer to should actually be the minimal + # descriptor (ie, without the entries) + + expected_document = get_network_status_document_v3() + + descriptor_file = StringIO.StringIO(content) + entries = list(parse_file(descriptor_file)) + + self.assertEquals(entry1, entries[0]) + self.assertEquals(entry2, entries[1]) + self.assertEquals(expected_document, entries[0].document) + + def test_missing_fields(self): + """ + Excludes mandatory fields from both a vote and consensus document. + """ + + for is_consensus in (True, False): + attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"} + is_vote = not is_consensus + + for entries in (HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS): + for field, in_votes, in_consensus, is_mandatory in entries: + if is_mandatory and ((is_consensus and in_consensus) or (is_vote and in_votes)): + content = get_network_status_document_v3(attr, exclude = (field,), content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + NetworkStatusDocumentV3(content, False) # constructs without validation + + def test_unrecognized_line(self): + """ + Includes unrecognized content in the document. + """ + + document = get_network_status_document_v3({"pepperjack": "is oh so tasty!"}) + self.assertEquals(["pepperjack is oh so tasty!"], document.get_unrecognized_lines()) + + def test_misordered_fields(self): + """ + Rearranges our descriptor fields. + """ + + for is_consensus in (True, False): + attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"} + lines = get_network_status_document_v3(attr, content = True).split("\n") + + for i in xrange(len(lines) - 1): + # once we reach the crypto blob we're done since swapping those won't + # be detected + if lines[i].startswith(CRYPTO_BLOB[1:10]): break + + # swaps this line with the one after it + test_lines = list(lines) + test_lines[i], test_lines[i + 1] = test_lines[i + 1], test_lines[i] + + content = "\n".join(test_lines) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + NetworkStatusDocumentV3(content, False) # constructs without validation + + def test_duplicate_fields(self): + """ + Almost all fields can only appear once. Checking that duplicates cause + validation errors. + """ + + for is_consensus in (True, False): + attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"} + lines = get_network_status_document_v3(attr, content = True).split("\n") + + for i in xrange(len(lines)): + # Stop when we hit the 'directory-signature' for a couple reasons... + # - that is the one field that can validly appear multiple times + # - after it is a crypto blob, which won't trigger this kind of + # validation failure + + test_lines = list(lines) + if test_lines[i].startswith("directory-signature "): + break + + # duplicates the line + test_lines.insert(i, test_lines[i]) + + content = "\n".join(test_lines) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + NetworkStatusDocumentV3(content, False) # constructs without validation + + def test_version(self): + """ + Parses the network-status-version field, including trying to handle a + different document version with the v3 parser. + """ + + document = get_network_status_document_v3({"network-status-version": "3"}) + self.assertEquals(3, document.version) + self.assertEquals(None, document.version_flavor) + self.assertEquals(False, document.is_microdescriptor) + + document = get_network_status_document_v3({"network-status-version": "3 microdesc"}) + self.assertEquals(3, document.version) + self.assertEquals('microdesc', document.version_flavor) + self.assertEquals(True, document.is_microdescriptor) + + content = get_network_status_document_v3({"network-status-version": "4"}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(4, document.version) + self.assertEquals(None, document.version_flavor) + self.assertEquals(False, document.is_microdescriptor) + + def test_vote_status(self): + """ + Parses the vote-status field. + """ + + document = get_network_status_document_v3({"vote-status": "vote"}) + self.assertEquals(False, document.is_consensus) + self.assertEquals(True, document.is_vote) + + content = get_network_status_document_v3({"vote-status": "consensus"}, content = True) + document = NetworkStatusDocumentV3(content) + self.assertEquals(True, document.is_consensus) + self.assertEquals(False, document.is_vote) + + test_values = ( + "", + " ", + "votee", + ) + + for test_value in test_values: + content = get_network_status_document_v3({"vote-status": test_value}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(True, document.is_consensus) + self.assertEquals(False, document.is_vote) + + def test_consensus_methods(self): + """ + Parses the consensus-methods field. + """ + + document = get_network_status_document_v3({"vote-status": "vote", "consensus-methods": "12 3 1 780"}) + self.assertEquals([12, 3, 1, 780], document.consensus_methods) + + # check that we default to including consensus-method 1 + content = get_network_status_document_v3({"vote-status": "vote"}, ("consensus-methods",), content = True) + document = NetworkStatusDocumentV3(content, False) + self.assertEquals([1], document.consensus_methods) + self.assertEquals(None, document.consensus_method) + + test_values = ( + ("", []), + (" ", []), + ("1 2 3 a 5", [1, 2, 3, 5]), + ("1 2 3 4.0 5", [1, 2, 3, 5]), + ("2 3 4", [2, 3, 4]), # spec says version one must be included + ) + + for test_value, expected_consensus_methods in test_values: + content = get_network_status_document_v3({"vote-status": "vote", "consensus-methods": test_value}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(expected_consensus_methods, document.consensus_methods) + + def test_consensus_method(self): + """ + Parses the consensus-method field. + """ + + document = get_network_status_document_v3({"consensus-method": "12"}) + self.assertEquals(12, document.consensus_method) + + # check that we default to being consensus-method 1 + content = get_network_status_document_v3(exclude = ("consensus-method",), content = True) + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(1, document.consensus_method) + self.assertEquals([], document.consensus_methods) + + test_values = ( + "", + " ", + "a", + "1 2", + "2.0", + ) + + for test_value in test_values: + content = get_network_status_document_v3({"consensus-method": test_value}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(1, document.consensus_method) + + def test_time_fields(self): + """ + Parses invalid published, valid-after, fresh-until, and valid-until fields. + All are simply datetime values. + """ + + expected = datetime.datetime(2012, 9, 2, 22, 0, 0) + test_value = "2012-09-02 22:00:00" + + document = get_network_status_document_v3({ + "vote-status": "vote", + "published": test_value, + "valid-after": test_value, + "fresh-until": test_value, + "valid-until": test_value, + }) + + self.assertEquals(expected, document.published) + self.assertEquals(expected, document.valid_after) + self.assertEquals(expected, document.fresh_until) + self.assertEquals(expected, document.valid_until) + + test_values = ( + "", + " ", + "2012-12-12", + "2012-12-12 01:01:", + "2012-12-12 01:a1:01", + ) + + for field in ('published', 'valid-after', 'fresh-until', 'valid-until'): + attr = field.replace('-', '_') + + for test_value in test_values: + content = get_network_status_document_v3({"vote-status": "vote", field: test_value}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(None, getattr(document, attr)) + + def test_voting_delay(self): + """ + Parses the voting-delay field. + """ + + document = get_network_status_document_v3({"voting-delay": "12 345"}) + self.assertEquals(12, document.vote_delay) + self.assertEquals(345, document.dist_delay) + + test_values = ( + "", + " ", + "1 a", + "1\t2", + "1 2.0", + ) + + for test_value in test_values: + content = get_network_status_document_v3({"voting-delay": test_value}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(None, document.vote_delay) + self.assertEquals(None, document.dist_delay) + + def test_version_lists(self): + """ + Parses client-versions and server-versions fields. Both are comma separated + lists of tor versions. + """ + + expected = [stem.version.Version("1.2.3.4"), stem.version.Version("56.789.12.34-alpha")] + test_value = "1.2.3.4,56.789.12.34-alpha" + + document = get_network_status_document_v3({"client-versions": test_value, "server-versions": test_value}) + self.assertEquals(expected, document.client_versions) + self.assertEquals(expected, document.server_versions) + + test_values = ( + ("", []), + (" ", []), + ("1.2.3.4,", [stem.version.Version("1.2.3.4")]), + ("1.2.3.4,1.2.3.a", [stem.version.Version("1.2.3.4")]), + ) + + for field in ('client-versions', 'server-versions'): + attr = field.replace('-', '_') + + for test_value, expected_value in test_values: + content = get_network_status_document_v3({field: test_value}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(expected_value, getattr(document, attr)) + + def test_known_flags(self): + """ + Parses some known-flag entries. Just exercising the field, there's not much + to test here. + """ + + test_values = ( + ("", []), + (" ", []), + ("BadExit", [Flag.BADEXIT]), + ("BadExit ", [Flag.BADEXIT]), + ("BadExit ", [Flag.BADEXIT]), + ("BadExit Fast", [Flag.BADEXIT, Flag.FAST]), + ("BadExit Unrecognized Fast", [Flag.BADEXIT, "Unrecognized", Flag.FAST]), + ) + + for test_value, expected_value in test_values: + document = get_network_status_document_v3({"known-flags": test_value}) + self.assertEquals(expected_value, document.known_flags) + + def test_params(self): + """ + General testing for the 'params' line, exercising the happy cases. + """ + + document = get_network_status_document_v3({"params": "CircuitPriorityHalflifeMsec=30000 bwauthpid=1 unrecognized=-122"}) + self.assertEquals(30000, document.params["CircuitPriorityHalflifeMsec"]) + self.assertEquals(1, document.params["bwauthpid"]) + self.assertEquals(-122, document.params["unrecognized"]) + + # empty params line + content = get_network_status_document_v3({"params": ""}, content = True) + document = NetworkStatusDocumentV3(content, default_params = True) + self.assertEquals(DEFAULT_PARAMS, document.params) + + content = get_network_status_document_v3({"params": ""}, content = True) + document = NetworkStatusDocumentV3(content, default_params = False) + self.assertEquals({}, document.params) + + def test_params_malformed(self): + """ + Parses a 'params' line with malformed content. + """ + + test_values = ( + "foo=", + "foo=abc", + "foo=+123", + "foo=12\tbar=12", + ) + + for test_value in test_values: + content = get_network_status_document_v3({"params": test_value}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(DEFAULT_PARAMS, document.params) + + def test_params_range(self): + """ + Check both the furthest valid 'params' values and values that are out of + bounds. + """ + + test_values = ( + ("foo=2147483648", {"foo": 2147483648}, False), + ("foo=-2147483649", {"foo": -2147483649}, False), + ("foo=2147483647", {"foo": 2147483647}, True), + ("foo=-2147483648", {"foo": -2147483648}, True), + + # param with special range constraints + ("circwindow=99", {"circwindow": 99}, False), + ("circwindow=1001", {"circwindow": 1001}, False), + ("circwindow=500", {"circwindow": 500}, True), + + # param that relies on another param for its constraints + ("cbtclosequantile=79 cbtquantile=80", {"cbtclosequantile": 79, "cbtquantile": 80}, False), + ("cbtclosequantile=80 cbtquantile=80", {"cbtclosequantile": 80, "cbtquantile": 80}, True), + ) + + for test_value, expected_value, is_ok in test_values: + content = get_network_status_document_v3({"params": test_value}, content = True) + + if is_ok: + document = NetworkStatusDocumentV3(content, default_params = False) + else: + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + document = NetworkStatusDocumentV3(content, False, default_params = False) + + self.assertEquals(expected_value, document.params) + + def test_params_misordered(self): + """ + Check that the 'params' line is rejected if out of order. + """ + + content = get_network_status_document_v3({"params": "unrecognized=-122 bwauthpid=1"}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False, default_params = False) + self.assertEquals({"unrecognized": -122, "bwauthpid": 1}, document.params) + + def test_footer_consensus_method_requirement(self): + """ + Check that validation will notice if a footer appears before it was + introduced. + """ + + content = get_network_status_document_v3({"consensus-method": "8"}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEqual([DOC_SIG], document.signatures) + self.assertEqual([], document.get_unrecognized_lines()) + + # excludes a footer from a version that shouldn't have it + + document = get_network_status_document_v3({"consensus-method": "8"}, ("directory-footer", "directory-signature")) + self.assertEqual([], document.signatures) + self.assertEqual([], document.get_unrecognized_lines()) + + def test_footer_with_value(self): + """ + Tries to parse a descriptor with content on the 'directory-footer' line. + """ + + content = get_network_status_document_v3({"directory-footer": "blarg"}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEqual([DOC_SIG], document.signatures) + self.assertEqual([], document.get_unrecognized_lines()) + + def test_bandwidth_wights_ok(self): + """ + Parses a properly formed 'bandwidth-wights' line. Negative bandwidth + weights might or might not be valid. The spec doesn't say, so making sure + that we accept them. + """ + + weight_entries, expected = [], {} + + for i in xrange(len(BANDWIDTH_WEIGHT_ENTRIES)): + key, value = BANDWIDTH_WEIGHT_ENTRIES[i], i - 5 + + weight_entries.append("%s=%i" % (key, value)) + expected[key] = value + + document = get_network_status_document_v3({"bandwidth-weights": " ".join(weight_entries)}) + self.assertEquals(expected, document.bandwidth_weights) + + def test_bandwidth_wights_malformed(self): + """ + Provides malformed content in the 'bandwidth-wights' line. + """ + + test_values = ( + "Wbe", + "Wbe=", + "Wbe=a", + "Wbe=+7", + ) + + base_weight_entry = " ".join(["%s=5" % e for e in BANDWIDTH_WEIGHT_ENTRIES]) + expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES if e != "Wbe"]) + + for test_value in test_values: + weight_entry = base_weight_entry.replace("Wbe=5", test_value) + content = get_network_status_document_v3({"bandwidth-weights": weight_entry}, content = True) + + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(expected, document.bandwidth_weights) + + def test_bandwidth_wights_misordered(self): + """ + Check that the 'bandwidth-wights' line is rejected if out of order. + """ + + weight_entry = " ".join(["%s=5" % e for e in reversed(BANDWIDTH_WEIGHT_ENTRIES)]) + expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES]) + + content = get_network_status_document_v3({"bandwidth-weights": weight_entry}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(expected, document.bandwidth_weights) + + def test_bandwidth_wights_in_vote(self): + """ + Tries adding a 'bandwidth-wights' line to a vote. + """ + + weight_entry = " ".join(["%s=5" % e for e in BANDWIDTH_WEIGHT_ENTRIES]) + expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES]) + + content = get_network_status_document_v3({"vote-status": "vote", "bandwidth-weights": weight_entry}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(expected, document.bandwidth_weights) + + def test_bandwidth_wights_omissions(self): + """ + Leaves entries out of the 'bandwidth-wights' line. + """ + + # try parsing an empty value + + content = get_network_status_document_v3({"bandwidth-weights": ""}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals({}, document.bandwidth_weights) + + # drop individual values + + for missing_entry in BANDWIDTH_WEIGHT_ENTRIES: + weight_entries = ["%s=5" % e for e in BANDWIDTH_WEIGHT_ENTRIES if e != missing_entry] + expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES if e != missing_entry]) + + content = get_network_status_document_v3({"bandwidth-weights": " ".join(weight_entries)}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(expected, document.bandwidth_weights) + + def test_microdescriptor_signature(self): + """ + The 'directory-signature' lines for normal and microdescriptor conensuses + differ slightly in their format. + """ + + # including a microdescriptor flavored 'directory-signature' line should work + + document = get_network_status_document_v3({"network-status-version": "3 microdesc"}) + self.assertEqual('sha256', document.signatures[0].method) + + # include a standard 'directory-signature' line in a microdescriptor + # consensus + + content = get_network_status_document_v3({ + "network-status-version": "3 microdesc", + "directory-signature": NETWORK_STATUS_DOCUMENT_FOOTER[2][1], + }, content = True) + + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, validate = False) + self.assertEqual([], document.signatures) + + # includes a microdescriptor flavored 'directory-signature' line in a + # normal consensus + + content = get_network_status_document_v3({ + "directory-signature": "sha256 " + NETWORK_STATUS_DOCUMENT_FOOTER[2][1], + }, content = True) + + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, validate = False) + self.assertEqual([], document.signatures) + + def test_malformed_signature(self): + """ + Provides malformed or missing content in the 'directory-signature' line. + """ + + test_values = ( + "", + "\n", + "blarg", + ) + + for test_value in test_values: + for test_attr in xrange(3): + attrs = [DOC_SIG.identity, DOC_SIG.key_digest, DOC_SIG.signature] + attrs[test_attr] = test_value + + content = get_network_status_document_v3({"directory-signature": "%s %s\n%s" % tuple(attrs)}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + NetworkStatusDocumentV3(content, False) # checks that it's still parseable without validation + + def test_with_router_status_entries(self): + """ + Includes router status entries within the document. This isn't to test the + RouterStatusEntry parsing but rather the inclusion of it within the + document. + """ + + entry1 = get_router_status_entry_v3({'s': "Fast"}) + entry2 = get_router_status_entry_v3({'s': "Valid"}) + document = get_network_status_document_v3(routers = (entry1, entry2)) + + self.assertEquals((entry1, entry2), document.routers) + + # try with an invalid RouterStatusEntry + + entry3 = RouterStatusEntryV3(get_router_status_entry_v3({'r': "ugabuga"}, content = True), False) + content = get_network_status_document_v3(routers = (entry3,), content = True) + + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + document = NetworkStatusDocumentV3(content, False) + self.assertEquals((entry3,), document.routers) + + # try including with a microdescriptor consensus + + content = get_network_status_document_v3({"network-status-version": "3 microdesc"}, routers = (entry1, entry2), content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + expected_routers = ( + RouterStatusEntryMicroV3(str(entry1), False), + RouterStatusEntryMicroV3(str(entry2), False), + ) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(expected_routers, document.routers) + + def test_with_microdescriptor_router_status_entries(self): + """ + Includes microdescriptor flavored router status entries within the + document. + """ + + entry1 = get_router_status_entry_micro_v3({'s': "Fast"}) + entry2 = get_router_status_entry_micro_v3({'s': "Valid"}) + document = get_network_status_document_v3({"network-status-version": "3 microdesc"}, routers = (entry1, entry2)) + + self.assertEquals((entry1, entry2), document.routers) + + # try with an invalid RouterStatusEntry + + entry3 = RouterStatusEntryMicroV3(get_router_status_entry_micro_v3({'r': "ugabuga"}, content = True), False) + content = get_network_status_document_v3({"network-status-version": "3 microdesc"}, routers = (entry3,), content = True) + + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + document = NetworkStatusDocumentV3(content, False) + self.assertEquals((entry3,), document.routers) + + # try including microdescriptor entries in a normal consensus + + content = get_network_status_document_v3(routers = (entry1, entry2), content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + expected_routers = ( + RouterStatusEntryV3(str(entry1), False), + RouterStatusEntryV3(str(entry2), False), + ) + + document = NetworkStatusDocumentV3(content, False) + self.assertEquals(expected_routers, document.routers) + + def test_with_directory_authorities(self): + """ + Includes a couple directory authorities in the document. + """ + + for is_document_vote in (False, True): + for is_authorities_vote in (False, True): + authority1 = get_directory_authority({'contact': 'doctor jekyll'}, is_vote = is_authorities_vote) + authority2 = get_directory_authority({'contact': 'mister hyde'}, is_vote = is_authorities_vote) + + vote_status = "vote" if is_document_vote else "consensus" + content = get_network_status_document_v3({"vote-status": vote_status}, authorities = (authority1, authority2), content = True) + + if is_document_vote == is_authorities_vote: + document = NetworkStatusDocumentV3(content) + self.assertEquals((authority1, authority2), document.directory_authorities) + else: + # authority votes in a consensus or consensus authorities in a vote + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + document = NetworkStatusDocumentV3(content, validate = False) + self.assertEquals((authority1, authority2), document.directory_authorities) + + def test_authority_validation_flag_propagation(self): + """ + Includes invalid certificate content in an authority entry. This is testing + that the 'validate' flag propagages from the document to authority, and + authority to certificate classes. + """ + + # make the dir-key-published field of the certiciate be malformed + authority_content = get_directory_authority(is_vote = True, content = True) + authority_content = authority_content.replace("dir-key-published 2011", "dir-key-published 2011a") + + content = get_network_status_document_v3({"vote-status": "vote"}, authorities = (authority_content,), content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content) + + document = NetworkStatusDocumentV3(content, validate = False) + self.assertEquals((DirectoryAuthority(authority_content, False, True),), document.directory_authorities) +

1 0

[stem/master] Add support for network status parsing
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit a9f35d5e86bffc4b04cb525736a23a7e72009efc Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Thu Jul 19 16:46:09 2012 +0200 Add support for network status parsing --- stem/descriptor/networkstatus_descriptor.py | 635 +++++++++++++++++++++++++++ 1 files changed, 635 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/networkstatus_descriptor.py b/stem/descriptor/networkstatus_descriptor.py new file mode 100644 index 0000000..4bcf9a9 --- /dev/null +++ b/stem/descriptor/networkstatus_descriptor.py @@ -0,0 +1,635 @@ +""" +Parsing for Tor network status documents. Currently supports parsing v3 network +status documents (both votes and consensus'). + +The network status documents also contain a list of router descriptors, +directory authorities, signatures etc. + +The votes and consensus' can be obtained from any of the following sources... + +* the 'cached-consensus' file in tor's data directory +* tor metrics, at https://metrics.torproject.org/data.html +* directory authorities and mirrors via their DirPort + +**Module Overview:** + +:: + + parse_file - parses a network status file and provides a NetworkStatusDocument + NetworkStatusDocument - Tor v3 network status document + +- MicrodescriptorConsensus - Tor microdescriptor consensus document + RouterDescriptor - Router descriptor; contains information about a Tor relay + +- RouterMicrodescriptor - Router microdescriptor; contains information that doesn't change often + DirectorySignature + DirectoryAuthority +""" + +import re +import base64 +import hashlib +import datetime + +import stem.prereq +import stem.descriptor +import stem.descriptor.extrainfo_descriptor +import stem.version +import stem.exit_policy +import stem.util.log as log +import stem.util.connection +import stem.util.tor_tools + +_bandwidth_weights_regex = re.compile(" ".join(["W%s=\d+" % weight for weight in ["bd", + "be", "bg", "bm", "db", "eb", "ed", "ee", "eg", "em", "gb", "gd", "gg", "gm", "mb", "md", "me", "mg", "mm"]])) + +def parse_file(document_file, validate = True): + """ + Parses a network status document file, and returns a NetworkStatusDocument + object. + + :param file document_file: file with network status document content + :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise + + :returns: NetworkStatusDocument object created by parsing the file + + :raises: + * ValueError if the contents is malformed and validate is True + * IOError if the file can't be read + """ + + data = document_file.read() + + # if the file has Metrics metadata + if data.startswith("@type network-status-consensus-3 1.0\n") or data.startswith("@type network-status-vote-3 1.0\n"): + return NetworkStatusDocument(data[data.find("\n") + 1:], validate) + + return NetworkStatusDocument(document_file.read(), validate) + +class DocumentParser: + """ + Helper class to parse documents. + + :var str line: current line to be being parsed + :var list lines: list of remaining lines to be parsed + """ + + def __init__(self, raw_content, validate): + """ + Create a new DocumentParser. + + :param str raw_content: content to be parsed + :param bool validate: if False, treats every keyword line as optional + """ + + self._raw_content = raw_content + self.lines = raw_content.split("\n") + self.validate = validate + self.line = self.lines.pop(0) + + def peek_keyword(self): + """ + Returns the first keyword in the next line. Respects the opt keyword and + returns the actual keyword if the first is "opt". + + :returns: the first keyword of the next line + """ + + if self.line: + if self.line.startswith("opt "): + return self.line.split(" ")[1] + return self.line.split(" ")[0] + + def read_keyword_line(self, keyword, optional = False): + """ + Returns the first keyword in the next line it matches the given keyword. + + If it doesn't match, a ValueError is raised if optional is True and if the + DocumentParser was created with validation enabled. If not, None is returned. + + Respects the opt keyword and returns the next keyword if the first is "opt". + + :param str keyword: keyword the line must begin with + :param bool optional: If the current line must begin with the given keyword + + :returns: the text after the keyword if the keyword matches the one provided, otherwise returns None or raises an exception + + :raises: ValueError if a non-optional keyword doesn't match when validation is enabled + """ + + keyword_regex = re.compile("(opt )?" + re.escape(keyword) + "($| )") + + if not self.line: + if not optional and self.validate: + raise ValueError("Unexpected end of document") + return + + if keyword_regex.match(self.line): + try: line, self.line = self.line, self.lines.pop(0) + except IndexError: line, self.line = self.line, None + + if line == "opt " + keyword or line == keyword: return "" + elif line.startswith("opt "): return line.split(" ", 2)[2] + else: return line.split(" ", 1)[1] + elif self.line.startswith("opt"): + # if this was something new introduced at some point in the future + # ignore it and go to the next line + self.read_line() + return self.read_keyword_line(self, keyword, optional) + elif not optional and self.validate: + raise ValueError("Error parsing network status document: Expected %s, received: %s" % (keyword, self.line)) + + def read_line(self): + """ + Returns the current line and shifts the parser to the next line. + + :returns: the current line if it exists, None otherwise + """ + + if self.line: + tmp, self.line = self.line, self.lines.pop(0) + return tmp + + def read_block(self, keyword): + """ + Returns a keyword block that begins with "-----BEGIN keyword-----\\n" and + ends with "-----END keyword-----\\n". + + :param str keyword: keyword block that must be read + + :returns: the data in the keyword block + """ + + lines = [] + + if self.line == "-----BEGIN " + keyword + "-----": + self.read_line() + while self.line != "-----END " + keyword + "-----": + lines.append(self.read_line()) + + return "\n".join(lines) + + def read_until(self, terminals = []): + """ + Returns the data in the parser until a line that begins with one of the keywords in terminals are found. + + :param list terminals: list of strings at which we should stop reading and return the data + + :returns: the current line if it exists, None otherwise + """ + + if self.line == None: return + lines, self.line = [self.line], self.lines.pop(0) + while self.line and not self.line.split(" ")[0] in terminals: + lines.append(self.line) + self.line = self.lines.pop(0) + + return "\n".join(lines) + + def remaining(self): + """ + Returns the data remaining in the parser. + + :returns: all a list of all unparsed lines + """ + + if self.line: + lines, self.lines = self.lines, [] + lines.insert(0, self.line) + return lines + else: + return [] + +def _strptime(string, validate = True, optional = False): + try: + return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") + except ValueError, exc: + if validate or not optional: raise exc + +class NetworkStatusDocument(stem.descriptor.Descriptor): + """ + A v3 network status document. + + This could be a v3 consensus or vote document. + + :var bool validated: **\*** whether the document is validated + :var str network_status_version: **\*** a document format version. For v3 documents this is "3" + :var str vote_status: **\*** status of the vote. Is either "vote" or "consensus" + :var list consensus_methods: A list of supported consensus generation methods (integers) + :var datetime published: time when the document was published + :var int consensus_method: consensus method used to generate a consensus + :var datetime valid_after: **\*** time when the consensus becomes valid + :var datetime fresh_until: **\*** time until when the consensus is considered to be fresh + :var datetime valid_until: **\*** time until when the consensus is valid + :var int vote_delay: **\*** number of seconds allowed for collecting votes from all authorities + :var int dist_delay: number of seconds allowed for collecting signatures from all authorities + :var list client_versions: list of recommended Tor client versions + :var list server_versions: list of recommended Tor server versions + :var list known_flags: **\*** list of known router flags + :var list params: dict of parameter(str) => value(int) mappings + :var list router_descriptors: **\*** list of RouterDescriptor objects defined in the document + :var list directory_authorities: **\*** list of DirectoryAuthority objects that have generated this document + :var dict bandwidth_weights: dict of weight(str) => value(int) mappings + :var list directory_signatures: **\*** list of signatures this document has + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, raw_content, validate = True): + """ + Parse a v3 network status document and provide a new NetworkStatusDocument object. + + :param str raw_content: raw network status document data + :param bool validate: True if the document is to be validated, False otherwise + + :raises: ValueError if the document is invalid + """ + + super(NetworkStatusDocument, self).__init__(raw_content) + + self.router_descriptors = [] + self.directory_authorities = [] + self.directory_signatures = [] + self.validated = validate + + self.network_status_version = None + self.vote_status = None + self.consensus_methods = [] + self.published = None + self.consensus_method = None + self.valid_after = None + self.fresh_until = None + self.valid_until = None + self.vote_delay = None + self.dist_delay = None + self.client_versions = [] + self.server_versions = [] + self.known_flags = [] + self.params = {} + self.bandwidth_weights = {} + + self._parse(raw_content) + + def _generate_router(self, raw_content, vote, validate): + return RouterDescriptor(raw_content, vote, validate) + + def _validate_network_status_version(self): + return self.network_status_version == "3" + + def get_unrecognized_lines(self): + """ + Returns any unrecognized trailing lines. + + :returns: a list of unrecognized trailing lines + """ + + return self._unrecognized_lines + + def _parse(self, raw_content): + # preamble + validate = self.validated + doc_parser = DocumentParser(raw_content, validate) + + read_keyword_line = lambda keyword, optional = False: setattr(self, keyword.replace("-", "_"), doc_parser.read_keyword_line(keyword, optional)) + + map(read_keyword_line, ["network-status-version", "vote-status"]) + if validate and not self._validate_network_status_version(): + raise ValueError("Invalid network-status-version: %s" % self.network_status_version) + + if self.vote_status == "vote": vote = True + elif self.vote_status == "consensus": vote = False + elif validate: raise ValueError("Unrecognized document type specified in vote-status") + + if vote: + read_keyword_line("consensus-methods", True) + self.consensus_methods = [int(method) for method in self.consensus_methods.split(" ")] + self.published = _strptime(doc_parser.read_keyword_line("published", True), validate, True) + else: + self.consensus_method = int(doc_parser.read_keyword_line("consensus-method", True)) + + map(read_keyword_line, ["valid-after", "fresh-until", "valid-until"]) + self.valid_after = _strptime(self.valid_after, validate) + self.fresh_until = _strptime(self.fresh_until, validate) + self.valid_until = _strptime(self.valid_until, validate) + voting_delay = doc_parser.read_keyword_line("voting-delay") + self.vote_delay, self.dist_delay = [int(delay) for delay in voting_delay.split(" ")] + + read_keyword_line("client-versions", True) + self.client_versions = [stem.version.Version(version_string) for version_string in self.client_versions.split(",")] + read_keyword_line("server-versions", True) + self.server_versions = [stem.version.Version(version_string) for version_string in self.server_versions.split(",")] + self.known_flags = doc_parser.read_keyword_line("known-flags").split(" ") + read_keyword_line("params", True) + if self.params: + self.params = dict([(param.split("=")[0], int(param.split("=")[1])) for param in self.params.split(" ")]) + + # authority section + while doc_parser.line.startswith("dir-source "): + dirauth_data = doc_parser.read_until(["dir-source", "r"]) + self.directory_authorities.append(DirectoryAuthority(dirauth_data, vote, validate)) + + # router descriptors + while doc_parser.line.startswith("r "): + router_data = doc_parser.read_until(["r", "directory-footer", "directory-signature"]) + self.router_descriptors.append(self._generate_router(router_data, vote, validate)) + + # footer section + if self.consensus_method > 9 or vote and filter(lambda x: x >= 9, self.consensus_methods): + if doc_parser.line == "directory-footer": + doc_parser.read_line() + elif validate: + raise ValueError("Network status document missing directory-footer") + + if not vote: + read_keyword_line("bandwidth-weights", True) + if _bandwidth_weights_regex.match(self.bandwidth_weights): + self.bandwidth_weights = dict([(weight.split("=")[0], int(weight.split("=")[1])) for weight in self.bandwidth_weights.split(" ")]) + elif validate: + raise ValueError("Invalid bandwidth-weights line") + + while doc_parser.line.startswith("directory-signature "): + signature_data = doc_parser.read_until(["directory-signature"]) + self.directory_signatures.append(DirectorySignature(signature_data)) + + self._unrecognized_lines = doc_parser.remaining() + if validate and self._unrecognized_lines: raise ValueError("Unrecognized trailing data") + +class DirectoryAuthority(stem.descriptor.Descriptor): + """ + Contains directory authority information obtained from v3 network status + documents. + + :var str nickname: directory authority's nickname + :var str identity: uppercase hex fingerprint of the authority's identity key + :var str address: hostname + :var str ip: current IP address + :var int dirport: current directory port + :var int orport: current orport + :var str contact: directory authority's contact information + :var str legacy_dir_key: fingerprint of and obsolete identity key + :var str vote_digest: digest of the authority that contributed to the consensus + """ + + def __init__(self, raw_content, vote = True, validate = True): + """ + Parse a directory authority entry in a v3 network status document and + provide a DirectoryAuthority object. + + :param str raw_content: raw directory authority entry information + :param bool validate: True if the document is to be validated, False otherwise + + :raises: ValueError if the raw data is invalid + """ + + super(DirectoryAuthority, self).__init__(raw_content) + parser = DocumentParser(raw_content, validate) + + dir_source = parser.read_keyword_line("dir-source") + self.nickname, self.identity, self.address, self.ip, self.dirport, self.orport = dir_source.split(" ") + self.dirport = int(self.dirport) + self.orport = int(self.orport) + + self.contact = parser.read_keyword_line("contact") + if vote: + self.legacy_dir_key = parser.read_keyword_line("legacy-dir-key", True) + else: + self.vote_digest = parser.read_keyword_line("vote-digest", True) + if parser.remaining() and validate: + raise ValueError("Unrecognized trailing data in directory authority information") + +class DirectorySignature(stem.descriptor.Descriptor): + """ + Contains directory signature information described in a v3 network status + document. + + :var str identity: signature identity + :var str key_digest: signature key digest + :var str method: method used to generate the signature + :var str signature: the signature data + """ + + def __init__(self, raw_content, validate = True): + """ + Parse a directory signature entry in a v3 network status document and + provide a DirectorySignature object. + + :param str raw_content: raw directory signature entry information + :param bool validate: True if the document is to be validated, False otherwise + + :raises: ValueError if the raw data is invalid + """ + + super(DirectorySignature, self).__init__(raw_content) + parser = DocumentParser(raw_content, validate) + + signature_line = parser.read_keyword_line("directory-signature").split(" ") + + if len(signature_line) == 2: + self.identity, self.key_digest = signature_line + if len(signature_line) == 3: # for microdescriptor consensuses + self.method, self.identity, self.key_digest = signature_line + + self.signature = parser.read_block("SIGNATURE") + if parser.remaining() and validate: + raise ValueError("Unrecognized trailing data in directory signature") + +class RouterDescriptor(stem.descriptor.Descriptor): + """ + Router descriptor object. Parses and stores router information in a router + entry read from a v3 network status document. + + :var str nickname: **\*** router's nickname + :var str identity: **\*** router's identity + :var str digest: **\*** router's digest + :var datetime publication: **\*** router's publication + :var str ip: **\*** router's IP address + :var int orport: **\*** router's ORPort + :var int dirport: **\*** router's DirPort + + :var bool is_valid: **\*** router is valid + :var bool is_guard: **\*** router is suitable for use as an entry guard + :var bool is_named: **\*** router is named + :var bool is_unnamed: **\*** router is unnamed + :var bool is_running: **\*** router is running and currently usable + :var bool is_stable: **\*** router is stable, i.e., it's suitable for for long-lived circuits + :var bool is_exit: **\*** router is an exit router + :var bool is_fast: **\*** router is Fast, i.e., it's usable for high-bandwidth circuits + :var bool is_authority: **\*** router is a directory authority + :var bool supports_v2dir: **\*** router supports v2dir + :var bool supports_v3dir: **\*** router supports v3dir + :var bool is_hsdir: **\*** router is a hidden status + :var bool is_badexit: **\*** router is marked a bad exit + :var bool is_baddirectory: **\*** router is a bad directory + + :var :class:`stem.version.Version`,str version: Version of the Tor protocol this router is running + + :var int bandwidth: router's claimed bandwidth + :var int measured_bandwidth: router's measured bandwidth + + :var :class:`stem.exit_policy.MicrodescriptorExitPolicy` exitpolicy: router's exitpolicy + + :var str mircodescriptor_hashes: "m" SP methods 1*(SP algorithm "=" digest) NL + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, raw_contents, vote = True, validate = True): + """ + Parse a router descriptor in a v3 network status document and provide a new + RouterDescriptor object. + + :param str raw_content: router descriptor content to be parsed + :param bool validate: whether the router descriptor should be validated + """ + + super(RouterDescriptor, self).__init__(raw_contents) + + self.nickname = None + self.identity = None + self.digest = None + self.publication = None + self.ip = None + self.orport = None + self.dirport = None + + self.is_valid = None + self.is_guard = None + self.is_named = None + self.is_unnamed = None + self.is_running = None + self.is_stable = None + self.is_exit = None + self.is_fast = None + self.is_authority = None + self.supports_v2dir = None + self.supports_v3dir = None + self.is_hsdir = None + self.is_badexit = None + self.is_baddirectory = None + + self.version = None + + self.bandwidth = None + self.measured_bandwidth = None + + self.exit_policy = None + + self.mircodescriptor_hashes = [] + + self._parse(raw_contents, vote, validate) + + def _parse(self, raw_content, vote, validate): + """ + :param dict raw_content: iptor contents to be applied + :param bool validate: checks the validity of descriptor content if True + + :raises: ValueError if an error occures in validation + """ + + parser = DocumentParser(raw_content, validate) + seen_keywords = set() + peek_check_kw = lambda keyword: keyword == parser.peek_keyword() + + r = parser.read_keyword_line("r") + # r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0 + # "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort SP DirPort NL + seen_keywords.add("r") + if r: + values = r.split(" ") + self.nickname, self.identity, self.digest = values[0], values[1], values[2] + self.publication = _strptime(" ".join((values[3], values[4])), validate) + self.ip, self.orport, self.dirport = values[5], int(values[6]), int(values[7]) + if self.dirport == 0: self.dirport = None + elif validate: raise ValueError("Invalid router descriptor: empty 'r' line" ) + + while parser.line: + if peek_check_kw("s"): + if "s" in seen_keywords: raise ValueError("Invalid router descriptor: 's' line appears twice") + line = parser.read_keyword_line("s") + if not line: continue + seen_keywords.add("s") + # s Named Running Stable Valid + #A series of space-separated status flags, in *lexical order* + flags = line.split(" ") + flag_map = { + "Valid": "is_valid", + "Guard": "is_guard", + "Named": "is_named", + "Unnamed": "is_unnamed", + "Running": "is_running", + "Stable": "is_stable", + "Exit": "is_exit", + "Fast": "is_fast", + "Authority": "is_authority", + "V2Dir": "supports_v2dir", + "V3Dir": "supports_v3dir", + "HSDir": "is_hsdir", + "BadExit": "is_badexit", + "BadDirectory": "is_baddirectory", + } + map(lambda flag: setattr(self, flag_map[flag], True), flags) + + if self.is_unnamed: self.is_named = False + elif self.is_named: self.is_unnamed = False + + elif peek_check_kw("v"): + if "v" in seen_keywords: raise ValueError("Invalid router descriptor: 'v' line appears twice") + line = parser.read_keyword_line("v", True) + seen_keywords.add("v") + # v Tor 0.2.2.35 + if line: + if line.startswith("Tor "): + self.version = stem.version.Version(line[4:]) + else: + self.version = line + elif validate: raise ValueError("Invalid router descriptor: empty 'v' line" ) + + elif peek_check_kw("w"): + if "w" in seen_keywords: raise ValueError("Invalid router descriptor: 'w' line appears twice") + w = parser.read_keyword_line("w", True) + # "w" SP "Bandwidth=" INT [SP "Measured=" INT] NL + seen_keywords.add("w") + if w: + values = w.split(" ") + if len(values) <= 2 and len(values) > 0: + key, value = values[0].split("=") + if key == "Bandwidth": self.bandwidth = int(value) + elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Bandwidth, read " + key) + + if len(values) == 2: + key, value = values[1].split("=") + if key == "Measured=": self.measured_bandwidth = int(value) + elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Measured, read " + key) + elif validate: raise ValueError("Router descriptor contains invalid 'w' line") + elif validate: raise ValueError("Router descriptor contains empty 'w' line") + + elif peek_check_kw("p"): + if "p" in seen_keywords: raise ValueError("Invalid router descriptor: 'p' line appears twice") + p = parser.read_keyword_line("p", True) + seen_keywords.add("p") + # "p" SP ("accept" / "reject") SP PortList NL + if p: + self.exit_policy = stem.exit_policy.MicrodescriptorExitPolicy(p) + #self.exit_policy = p + + elif vote and peek_check_kw("m"): + # microdescriptor hashes + m = parser.read_keyword_line("m", True) + methods, digests = m.split(" ", 1) + method_list = methods.split(",") + digest_dict = [digest.split("=", 1) for digest in digests.split(" ")] + self.microdescriptor_hashes.append((method_list, digest_dict)) + + elif validate: + raise ValueError("Router descriptor contains unrecognized trailing lines: %s" % parser.line) + + else: + self._unrecognized_lines.append(parser.read_line()) # ignore unrecognized lines if we aren't validating + + def get_unrecognized_lines(self): + """ + Returns any unrecognized lines. + + :returns: a list of unrecognized lines + """ + + return self._unrecognized_lines +

1 0

[stem/master] Move DocumentParser to stem.descriptor.DescriptorParser
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 8ed384d31aa446dd3c5954dd4353e059657ed7a3 Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Mon Aug 6 05:14:51 2012 +0530 Move DocumentParser to stem.descriptor.DescriptorParser --- run_tests.py | 2 + stem/descriptor/__init__.py | 142 ++++++++++++ stem/descriptor/networkstatus_descriptor.py | 317 ++++++++++++--------------- 3 files changed, 285 insertions(+), 176 deletions(-) diff --git a/run_tests.py b/run_tests.py index 0ed5612..10aa24c 100755 --- a/run_tests.py +++ b/run_tests.py @@ -46,6 +46,7 @@ import test.integ.socket.control_socket import test.integ.descriptor.reader import test.integ.descriptor.server_descriptor import test.integ.descriptor.extrainfo_descriptor +import test.integ.descriptor.networkstatus_descriptor import test.integ.response.protocolinfo import test.integ.util.conf import test.integ.util.proc @@ -134,6 +135,7 @@ INTEG_TESTS = ( test.integ.descriptor.reader.TestDescriptorReader, test.integ.descriptor.server_descriptor.TestServerDescriptor, test.integ.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, + test.integ.descriptor.networkstatus_descriptor.TestNetworkStatusDocument, test.integ.version.TestVersion, test.integ.response.protocolinfo.TestProtocolInfo, test.integ.process.TestProcess, diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 45cd016..072564c 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -17,6 +17,7 @@ __all__ = [ "reader", "extrainfo_descriptor", "server_descriptor", + "networkstatus_descriptor", "parse_file", "Descriptor", ] @@ -46,6 +47,7 @@ def parse_file(path, descriptor_file): import stem.descriptor.server_descriptor import stem.descriptor.extrainfo_descriptor + import stem.descriptor.networkstatus_descriptor # The tor descriptor specifications do not provide a reliable method for # identifying a descriptor file's type and version so we need to guess @@ -61,6 +63,8 @@ def parse_file(path, descriptor_file): file_parser = stem.descriptor.server_descriptor.parse_file elif filename == "cached-extrainfo": file_parser = stem.descriptor.extrainfo_descriptor.parse_file + elif filename == "cached-consensus": + file_parser = stem.descriptor.extrainfo_descriptor.parse_file if file_parser: for desc in file_parser(descriptor_file): @@ -92,6 +96,8 @@ def parse_file(path, descriptor_file): # https://trac.torproject.org/6257 desc = stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(descriptor_file.read()) + elif desc_type == "network-status-consensus-3" and major_version == 1: + desc = stem.descriptor.networkstatus_descriptor.NetworkStatusDocument(descriptor_file.read()) if desc: desc._set_path(path) @@ -273,3 +279,139 @@ def _get_descriptor_components(raw_contents, validate, extra_keywords): return entries, first_keyword, last_keyword, extra_entries +class DescriptorParser: + """ + Helper class to parse documents. + + :var str line: current line to be being parsed + :var list lines: list of remaining lines to be parsed + """ + + def __init__(self, raw_content, validate): + """ + Create a new DocumentParser. + + :param str raw_content: content to be parsed + :param bool validate: if False, treats every keyword line as optional + """ + + self._raw_content = raw_content + self.lines = raw_content.split("\n") + self.validate = validate + self.line = self.lines.pop(0) + + def peek_keyword(self): + """ + Returns the first keyword in the next line. Respects the opt keyword and + returns the actual keyword if the first is "opt". + + :returns: the first keyword of the next line + """ + + if self.line: + if self.line.startswith("opt "): + return self.line.split(" ")[1] + return self.line.split(" ")[0] + + def read_keyword_line(self, keyword, optional = False): + """ + Returns the first keyword in the next line it matches the given keyword. + + If it doesn't match, a ValueError is raised if optional is True and if the + DocumentParser was created with validation enabled. If not, None is returned. + + Respects the opt keyword and returns the next keyword if the first is "opt". + + :param str keyword: keyword the line must begin with + :param bool optional: If the current line must begin with the given keyword + + :returns: the text after the keyword if the keyword matches the one provided, otherwise returns None or raises an exception + + :raises: ValueError if a non-optional keyword doesn't match when validation is enabled + """ + + keyword_regex = re.compile("(opt )?" + re.escape(keyword) + "($| )") + + if not self.line: + if not optional and self.validate: + raise ValueError("Unexpected end of document") + return + + if keyword_regex.match(self.line): + try: line, self.line = self.line, self.lines.pop(0) + except IndexError: line, self.line = self.line, None + + if line == "opt " + keyword or line == keyword: return "" + elif line.startswith("opt "): return line.split(" ", 2)[2] + else: return line.split(" ", 1)[1] + elif self.line.startswith("opt"): + # if this was something new introduced at some point in the future + # ignore it and go to the next line + self.read_line() + return self.read_keyword_line(self, keyword, optional) + elif not optional and self.validate: + raise ValueError("Error parsing network status document: Expected %s, received: %s" % (keyword, self.line)) + + def read_line(self): + """ + Returns the current line and shifts the parser to the next line. + + :returns: the current line if it exists, None otherwise + """ + + if self.line: + tmp, self.line = self.line, self.lines.pop(0) + return tmp + + def read_block(self, keyword): + """ + Returns a keyword block that begins with "-----BEGIN keyword-----\\n" and + ends with "-----END keyword-----\\n". + + :param str keyword: keyword block that must be read + + :returns: the data in the keyword block + """ + + lines = [] + + if self.line == "-----BEGIN " + keyword + "-----": + self.read_line() + while self.line != "-----END " + keyword + "-----": + lines.append(self.read_line()) + + self.read_line() # pop out the END line + + return "\n".join(lines) + + def read_until(self, terminals = []): + """ + Returns the data in the parser until a line that begins with one of the keywords in terminals are found. + + :param list terminals: list of strings at which we should stop reading and return the data + + :returns: the current line if it exists, None otherwise + """ + + if self.line == None: return + lines, self.line = [self.line], self.lines.pop(0) + while self.line and not self.line.split(" ")[0] in terminals: + lines.append(self.line) + self.line = self.lines.pop(0) + + return "\n".join(lines) + + def remaining(self): + """ + Returns the data remaining in the parser. + + :returns: all a list of all unparsed lines + """ + + if self.line: + lines, self.lines = self.lines, [] + lines.insert(0, self.line) + return lines + else: + return [] + diff --git a/stem/descriptor/networkstatus_descriptor.py b/stem/descriptor/networkstatus_descriptor.py index 4bcf9a9..008b5dc 100644 --- a/stem/descriptor/networkstatus_descriptor.py +++ b/stem/descriptor/networkstatus_descriptor.py @@ -38,7 +38,7 @@ import stem.util.log as log import stem.util.connection import stem.util.tor_tools -_bandwidth_weights_regex = re.compile(" ".join(["W%s=\d+" % weight for weight in ["bd", +_bandwidth_weights_regex = re.compile(" ".join(["W%s=\d+" % weight for weight in ["bd", "be", "bg", "bm", "db", "eb", "ed", "ee", "eg", "em", "gb", "gd", "gg", "gm", "mb", "md", "me", "mg", "mm"]])) def parse_file(document_file, validate = True): @@ -56,148 +56,8 @@ def parse_file(document_file, validate = True): * IOError if the file can't be read """ - data = document_file.read() - - # if the file has Metrics metadata - if data.startswith("@type network-status-consensus-3 1.0\n") or data.startswith("@type network-status-vote-3 1.0\n"): - return NetworkStatusDocument(data[data.find("\n") + 1:], validate) - return NetworkStatusDocument(document_file.read(), validate) -class DocumentParser: - """ - Helper class to parse documents. - - :var str line: current line to be being parsed - :var list lines: list of remaining lines to be parsed - """ - - def __init__(self, raw_content, validate): - """ - Create a new DocumentParser. - - :param str raw_content: content to be parsed - :param bool validate: if False, treats every keyword line as optional - """ - - self._raw_content = raw_content - self.lines = raw_content.split("\n") - self.validate = validate - self.line = self.lines.pop(0) - - def peek_keyword(self): - """ - Returns the first keyword in the next line. Respects the opt keyword and - returns the actual keyword if the first is "opt". - - :returns: the first keyword of the next line - """ - - if self.line: - if self.line.startswith("opt "): - return self.line.split(" ")[1] - return self.line.split(" ")[0] - - def read_keyword_line(self, keyword, optional = False): - """ - Returns the first keyword in the next line it matches the given keyword. - - If it doesn't match, a ValueError is raised if optional is True and if the - DocumentParser was created with validation enabled. If not, None is returned. - - Respects the opt keyword and returns the next keyword if the first is "opt". - - :param str keyword: keyword the line must begin with - :param bool optional: If the current line must begin with the given keyword - - :returns: the text after the keyword if the keyword matches the one provided, otherwise returns None or raises an exception - - :raises: ValueError if a non-optional keyword doesn't match when validation is enabled - """ - - keyword_regex = re.compile("(opt )?" + re.escape(keyword) + "($| )") - - if not self.line: - if not optional and self.validate: - raise ValueError("Unexpected end of document") - return - - if keyword_regex.match(self.line): - try: line, self.line = self.line, self.lines.pop(0) - except IndexError: line, self.line = self.line, None - - if line == "opt " + keyword or line == keyword: return "" - elif line.startswith("opt "): return line.split(" ", 2)[2] - else: return line.split(" ", 1)[1] - elif self.line.startswith("opt"): - # if this was something new introduced at some point in the future - # ignore it and go to the next line - self.read_line() - return self.read_keyword_line(self, keyword, optional) - elif not optional and self.validate: - raise ValueError("Error parsing network status document: Expected %s, received: %s" % (keyword, self.line)) - - def read_line(self): - """ - Returns the current line and shifts the parser to the next line. - - :returns: the current line if it exists, None otherwise - """ - - if self.line: - tmp, self.line = self.line, self.lines.pop(0) - return tmp - - def read_block(self, keyword): - """ - Returns a keyword block that begins with "-----BEGIN keyword-----\\n" and - ends with "-----END keyword-----\\n". - - :param str keyword: keyword block that must be read - - :returns: the data in the keyword block - """ - - lines = [] - - if self.line == "-----BEGIN " + keyword + "-----": - self.read_line() - while self.line != "-----END " + keyword + "-----": - lines.append(self.read_line()) - - return "\n".join(lines) - - def read_until(self, terminals = []): - """ - Returns the data in the parser until a line that begins with one of the keywords in terminals are found. - - :param list terminals: list of strings at which we should stop reading and return the data - - :returns: the current line if it exists, None otherwise - """ - - if self.line == None: return - lines, self.line = [self.line], self.lines.pop(0) - while self.line and not self.line.split(" ")[0] in terminals: - lines.append(self.line) - self.line = self.lines.pop(0) - - return "\n".join(lines) - - def remaining(self): - """ - Returns the data remaining in the parser. - - :returns: all a list of all unparsed lines - """ - - if self.line: - lines, self.lines = self.lines, [] - lines.insert(0, self.line) - return lines - else: - return [] - def _strptime(string, validate = True, optional = False): try: return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") @@ -212,7 +72,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :var bool validated: **\*** whether the document is validated :var str network_status_version: **\*** a document format version. For v3 documents this is "3" - :var str vote_status: **\*** status of the vote. Is either "vote" or "consensus" + :var str vote_status: **\*** status of the vote (is either "vote" or "consensus") :var list consensus_methods: A list of supported consensus generation methods (integers) :var datetime published: time when the document was published :var int consensus_method: consensus method used to generate a consensus @@ -221,7 +81,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :var datetime valid_until: **\*** time until when the consensus is valid :var int vote_delay: **\*** number of seconds allowed for collecting votes from all authorities :var int dist_delay: number of seconds allowed for collecting signatures from all authorities - :var list client_versions: list of recommended Tor client versions + :var list client_versions: list of recommended Tor client versions :var list server_versions: list of recommended Tor server versions :var list known_flags: **\*** list of known router flags :var list params: dict of parameter(str) => value(int) mappings @@ -280,13 +140,13 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :returns: a list of unrecognized trailing lines """ - + return self._unrecognized_lines def _parse(self, raw_content): # preamble validate = self.validated - doc_parser = DocumentParser(raw_content, validate) + doc_parser = stem.descriptor.DescriptorParser(raw_content, validate) read_keyword_line = lambda keyword, optional = False: setattr(self, keyword.replace("-", "_"), doc_parser.read_keyword_line(keyword, optional)) @@ -326,8 +186,13 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): dirauth_data = doc_parser.read_until(["dir-source", "r"]) self.directory_authorities.append(DirectoryAuthority(dirauth_data, vote, validate)) + i = 1 # router descriptors while doc_parser.line.startswith("r "): + i = i + 1 + if i % 100 == 0: + import pdb + pdb.set_trace() router_data = doc_parser.read_until(["r", "directory-footer", "directory-signature"]) self.router_descriptors.append(self._generate_router(router_data, vote, validate)) @@ -365,6 +230,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): :var int orport: current orport :var str contact: directory authority's contact information :var str legacy_dir_key: fingerprint of and obsolete identity key + :var :class:`stem.descriptor.networkstatus_descriptor.KeyCertificate` key_certificate: directory authority's current key certificate :var str vote_digest: digest of the authority that contributed to the consensus """ @@ -380,7 +246,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): """ super(DirectoryAuthority, self).__init__(raw_content) - parser = DocumentParser(raw_content, validate) + parser = stem.descriptor.DescriptorParser(raw_content, validate) dir_source = parser.read_keyword_line("dir-source") self.nickname, self.identity, self.address, self.ip, self.dirport, self.orport = dir_source.split(" ") @@ -390,11 +256,109 @@ class DirectoryAuthority(stem.descriptor.Descriptor): self.contact = parser.read_keyword_line("contact") if vote: self.legacy_dir_key = parser.read_keyword_line("legacy-dir-key", True) + self.key_certificate = KeyCertificate(parser.remaining(), validate) else: self.vote_digest = parser.read_keyword_line("vote-digest", True) if parser.remaining() and validate: raise ValueError("Unrecognized trailing data in directory authority information") +class KeyCertificate(stem.descriptor.Descriptor): + """ + Directory key certificate. + + :var str key_certificate_version: **\*** version of the key certificate (Should be "3") + :var str ip: IP address on which the directory authority is listening + :var int port: port on which the directory authority is listening + :var str fingerprint: **\*** hex encoded fingerprint of the authority's identity key + :var str identity_key: **\*** long term authority identity key + :var datetime published: **\*** time (in GMT) when this document & the key were last generated + :var str expires: **\*** time (in GMT) after which this key becomes invalid + :var str signing_key: **\*** directory server's public signing key + :var str crosscert: signature made using certificate's signing key + :var str certification: **\*** signature of this key certificate signed with the identity key + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, raw_content, validate = True): + """ + Parse a key certificate entry and provide a KeyCertificate object. + + :param str raw_content: raw key certificate information + :param bool validate: True if the document is to be validated, False otherwise + + :raises: ValueError if the raw data is invalid + """ + + super(DirectoryAuthority, self).__init__(raw_content) + parser = stem.descriptor.DescriptorParser(raw_content, validate) + peek_check_kw = lambda keyword: keyword == parser.peek_keyword() + seen_keywords = set() + + self.key_certificate_version = parser.read_keyword_line("dir-key-certificate-version") + if validate and self.key_certificate_version != "3": raise ValueError("Unrecognized dir-key-certificate-version") + + def _read_keyword_line(keyword): + if validate and keyword in seen_keywords: + raise ValueError("Invalid key certificate: '%s' appears twice" % keyword) + seen_keywords.add(keyword) + return parser.read_keyword_line(keyword) + + while parser.line: + if peek_check_kw("dir-address"): + line = _read_keyword_line("dir-address") + try: + self.ip, self.port = line.rsplit(":", 1) + self.port = int(self.port) + except Exception: + if validate: raise ValueError("Invalid dir-address line: %s" % line) + + elif peek_check_kw("fingerprint"): + self.fingerprint = _read_keyword_line("fingerprint") + + elif peek_check_kw("dir-identity-key"): + _read_keyword_line("dir-identity-key") + self.identity_key = parser.read_block("RSA PUBLIC KEY") + + elif peek_check_kw("dir-key-published"): + self.published = _strptime(_read_keyword_line("dir-key-published")) + + elif peek_check_kw("dir-key-expires"): + self.expires = _strptime(_read_keyword_line("dir-key-expires")) + + elif peek_check_kw("dir-signing-key"): + _read_keyword_line("dir-signing-key") + self.signing_key = parser.read_block("RSA PUBLIC KEY") + + elif peek_check_kw("dir-key-crosscert"): + _read_keyword_line("dir-key-crosscert") + self.crosscert = parser.read_block("ID SIGNATURE") + + elif peek_check_kw("dir-key-certification"): + _read_keyword_line("dir-key-certification") + self.certification = parser.read_block("SIGNATURE") + break + + elif validate: + raise ValueError("Key certificate contains unrecognized lines: %s" % parser.line) + + else: + # ignore unrecognized lines if we aren't validating + self._unrecognized_lines.append(parser.read_line()) + + if parser.remaining(): + if validate: raise ValueError("Unrecognized trailing data in key certificate") + else: self._unrecognized_lines.append(parser.read_line()) + + def get_unrecognized_lines(self): + """ + Returns any unrecognized lines. + + :returns: a list of unrecognized lines + """ + + return self._unrecognized_lines + class DirectorySignature(stem.descriptor.Descriptor): """ Contains directory signature information described in a v3 network status @@ -418,18 +382,19 @@ class DirectorySignature(stem.descriptor.Descriptor): """ super(DirectorySignature, self).__init__(raw_content) - parser = DocumentParser(raw_content, validate) + parser = stem.descriptor.DescriptorParser(raw_content, validate) signature_line = parser.read_keyword_line("directory-signature").split(" ") - + if len(signature_line) == 2: self.identity, self.key_digest = signature_line if len(signature_line) == 3: # for microdescriptor consensuses self.method, self.identity, self.key_digest = signature_line - + self.signature = parser.read_block("SIGNATURE") - if parser.remaining() and validate: - raise ValueError("Unrecognized trailing data in directory signature") + if parser.remaining(): + if validate: raise ValueError("Unrecognized trailing data in directory signature") + else: self._unrecognized_lines.append(parser.read_line()) class RouterDescriptor(stem.descriptor.Descriptor): """ @@ -466,7 +431,7 @@ class RouterDescriptor(stem.descriptor.Descriptor): :var :class:`stem.exit_policy.MicrodescriptorExitPolicy` exitpolicy: router's exitpolicy - :var str mircodescriptor_hashes: "m" SP methods 1*(SP algorithm "=" digest) NL + :var str microdescriptor_hashes: a list of two-tuples with a list of consensus methods(int) that may produce the digest and a dict with algorithm(str) => digest(str) mappings. algorithm is the hashing algorithm (usually "sha256") that is used to produce digest (the base64 encoding of the hash of the router's microdescriptor with trailing =s omitted). **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined """ @@ -490,20 +455,20 @@ class RouterDescriptor(stem.descriptor.Descriptor): self.orport = None self.dirport = None - self.is_valid = None - self.is_guard = None - self.is_named = None - self.is_unnamed = None - self.is_running = None - self.is_stable = None - self.is_exit = None - self.is_fast = None - self.is_authority = None - self.supports_v2dir = None - self.supports_v3dir = None - self.is_hsdir = None - self.is_badexit = None - self.is_baddirectory = None + self.is_valid = False + self.is_guard = False + self.is_named = False + self.is_unnamed = False + self.is_running = False + self.is_stable = False + self.is_exit = False + self.is_fast = False + self.is_authority = False + self.supports_v2dir = False + self.supports_v3dir = False + self.is_hsdir = False + self.is_badexit = False + self.is_baddirectory = False self.version = None @@ -512,7 +477,7 @@ class RouterDescriptor(stem.descriptor.Descriptor): self.exit_policy = None - self.mircodescriptor_hashes = [] + self.microdescriptor_hashes = [] self._parse(raw_contents, vote, validate) @@ -524,7 +489,7 @@ class RouterDescriptor(stem.descriptor.Descriptor): :raises: ValueError if an error occures in validation """ - parser = DocumentParser(raw_content, validate) + parser = stem.descriptor.DescriptorParser(raw_content, validate) seen_keywords = set() peek_check_kw = lambda keyword: keyword == parser.peek_keyword() @@ -593,7 +558,7 @@ class RouterDescriptor(stem.descriptor.Descriptor): key, value = values[0].split("=") if key == "Bandwidth": self.bandwidth = int(value) elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Bandwidth, read " + key) - + if len(values) == 2: key, value = values[1].split("=") if key == "Measured=": self.measured_bandwidth = int(value) @@ -613,23 +578,23 @@ class RouterDescriptor(stem.descriptor.Descriptor): elif vote and peek_check_kw("m"): # microdescriptor hashes m = parser.read_keyword_line("m", True) - methods, digests = m.split(" ", 1) - method_list = methods.split(",") - digest_dict = [digest.split("=", 1) for digest in digests.split(" ")] - self.microdescriptor_hashes.append((method_list, digest_dict)) + #methods, digests = m.split(" ", 1) + #method_list = methods.split(",") + #digest_dict = [digest.split("=", 1) for digest in digests.split(" ")] + #self.microdescriptor_hashes.append((method_list, digest_dict)) elif validate: raise ValueError("Router descriptor contains unrecognized trailing lines: %s" % parser.line) else: self._unrecognized_lines.append(parser.read_line()) # ignore unrecognized lines if we aren't validating - + def get_unrecognized_lines(self): """ Returns any unrecognized lines. :returns: a list of unrecognized lines """ - + return self._unrecognized_lines - +

1 0

[stem/master] match Measured instead of Measured=
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit fc1ad3f21f8d23233551514dc87524a40697621d Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Mon Aug 6 05:28:41 2012 +0530 match Measured instead of Measured= --- stem/descriptor/networkstatus_descriptor.py | 15 +++++---------- 1 files changed, 5 insertions(+), 10 deletions(-) diff --git a/stem/descriptor/networkstatus_descriptor.py b/stem/descriptor/networkstatus_descriptor.py index 008b5dc..6c5544f 100644 --- a/stem/descriptor/networkstatus_descriptor.py +++ b/stem/descriptor/networkstatus_descriptor.py @@ -186,13 +186,8 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): dirauth_data = doc_parser.read_until(["dir-source", "r"]) self.directory_authorities.append(DirectoryAuthority(dirauth_data, vote, validate)) - i = 1 # router descriptors while doc_parser.line.startswith("r "): - i = i + 1 - if i % 100 == 0: - import pdb - pdb.set_trace() router_data = doc_parser.read_until(["r", "directory-footer", "directory-signature"]) self.router_descriptors.append(self._generate_router(router_data, vote, validate)) @@ -561,7 +556,7 @@ class RouterDescriptor(stem.descriptor.Descriptor): if len(values) == 2: key, value = values[1].split("=") - if key == "Measured=": self.measured_bandwidth = int(value) + if key == "Measured": self.measured_bandwidth = int(value) elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Measured, read " + key) elif validate: raise ValueError("Router descriptor contains invalid 'w' line") elif validate: raise ValueError("Router descriptor contains empty 'w' line") @@ -578,10 +573,10 @@ class RouterDescriptor(stem.descriptor.Descriptor): elif vote and peek_check_kw("m"): # microdescriptor hashes m = parser.read_keyword_line("m", True) - #methods, digests = m.split(" ", 1) - #method_list = methods.split(",") - #digest_dict = [digest.split("=", 1) for digest in digests.split(" ")] - #self.microdescriptor_hashes.append((method_list, digest_dict)) + methods, digests = m.split(" ", 1) + method_list = methods.split(",") + digest_dict = [digest.split("=", 1) for digest in digests.split(" ")] + self.microdescriptor_hashes.append((method_list, digest_dict)) elif validate: raise ValueError("Router descriptor contains unrecognized trailing lines: %s" % parser.line)

1 0