[tor-commits] [stem/master] Supporting microdescriptor flavored consensuses
atagar at torproject.org
atagar at torproject.org
Sat Oct 13 18:35:45 UTC 2012
commit 4216b5f1d5762d229945306508ea078c9fd1902c
Author: Damian Johnson <atagar at torproject.org>
Date: Sun Oct 7 18:47:27 2012 -0700
Supporting microdescriptor flavored consensuses
Adding support for microdescriptor flavored consensuses into the
NetworkStatusDocument class. It made sense to have a separate class for it, but
on the other hand it *is* still a v3 consensus and the only impact the flavor
has is alternate router status entries so just blending a 'flavor' and
'is_microdescriptor' attribute in.
---
stem/descriptor/networkstatus.py | 208 ++++--------------------
test/unit/descriptor/networkstatus/document.py | 72 ++++++++-
2 files changed, 100 insertions(+), 180 deletions(-)
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index 38207b8..d9aab4f 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -156,11 +156,9 @@ def parse_file(document_file, validate = True, is_microdescriptor = False):
document_content = "".join(header + footer)
if not is_microdescriptor:
- document = NetworkStatusDocument(document_content, validate)
router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3
else:
- document = MicrodescriptorConsensus(document_content, validate)
- router_type = RouterMicrodescriptor
+ router_type = stem.descriptor.router_status_entry.RouterStatusEntryMicroV3
desc_iterator = _get_entries(
document_file,
@@ -169,7 +167,7 @@ def parse_file(document_file, validate = True, is_microdescriptor = False):
entry_keyword = ROUTERS_START,
start_position = routers_start,
end_position = routers_end,
- extra_args = (document,),
+ extra_args = (NetworkStatusDocument(document_content, validate),),
)
for desc in desc_iterator:
@@ -221,9 +219,11 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
:var tuple routers: RouterStatusEntryV3 contained in the document
- :var str version: **\*** document version
+ :var int version: **\*** document version
+ :var str version_flavor: **\*** flavor associated with the document (such as 'microdesc')
:var bool is_consensus: **\*** true if the document is a consensus
:var bool is_vote: **\*** true if the document is a vote
+ :var bool is_microdescriptor: **\*** true if this is a microdescriptor flavored document, false otherwise
:var datetime valid_after: **\*** time when the consensus became valid
:var datetime fresh_until: **\*** time when the next consensus should be produced
:var datetime valid_until: **\*** time when this consensus becomes obsolete
@@ -262,6 +262,14 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
document_file = StringIO(raw_content)
self._header = _DocumentHeader(document_file, validate, default_params)
+ self._unrecognized_lines = []
+
+ # merge header attributes into us
+ for attr, value in vars(self._header).items():
+ if attr != "_unrecognized_lines":
+ setattr(self, attr, value)
+ else:
+ self._unrecognized_lines += value
self.directory_authorities = tuple(_get_entries(
document_file,
@@ -272,28 +280,29 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
extra_args = (self._header.is_vote,),
))
+ if not self._header.is_microdescriptor:
+ router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3
+ else:
+ router_type = stem.descriptor.router_status_entry.RouterStatusEntryMicroV3
+
self.routers = tuple(_get_entries(
document_file,
validate,
- entry_class = self._get_router_type(),
+ entry_class = router_type,
entry_keyword = ROUTERS_START,
section_end_keywords = FOOTER_START,
extra_args = (self,),
))
self._footer = _DocumentFooter(document_file, validate, self._header)
- self._unrecognized_lines = []
- # copy the header and footer attributes into us
- for attr, value in vars(self._header).items() + vars(self._footer).items():
+ # merge header attributes into us
+ for attr, value in vars(self._footer).items():
if attr != "_unrecognized_lines":
setattr(self, attr, value)
else:
self._unrecognized_lines += value
- def _get_router_type(self):
- return stem.descriptor.router_status_entry.RouterStatusEntryV3
-
def meets_consensus_method(self, method):
"""
Checks if we meet the given consensus-method. This works for both votes and
@@ -319,8 +328,10 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
class _DocumentHeader(object):
def __init__(self, document_file, validate, default_params):
self.version = None
+ self.version_flavor = None
self.is_consensus = True
self.is_vote = False
+ self.is_microdescriptor = False
self.consensus_methods = []
self.published = None
self.consensus_method = None
@@ -362,12 +373,20 @@ class _DocumentHeader(object):
if keyword == 'network-status-version':
# "network-status-version" version
- self.version = value
+ if ' ' in value:
+ version, flavor = value.split(' ', 1)
+ else:
+ version, flavor = value, None
+
+ if not version.isdigit():
+ if not validate: continue
+ raise ValueError("Network status document has a non-numeric version: %s" % line)
- # TODO: Obviously not right when we extend this to parse v2 documents,
- # but we'll cross that bridge when we come to it.
+ self.version = int(version)
+ self.version_flavor = flavor
+ self.is_microdescriptor = flavor == 'microdesc'
- if validate and self.version != "3":
+ if validate and self.version != 3:
raise ValueError("Expected a version 3 network status documents, got version '%s' instead" % self.version)
elif keyword == 'vote-status':
# "vote-status" type
@@ -1034,160 +1053,3 @@ class DocumentSignature(object):
return 0
-class MicrodescriptorConsensus(NetworkStatusDocument):
- """
- A v3 microdescriptor consensus.
-
- :var str version: **\*** a document format version. For v3 microdescriptor consensuses this is "3 microdesc"
- :var bool is_consensus: **\*** true if the document is a consensus
- :var bool is_vote: **\*** true if the document is a vote
- :var int consensus_method: **~** consensus method used to generate a consensus
- :var datetime valid_after: **\*** time when the consensus becomes valid
- :var datetime fresh_until: **\*** time until when the consensus is considered to be fresh
- :var datetime valid_until: **\*** time until when the consensus is valid
- :var int vote_delay: **\*** number of seconds allowed for collecting votes from all authorities
- :var int dist_delay: number of seconds allowed for collecting signatures from all authorities
- :var list client_versions: list of recommended Tor client versions
- :var list server_versions: list of recommended Tor server versions
- :var list known_flags: **\*** list of known router flags
- :var list params: dict of parameter(str) => value(int) mappings
- :var list directory_authorities: **\*** list of DirectoryAuthority objects that have generated this document
- :var dict bandwidth_weights: **~** dict of weight(str) => value(int) mappings
- :var list signatures: **\*** list of signatures this document has
-
- | **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined
- | **~** attribute appears only in consensuses
- """
-
- def _get_router_type(self):
- return RouterMicrodescriptor
-
- def _validate_network_status_version(self):
- return self.version == "3 microdesc"
-
-class RouterMicrodescriptor(stem.descriptor.router_status_entry.RouterStatusEntry):
- """
- Router microdescriptor object. Parses and stores router information in a router
- microdescriptor from a v3 microdescriptor consensus.
-
- :var MicrodescriptorConsensus document: **\*** document this descriptor came from
-
- :var str nickname: **\*** router's nickname
- :var str fingerprint: **\*** router's fingerprint
- :var datetime published: **\*** router's publication
- :var str ip: **\*** router's IP address
- :var int or_port: **\*** router's ORPort
- :var int dir_port: **\*** router's DirPort
-
- :var list flags: **\*** list of status flags
-
- :var :class:`stem.version.Version`,str version: Version of the Tor protocol this router is running
-
- :var int bandwidth: router's claimed bandwidth
- :var int measured_bandwidth: router's measured bandwidth
-
- :var str digest: base64 of the hash of the router's microdescriptor with trailing =s omitted
-
- | **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined
- """
-
- def __init__(self, raw_contents, validate, document):
- """
- Parse a router descriptor in a v3 microdescriptor consensus and provide a new
- RouterMicrodescriptor object.
-
- :param str raw_content: router descriptor content to be parsed
- :param MicrodescriptorConsensus document: document this descriptor came from
- :param bool validate: whether the router descriptor should be validated
-
- :raises: ValueError if the descriptor data is invalid
- """
-
- super(RouterMicrodescriptor, self).__init__(raw_contents, validate, document)
-
- self.document = document
-
- def _parse(self, raw_content, validate):
- """
- :param dict raw_content: router descriptor contents to be parsed
- :param bool validate: checks the validity of descriptor content if True
-
- :raises: ValueError if an error occures in validation
- """
-
- content = StringIO(raw_content)
- seen_keywords = set()
- peek_check_kw = lambda keyword: keyword == _peek_keyword(content)
-
- r = _read_keyword_line("r", content, validate)
- # r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0
- # "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort SP DirPort NL
- if r:
- seen_keywords.add("r")
- values = r.split(" ")
- self.nickname, self.fingerprint = values[0], _decode_fingerprint(values[1], validate)
- self.published = _strptime(" ".join((values[2], values[3])), validate)
- self.ip, self.or_port, self.dir_port = values[4], int(values[5]), int(values[6])
- if self.dir_port == 0: self.dir_port = None
- elif validate: raise ValueError("Invalid router descriptor: empty 'r' line")
-
- while _peek_line(content):
- if peek_check_kw("s"):
- if "s" in seen_keywords: raise ValueError("Invalid router descriptor: 's' line appears twice")
- line = _read_keyword_line("s", content, validate)
- if not line: continue
- seen_keywords.add("s")
- # s Named Running Stable Valid
- #A series of space-separated status flags, in *lexical order*
- self.flags = line.split(" ")
-
- elif peek_check_kw("v"):
- if "v" in seen_keywords: raise ValueError("Invalid router descriptor: 'v' line appears twice")
- line = _read_keyword_line("v", content, validate, True)
- seen_keywords.add("v")
- # v Tor 0.2.2.35
- if line:
- if line.startswith("Tor "):
- self.version = stem.version.Version(line[4:])
- else:
- self.version = line
- elif validate: raise ValueError("Invalid router descriptor: empty 'v' line" )
-
- elif peek_check_kw("w"):
- if "w" in seen_keywords: raise ValueError("Invalid router descriptor: 'w' line appears twice")
- w = _read_keyword_line("w", content, validate, True)
- # "w" SP "Bandwidth=" INT [SP "Measured=" INT] NL
- seen_keywords.add("w")
- if w:
- values = w.split(" ")
- if len(values) <= 2 and len(values) > 0:
- key, value = values[0].split("=")
- if key == "Bandwidth": self.bandwidth = int(value)
- elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Bandwidth, read " + key)
-
- if len(values) == 2:
- key, value = values[1].split("=")
- if key == "Measured": self.measured_bandwidth = int(value)
- elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Measured, read " + key)
- elif validate: raise ValueError("Router descriptor contains invalid 'w' line")
- elif validate: raise ValueError("Router descriptor contains empty 'w' line")
-
- elif peek_check_kw("m"):
- # microdescriptor hashes
- self.digest = _read_keyword_line("m", content, validate, True)
-
- elif validate:
- raise ValueError("Router descriptor contains unrecognized trailing lines: %s" % content.readline())
-
- else:
- self.unrecognized_lines.append(content.readline()) # ignore unrecognized lines if we aren't validating
-
- def get_unrecognized_lines(self):
- """
- Returns any unrecognized lines.
-
- :returns: a list of unrecognized lines
- """
-
- return self.unrecognized_lines
-
diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py
index e92aee2..8009fbb 100644
--- a/test/unit/descriptor/networkstatus/document.py
+++ b/test/unit/descriptor/networkstatus/document.py
@@ -9,8 +9,8 @@ import StringIO
import stem.version
from stem.descriptor import Flag
from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, DirectoryAuthority, NetworkStatusDocument, parse_file
-from stem.descriptor.router_status_entry import RouterStatusEntryV3
-from test.mocking import get_router_status_entry_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG
+from stem.descriptor.router_status_entry import RouterStatusEntryV3, RouterStatusEntryMicroV3
+from test.mocking import get_router_status_entry_v3, get_router_status_entry_micro_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG
class TestNetworkStatusDocument(unittest.TestCase):
def test_minimal_consensus(self):
@@ -25,9 +25,11 @@ class TestNetworkStatusDocument(unittest.TestCase):
Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID]
self.assertEqual((), document.routers)
- self.assertEqual("3", document.version)
+ self.assertEqual(3, document.version)
+ self.assertEqual(None, document.version_flavor)
self.assertEqual(True, document.is_consensus)
self.assertEqual(False, document.is_vote)
+ self.assertEqual(False, document.is_microdescriptor)
self.assertEqual(9, document.consensus_method)
self.assertEqual([], document.consensus_methods)
self.assertEqual(None, document.published)
@@ -57,7 +59,7 @@ class TestNetworkStatusDocument(unittest.TestCase):
Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID]
self.assertEqual((), document.routers)
- self.assertEqual("3", document.version)
+ self.assertEqual(3, document.version)
self.assertEqual(False, document.is_consensus)
self.assertEqual(True, document.is_vote)
self.assertEqual(None, document.consensus_method)
@@ -178,13 +180,22 @@ class TestNetworkStatusDocument(unittest.TestCase):
"""
document = get_network_status_document({"network-status-version": "3"})
- self.assertEquals("3", document.version)
+ self.assertEquals(3, document.version)
+ self.assertEquals(None, document.version_flavor)
+ self.assertEquals(False, document.is_microdescriptor)
+
+ document = get_network_status_document({"network-status-version": "3 microdesc"})
+ self.assertEquals(3, document.version)
+ self.assertEquals('microdesc', document.version_flavor)
+ self.assertEquals(True, document.is_microdescriptor)
content = get_network_status_document({"network-status-version": "4"}, content = True)
self.assertRaises(ValueError, NetworkStatusDocument, content)
document = NetworkStatusDocument(content, False)
- self.assertEquals("4", document.version)
+ self.assertEquals(4, document.version)
+ self.assertEquals(None, document.version_flavor)
+ self.assertEquals(False, document.is_microdescriptor)
def test_vote_status(self):
"""
@@ -616,7 +627,7 @@ class TestNetworkStatusDocument(unittest.TestCase):
def test_with_router_status_entries(self):
"""
- Includes a router status entry within the document. This isn't to test the
+ Includes router status entries within the document. This isn't to test the
RouterStatusEntry parsing but rather the inclusion of it within the
document.
"""
@@ -635,6 +646,53 @@ class TestNetworkStatusDocument(unittest.TestCase):
self.assertRaises(ValueError, NetworkStatusDocument, content)
document = NetworkStatusDocument(content, False)
self.assertEquals((entry3,), document.routers)
+
+ # try including with a microdescriptor consensus
+
+ content = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry1, entry2), content = True)
+ self.assertRaises(ValueError, NetworkStatusDocument, content)
+
+ expected_routers = (
+ RouterStatusEntryMicroV3(str(entry1), False),
+ RouterStatusEntryMicroV3(str(entry2), False),
+ )
+
+ document = NetworkStatusDocument(content, False)
+ self.assertEquals(expected_routers, document.routers)
+
+ def test_with_microdescriptor_router_status_entries(self):
+ """
+ Includes microdescriptor flavored router status entries within the
+ document.
+ """
+
+ entry1 = get_router_status_entry_micro_v3({'s': "Fast"})
+ entry2 = get_router_status_entry_micro_v3({'s': "Valid"})
+ document = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry1, entry2))
+
+ self.assertEquals((entry1, entry2), document.routers)
+
+ # try with an invalid RouterStatusEntry
+
+ entry3 = RouterStatusEntryMicroV3(get_router_status_entry_micro_v3({'r': "ugabuga"}, content = True), False)
+ content = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry3,), content = True)
+
+ self.assertRaises(ValueError, NetworkStatusDocument, content)
+ document = NetworkStatusDocument(content, False)
+ self.assertEquals((entry3,), document.routers)
+
+ # try including microdescriptor entries in a normal consensus
+
+ content = get_network_status_document(routers = (entry1, entry2), content = True)
+ self.assertRaises(ValueError, NetworkStatusDocument, content)
+
+ expected_routers = (
+ RouterStatusEntryV3(str(entry1), False),
+ RouterStatusEntryV3(str(entry2), False),
+ )
+
+ document = NetworkStatusDocument(content, False)
+ self.assertEquals(expected_routers, document.routers)
def test_with_directory_authorities(self):
"""
More information about the tor-commits
mailing list