commit 35c5eaf2933dad64bab61cc65657d60db276a54a Author: Damian Johnson atagar@torproject.org Date: Sun Aug 19 00:21:08 2012 -0700
Inverting network status' document and router relationship
This code was trying very hard to produce a network status document that contained router entries. This is right and proper from an object oriented standpoint, but not from a practical one. The document cannot contain the router entries unless we keep them all in memory, and for the vast majority of users that is not desirable.
In later revisions Ravi addressed the memory concerns by turning the router_descriptors attribute into an iterator. This works, but it's confusing and the router_descriptors' itertor only works as long as we keep the file open (or buffer the content).
Instead, inverting the relationship so this module provides an iterator of routers and those routers have a reference to the document they came from. As a side benefit this lets us tidy up the code a bit.
There like are some use cases where we would prefer a document object that contains routers, so I'll probably add that option too later.
This also gets rid of the "Flavour" enum. I'm not entirely sure what it was for - it wasn't documented and its NS value was unused. Maybe it was vestigial from some prior changes. --- stem/descriptor/__init__.py | 19 +++-- stem/descriptor/networkstatus.py | 127 ++++++++++++++++---------------- test/integ/descriptor/networkstatus.py | 15 ++-- 3 files changed, 82 insertions(+), 79 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 151f13e..74b538c 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -65,9 +65,9 @@ def parse_file(path, descriptor_file): elif filename == "cached-extrainfo": file_parser = stem.descriptor.extrainfo_descriptor.parse_file elif filename == "cached-consensus": - file_parser = lambda f: stem.descriptor.networkstatus.parse_file(f).router_descriptors + file_parser = stem.descriptor.networkstatus.parse_file elif filename == "cached-microdesc-consensus": - file_parser = lambda f: stem.descriptor.networkstatus.parse_file(f, True, "microdesc").router_descriptors + file_parser = lambda f: stem.descriptor.networkstatus.parse_file(f, True, True) else: # Metrics descriptor handling first_line, desc = descriptor_file.readline().strip(), None @@ -107,14 +107,10 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto
yield stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(descriptor_file.read()) elif descriptor_type in ("network-status-consensus-3", "network-status-vote-3") and major_version == 1: - consensus = stem.descriptor.networkstatus.parse_file(descriptor_file) - - for desc in consensus.router_descriptors: + for desc in stem.descriptor.networkstatus.parse_file(descriptor_file): yield desc elif descriptor_type == "network-status-microdesc-consensus-3" and major_version == 1: - consensus = stem.descriptor.networkstatus.parse_file(descriptor_file, flavour = "microdesc") - - for desc in consensus.router_descriptors: + for desc in stem.descriptor.networkstatus.parse_file(descriptor_file, is_microdescriptor = True): yield desc else: raise TypeError("Unrecognized metrics descriptor format. type: '%s', version: '%i.%i'" % (descriptor_type, major_version, minor_version)) @@ -253,7 +249,7 @@ def _read_keyword_line_str(keyword, lines, validate = True, optional = False): raise ValueError("Error parsing network status document: Expected %s, received: %s" % (keyword, lines[0])) else: return None
-def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False, skip = False): +def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False, skip = False, end_position = None): """ Reads from the descriptor file until we get to one of the given keywords or reach the end of the file. @@ -263,6 +259,7 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi :param bool inclusive: includes the line with the keyword if True :param bool ignore_first: doesn't check if the first line read has one of the given keywords :param bool skip: skips buffering content, returning None + :param int end_position: end if we reach this point in the file
:returns: list with the lines until we find one of the keywords """ @@ -278,6 +275,10 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi
while True: last_position = descriptor_file.tell() + + if end_position and last_position >= end_position: + break + line = descriptor_file.readline() if not line: break # EOF
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 3b592ec..5054628 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -59,14 +59,6 @@ from stem.descriptor import _read_keyword_line, _read_keyword_line_str, _get_pse _bandwidth_weights_regex = re.compile(" ".join(["W%s=\d+" % weight for weight in ["bd", "be", "bg", "bm", "db", "eb", "ed", "ee", "eg", "em", "gb", "gd", "gg", "gm", "mb", "md", "me", "mg", "mm"]]))
-_router_desc_end_kws = ["r", "bandwidth-weights", "directory-footer", "directory-signature"] - -Flavour = stem.util.enum.Enum( - ("NONE", ""), - ("NS", "ns"), - ("MICRODESCRIPTOR", "microdesc"), -) - Flag = stem.util.enum.Enum( ("AUTHORITY", "Authority"), ("BADEXIT", "BadExit"), @@ -82,12 +74,13 @@ Flag = stem.util.enum.Enum( ("VALID", "Valid"), )
-def parse_file(document_file, validate = True, flavour = Flavour.NONE): +def parse_file(document_file, validate = True, is_microdescriptor = False): """ Parses a network status document and provides a NetworkStatusDocument object.
:param file document_file: file with network status document content :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise + :param bool is_microdescriptor: True if this is for a microdescriptor consensus, False otherwise
:returns: :class:`stem.descriptor.networkstatus.NetworkStatusDocument` object
@@ -96,35 +89,52 @@ def parse_file(document_file, validate = True, flavour = Flavour.NONE): * IOError if the file can't be read """
- # parse until "r" - document_data = "".join(_read_until_keywords("r", document_file)) - # store offset - r_offset = document_file.tell() - # skip until end of router descriptors - _read_until_keywords(["bandwidth-weights", "directory-footer", "directory-signature"], document_file, skip = True) - # parse until end - document_data = document_data + document_file.read() - - if flavour == Flavour.NONE: - document = NetworkStatusDocument(document_data, validate) - document_file.seek(r_offset) - document.router_descriptors = _ns_router_desc_generator(document_file, document.vote_status == "vote", validate) - return document - elif flavour == Flavour.MICRODESCRIPTOR: - document = MicrodescriptorConsensus(document_data, validate) - document_file.seek(r_offset) - document.router_descriptors = _router_microdesc_generator(document_file, validate, document.known_flags) - return document - -def _ns_router_desc_generator(document_file, vote, validate): - while _peek_keyword(document_file) == "r": - desc_content = "".join(_read_until_keywords(_router_desc_end_kws, document_file, False, True)) - yield RouterDescriptor(desc_content, vote, validate) + if not is_microdescriptor: + document_type, router_type = NetworkStatusDocument, RouterDescriptor + else: + document_type, router_type = MicrodescriptorConsensus, RouterMicrodescriptor + + document, routers_start, routers_end = _get_document(document_file, validate, document_type) + document_file.seek(routers_start) + + while document_file.tell() < routers_end: + desc_content = "".join(_read_until_keywords("r", document_file, ignore_first = True, end_position = routers_end)) + yield router_type(desc_content, document, validate)
-def _router_microdesc_generator(document_file, validate, known_flags): - while _peek_keyword(document_file) == "r": - desc_content = "".join(_read_until_keywords(_router_desc_end_kws, document_file, False, True)) - yield RouterMicrodescriptor(desc_content, validate, known_flags) +def _get_document(document_file, validate, document_type): + """ + Network status documents consist of three sections: header, router entries, + and the footer. This provides back a tuple with the following... + (NetworkStatusDocument, routers_start, routers_end) + + :param file document_file: file with network status document content + :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise + :param object document_type: consensus document class to construct + + :returns: tuple with the network status document and range that has the routers + + :raises: + * ValueError if the contents is malformed and validate is True + * IOError if the file can't be read + """ + + # parse until the first router record + + header = _read_until_keywords("r", document_file) + routers_start = document_file.tell() + + # figure out the network status version + + # TODO: we should pick either 'directory-footer' or 'directory-signature' + # based on the header's network-status-version + + _read_until_keywords(["directory-footer", "directory-signature"], document_file, skip = True) + routers_end = document_file.tell() + footer = document_file.readlines() + + document_data = "".join(header + footer) + + return (document_type(document_data, validate), routers_start, routers_end)
class NetworkStatusDocument(stem.descriptor.Descriptor): """ @@ -147,7 +157,6 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :var list server_versions: list of recommended Tor server versions :var list known_flags: ***** list of known router flags :var list params: dict of parameter(str) => value(int) mappings - :var list router_descriptors: ***** iterator for RouterDescriptor objects defined in the document :var list directory_authorities: ***** list of DirectoryAuthority objects that have generated this document :var dict bandwidth_weights: **~** dict of weight(str) => value(int) mappings :var list directory_signatures: ***** list of signatures this document has @@ -169,7 +178,6 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
super(NetworkStatusDocument, self).__init__(raw_content)
- self.router_descriptors = [] self.directory_authorities = [] self.directory_signatures = [] self.validated = validate @@ -192,11 +200,6 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
self._parse(raw_content)
- def _router_desc_generator(self, document_file): - while _peek_keyword(document_file) == "r": - desc_content = "".join(_read_until_keywords(_router_desc_end_kws, document_file, False, True)) - yield RouterDescriptor(desc_content, self.vote_status == "vote", self.validated, self.known_flags) - def _validate_network_status_version(self): return self.network_status_version == "3"
@@ -255,11 +258,6 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): dirauth_data = "".join(dirauth_data).rstrip() self.directory_authorities.append(DirectoryAuthority(dirauth_data, vote, validate))
- # router descriptors - if _peek_keyword(content) == "r": - router_descriptors_data = "".join(_read_until_keywords(["bandwidth-weights", "directory-footer", "directory-signature"], content, False, True)) - self.router_descriptors = self._router_desc_generator(StringIO(router_descriptors_data)) - # footer section if self.consensus_method > 9 or vote and filter(lambda x: x >= 9, self.consensus_methods): if _peek_keyword(content) == "directory-footer": @@ -396,6 +394,8 @@ class RouterDescriptor(stem.descriptor.Descriptor): Router descriptor object. Parses and stores router information in a router entry read from a v3 network status document.
+ :var NetworkStatusDocument document: ***** document this descriptor came from + :var str nickname: ***** router's nickname :var str identity: ***** router's identity :var str digest: ***** router's digest @@ -420,13 +420,13 @@ class RouterDescriptor(stem.descriptor.Descriptor): | exit_policy appears only in votes """
- def __init__(self, raw_contents, vote = True, validate = True, known_flags = Flag): + def __init__(self, raw_contents, document, validate = True, known_flags = Flag): """ Parse a router descriptor in a v3 network status document and provide a new RouterDescriptor object.
:param str raw_content: router descriptor content to be parsed - :param bool vote: True if the descriptor is from a vote document + :param NetworkStatusDocument document: document this descriptor came from :param bool validate: whether the router descriptor should be validated :param bool known_flags: list of known router status flags
@@ -435,6 +435,8 @@ class RouterDescriptor(stem.descriptor.Descriptor):
super(RouterDescriptor, self).__init__(raw_contents)
+ self.document = document + self.nickname = None self.identity = None self.digest = None @@ -455,18 +457,18 @@ class RouterDescriptor(stem.descriptor.Descriptor):
self.microdescriptor_hashes = []
- self._parse(raw_contents, vote, validate, known_flags) + self._parse(raw_contents, validate, known_flags)
- def _parse(self, raw_content, vote, validate, known_flags): + def _parse(self, raw_content, validate, known_flags): """ :param dict raw_content: iptor contents to be applied - :param bool vote: True if the descriptor is from a vote document :param bool validate: checks the validity of descriptor content if True :param bool known_flags: list of known router status flags
:raises: ValueError if an error occures in validation """
+ vote = self.document.vote_status == "vote" content = StringIO(raw_content) seen_keywords = set() peek_check_kw = lambda keyword: keyword == _peek_keyword(content) @@ -576,7 +578,6 @@ class MicrodescriptorConsensus(NetworkStatusDocument): :var list server_versions: list of recommended Tor server versions :var list known_flags: ***** list of known router flags :var list params: dict of parameter(str) => value(int) mappings - :var list router_descriptors: ***** iterator for RouterDescriptor objects defined in the document :var list directory_authorities: ***** list of DirectoryAuthority objects that have generated this document :var dict bandwidth_weights: **~** dict of weight(str) => value(int) mappings :var list directory_signatures: ***** list of signatures this document has @@ -585,11 +586,6 @@ class MicrodescriptorConsensus(NetworkStatusDocument): | **~** attribute appears only in consensuses """
- def _router_desc_generator(self, document_file): - while _peek_keyword(document_file) == "r": - desc_content = "".join(_read_until_keywords(_router_desc_end_kws, document_file, False, True)) - yield RouterMicrodescriptor(desc_content, self.validated, self.known_flags) - def _validate_network_status_version(self): return self.network_status_version == "3 microdesc"
@@ -598,6 +594,8 @@ class RouterMicrodescriptor(RouterDescriptor): Router microdescriptor object. Parses and stores router information in a router microdescriptor from a v3 microdescriptor consensus.
+ :var MicrodescriptorConsensus document: ***** document this descriptor came from + :var str nickname: ***** router's nickname :var str identity: ***** router's identity :var datetime publication: ***** router's publication @@ -618,21 +616,24 @@ class RouterMicrodescriptor(RouterDescriptor): | ***** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined """
- def __init__(self, raw_contents, validate = True, known_flags = Flag): + def __init__(self, raw_contents, document, validate = True, known_flags = Flag): """ Parse a router descriptor in a v3 microdescriptor consensus and provide a new RouterMicrodescriptor object.
:param str raw_content: router descriptor content to be parsed + :param MicrodescriptorConsensus document: document this descriptor came from :param bool validate: whether the router descriptor should be validated :param bool known_flags: list of known router status flags
:raises: ValueError if the descriptor data is invalid """
- super(RouterMicrodescriptor, self).__init__(raw_contents, False, validate, known_flags) + super(RouterMicrodescriptor, self).__init__(raw_contents, document, validate, known_flags) + + self.document = document
- def _parse(self, raw_content, _, validate, known_flags): + def _parse(self, raw_content, validate, known_flags): """ :param dict raw_content: router descriptor contents to be parsed :param bool validate: checks the validity of descriptor content if True diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index 11bd3e3..f95270b 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -13,7 +13,6 @@ import stem.exit_policy import stem.version import stem.descriptor.networkstatus import test.integ.descriptor -from stem.descriptor.networkstatus import Flavour
def _strptime(string): return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") @@ -39,7 +38,7 @@ class TestNetworkStatusDocument(unittest.TestCase):
count = 0 with open(descriptor_path) as descriptor_file: - for desc in stem.descriptor.networkstatus.parse_file(descriptor_file).router_descriptors: + for desc in stem.descriptor.networkstatus.parse_file(descriptor_file): if resource.getrusage(resource.RUSAGE_SELF).ru_maxrss > 200000: # if we're using > 200 MB we should fail self.fail() @@ -75,7 +74,8 @@ class TestNetworkStatusDocument(unittest.TestCase): descriptor_path = test.integ.descriptor.get_resource("cached-consensus")
descriptor_file = file(descriptor_path) - desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) + router1 = next(stem.descriptor.networkstatus.parse_file(descriptor_file)) + desc = router1.document descriptor_file.close()
self.assertEquals(True, desc.validated) @@ -102,7 +102,7 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertEquals(set(desc.known_flags), set(["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", "Named", "Running", "Stable", "Unnamed", "V2Dir", "Valid"])) expected_params = {"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1} self.assertEquals(expected_params, desc.params) - router1 = next(desc.router_descriptors) + self.assertEquals("sumkledi", router1.nickname) self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router1.identity) self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router1.digest) @@ -167,7 +167,8 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= descriptor_path = test.integ.descriptor.get_resource("vote")
descriptor_file = file(descriptor_path) - desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) + router1 = next(stem.descriptor.networkstatus.parse_file(descriptor_file)) + desc = router1.document descriptor_file.close()
self.assertEquals(True, desc.validated) @@ -186,7 +187,7 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals(set(desc.known_flags), set(["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", "Running", "Stable", "V2Dir", "Valid"])) expected_params = {"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1} self.assertEquals(expected_params, desc.params) - router1 = next(desc.router_descriptors) + self.assertEquals("sumkledi", router1.nickname) self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router1.identity) self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router1.digest) @@ -272,7 +273,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w=
count = 0 with open(descriptor_path) as descriptor_file: - for desc in stem.descriptor.networkstatus.parse_file(descriptor_file, True, flavour = Flavour.MICRODESCRIPTOR).router_descriptors: + for desc in stem.descriptor.networkstatus.parse_file(descriptor_file, True, is_microdescriptor = True): assert desc.nickname # check that the router has a nickname count += 1