- tor-commits - lists.torproject.org

[stem/master] Iterating over routers with parse_file()
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 7927579282e6b6a3071dc1901c0a725a94fa7563 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Aug 18 16:45:43 2012 -0700 Iterating over routers with parse_file() Having parse_file() iterate over the RouterDescriptors in a consensus rather than providing the document itself. 99.9% of the time our callers want to do something with the routers, and providing them a document instead just forces them to unwrap it. I'm thinking about changing the networkstatus module to provide RouterDescriptors that reference the NetworkStatusDocument they came from rather than the other way around. We'll see how it works. --- stem/descriptor/__init__.py | 14 ++++++++++---- test/integ/descriptor/networkstatus.py | 8 ++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 8e2bdc7..e6f3e4c 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -65,9 +65,9 @@ def parse_file(path, descriptor_file): elif filename == "cached-extrainfo": file_parser = stem.descriptor.extrainfo_descriptor.parse_file elif filename == "cached-consensus": - file_parser = lambda f: [stem.descriptor.networkstatus.parse_file(f)] + file_parser = lambda f: stem.descriptor.networkstatus.parse_file(f).router_descriptors elif filename == "cached-microdesc-consensus": - file_parser = lambda f: [stem.descriptor.networkstatus.parse_file(f, True, "microdesc")] + file_parser = lambda f: stem.descriptor.networkstatus.parse_file(f, True, "microdesc").router_descriptors else: # Metrics descriptor handling first_line, desc = descriptor_file.readline().strip(), None @@ -107,9 +107,15 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto yield stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(descriptor_file.read()) elif descriptor_type in ("network-status-consensus-3", "network-status-vote-3") and major_version == 1: - yield stem.descriptor.networkstatus.parse_file(descriptor_file) + consensus = stem.descriptor.networkstatus.parse_file(descriptor_file) + + for desc in consensus.router_descriptors: + yield desc elif descriptor_type == "network-status-microdesc-consensus-3" and major_version == 1: - yield stem.descriptor.networkstatus.parse_file(descriptor_file, flavour = "microdesc") + consensus = stem.descriptor.networkstatus.parse_file(descriptor_file, flavour = "microdesc") + + for desc in consensus.router_descriptors: + yield desc else: raise TypeError("Unrecognized metrics descriptor format. type: '%s', version: '%i.%i'" % (descriptor_type, major_version, minor_version)) diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index 4cbf42c..11bd3e3 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -56,9 +56,9 @@ class TestNetworkStatusDocument(unittest.TestCase): descriptor_path = test.integ.descriptor.get_resource("metrics_consensus") with file(descriptor_path) as descriptor_file: - desc = stem.descriptor.parse_file(descriptor_path, descriptor_file) + descriptors = stem.descriptor.parse_file(descriptor_path, descriptor_file) - router = next(next(desc).router_descriptors) + router = next(descriptors) self.assertEquals("sumkledi", router.nickname) self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router.identity) self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router.digest) @@ -148,9 +148,9 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= descriptor_path = test.integ.descriptor.get_resource("metrics_vote") with file(descriptor_path) as descriptor_file: - desc = stem.descriptor.parse_file(descriptor_path, descriptor_file) + descriptors = stem.descriptor.parse_file(descriptor_path, descriptor_file) - router = next(next(desc).router_descriptors) + router = next(descriptors) self.assertEquals("sumkledi", router.nickname) self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router.identity) self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router.digest)

1 0

[stem/master] Hybrid approach between iterator and attribute
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit f9a2ef8d0dd60a73c7cfe5a329b20329c4b4218c Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 19 11:54:29 2012 -0700 Hybrid approach between iterator and attribute Ok, changed my mind. At least partly. If the user is using parse_file() then we want to provide them with router entries and a bare document, like I was. However, if they're calling the document constructors then they've already read the complete document into memory so there's little reason not to parse it. In short, documents now have a 'routers' attribute. That attribute is filled with the parsed routers if we use the constructor directly, and left empty if we used parse_file(). --- stem/descriptor/networkstatus.py | 75 +++++++++++++++++++++++++------- test/integ/descriptor/networkstatus.py | 8 ++-- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 5054628..a91dcc5 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -76,7 +76,9 @@ Flag = stem.util.enum.Enum( def parse_file(document_file, validate = True, is_microdescriptor = False): """ - Parses a network status document and provides a NetworkStatusDocument object. + Parses a network status and iterates over the RouterDescriptor or + RouterMicrodescriptor in it. The document that these instances reference have + an empty 'rotuers' attribute to allow for limited memory usage. :param file document_file: file with network status document content :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise @@ -89,29 +91,31 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): * IOError if the file can't be read """ + header, footer, routers_end = _get_document_content(document_file, validate) + document_data = "".join(header + footer) + if not is_microdescriptor: - document_type, router_type = NetworkStatusDocument, RouterDescriptor + document = NetworkStatusDocument(document_data, validate) + router_type = RouterDescriptor else: - document_type, router_type = MicrodescriptorConsensus, RouterMicrodescriptor - - document, routers_start, routers_end = _get_document(document_file, validate, document_type) - document_file.seek(routers_start) + document = MicrodescriptorConsensus(document_data, validate) + router_type = RouterMicrodescriptor - while document_file.tell() < routers_end: - desc_content = "".join(_read_until_keywords("r", document_file, ignore_first = True, end_position = routers_end)) - yield router_type(desc_content, document, validate) + for desc in _get_routers(document_file, validate, document, routers_end, router_type): + yield desc -def _get_document(document_file, validate, document_type): +def _get_document_content(document_file, validate): """ Network status documents consist of three sections: header, router entries, and the footer. This provides back a tuple with the following... - (NetworkStatusDocument, routers_start, routers_end) + (header_lines, footer_lines, routers_end) + + This leaves the document_file at the start of the router entries. :param file document_file: file with network status document content :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise - :param object document_type: consensus document class to construct - :returns: tuple with the network status document and range that has the routers + :returns: tuple with the network status document content and ending position of the routers :raises: * ValueError if the contents is malformed and validate is True @@ -132,9 +136,31 @@ def _get_document(document_file, validate, document_type): routers_end = document_file.tell() footer = document_file.readlines() - document_data = "".join(header + footer) + document_file.seek(routers_start) + return (header, footer, routers_end) + +def _get_routers(document_file, validate, document, end_position, router_type): + """ + Iterates over the router entries in a given document. The document_file is + expected to be at the start of the router section and the end_position + desigates where that section ends. + + :param file document_file: file with network status document content + :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise + :param object document: document the descriptors originate from + :param int end_position: location in the document_file where the router section ends + :param class router_type: router class to construct - return (document_type(document_data, validate), routers_start, routers_end) + :returns: iterator over router_type instances + + :raises: + * ValueError if the contents is malformed and validate is True + * IOError if the file can't be read + """ + + while document_file.tell() < end_position: + desc_content = "".join(_read_until_keywords("r", document_file, ignore_first = True, end_position = end_position)) + yield router_type(desc_content, document, validate) class NetworkStatusDocument(stem.descriptor.Descriptor): """ @@ -142,6 +168,8 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): This could be a v3 consensus or vote document. + :var tuple routers: RouterDescriptor contained in the document + :var bool validated: **\*** whether the document is validated :var str network_status_version: **\*** a document format version. For v3 documents this is "3" :var str vote_status: **\*** status of the vote (is either "vote" or "consensus") @@ -198,7 +226,19 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): self.params = {} self.bandwidth_weights = {} - self._parse(raw_content) + document_file = StringIO(raw_content) + header, footer, routers_end = _get_document_content(document_file, validate) + + document_content = "".join(header + footer) + self._parse(document_content) + + if document_file.tell() < routers_end: + self.routers = tuple(_get_routers(document_file, validate, self, routers_end, self._get_router_type())) + else: + self.routers = () + + def _get_router_type(self): + return RouterDescriptor def _validate_network_status_version(self): return self.network_status_version == "3" @@ -586,6 +626,9 @@ class MicrodescriptorConsensus(NetworkStatusDocument): | **~** attribute appears only in consensuses """ + def _get_router_type(self): + return RouterMicrodescriptor + def _validate_network_status_version(self): return self.network_status_version == "3 microdesc" diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index f95270b..d16929d 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -74,8 +74,8 @@ class TestNetworkStatusDocument(unittest.TestCase): descriptor_path = test.integ.descriptor.get_resource("cached-consensus") descriptor_file = file(descriptor_path) - router1 = next(stem.descriptor.networkstatus.parse_file(descriptor_file)) - desc = router1.document + desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) + router1 = desc.routers[0] descriptor_file.close() self.assertEquals(True, desc.validated) @@ -167,8 +167,8 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= descriptor_path = test.integ.descriptor.get_resource("vote") descriptor_file = file(descriptor_path) - router1 = next(stem.descriptor.networkstatus.parse_file(descriptor_file)) - desc = router1.document + desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) + router1 = desc.routers[0] descriptor_file.close() self.assertEquals(True, desc.validated)

1 0

[stem/master] Inverting network status' document and router relationship
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 35c5eaf2933dad64bab61cc65657d60db276a54a Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 19 00:21:08 2012 -0700 Inverting network status' document and router relationship This code was trying very hard to produce a network status document that contained router entries. This is right and proper from an object oriented standpoint, but not from a practical one. The document cannot contain the router entries unless we keep them all in memory, and for the vast majority of users that is not desirable. In later revisions Ravi addressed the memory concerns by turning the router_descriptors attribute into an iterator. This works, but it's confusing and the router_descriptors' itertor only works as long as we keep the file open (or buffer the content). Instead, inverting the relationship so this module provides an iterator of routers and those routers have a reference to the document they came from. As a side benefit this lets us tidy up the code a bit. There like are some use cases where we would prefer a document object that contains routers, so I'll probably add that option too later. This also gets rid of the "Flavour" enum. I'm not entirely sure what it was for - it wasn't documented and its NS value was unused. Maybe it was vestigial from some prior changes. --- stem/descriptor/__init__.py | 19 +++-- stem/descriptor/networkstatus.py | 127 ++++++++++++++++---------------- test/integ/descriptor/networkstatus.py | 15 ++-- 3 files changed, 82 insertions(+), 79 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 151f13e..74b538c 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -65,9 +65,9 @@ def parse_file(path, descriptor_file): elif filename == "cached-extrainfo": file_parser = stem.descriptor.extrainfo_descriptor.parse_file elif filename == "cached-consensus": - file_parser = lambda f: stem.descriptor.networkstatus.parse_file(f).router_descriptors + file_parser = stem.descriptor.networkstatus.parse_file elif filename == "cached-microdesc-consensus": - file_parser = lambda f: stem.descriptor.networkstatus.parse_file(f, True, "microdesc").router_descriptors + file_parser = lambda f: stem.descriptor.networkstatus.parse_file(f, True, True) else: # Metrics descriptor handling first_line, desc = descriptor_file.readline().strip(), None @@ -107,14 +107,10 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto yield stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(descriptor_file.read()) elif descriptor_type in ("network-status-consensus-3", "network-status-vote-3") and major_version == 1: - consensus = stem.descriptor.networkstatus.parse_file(descriptor_file) - - for desc in consensus.router_descriptors: + for desc in stem.descriptor.networkstatus.parse_file(descriptor_file): yield desc elif descriptor_type == "network-status-microdesc-consensus-3" and major_version == 1: - consensus = stem.descriptor.networkstatus.parse_file(descriptor_file, flavour = "microdesc") - - for desc in consensus.router_descriptors: + for desc in stem.descriptor.networkstatus.parse_file(descriptor_file, is_microdescriptor = True): yield desc else: raise TypeError("Unrecognized metrics descriptor format. type: '%s', version: '%i.%i'" % (descriptor_type, major_version, minor_version)) @@ -253,7 +249,7 @@ def _read_keyword_line_str(keyword, lines, validate = True, optional = False): raise ValueError("Error parsing network status document: Expected %s, received: %s" % (keyword, lines[0])) else: return None -def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False, skip = False): +def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False, skip = False, end_position = None): """ Reads from the descriptor file until we get to one of the given keywords or reach the end of the file. @@ -263,6 +259,7 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi :param bool inclusive: includes the line with the keyword if True :param bool ignore_first: doesn't check if the first line read has one of the given keywords :param bool skip: skips buffering content, returning None + :param int end_position: end if we reach this point in the file :returns: list with the lines until we find one of the keywords """ @@ -278,6 +275,10 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi while True: last_position = descriptor_file.tell() + + if end_position and last_position >= end_position: + break + line = descriptor_file.readline() if not line: break # EOF diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 3b592ec..5054628 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -59,14 +59,6 @@ from stem.descriptor import _read_keyword_line, _read_keyword_line_str, _get_pse _bandwidth_weights_regex = re.compile(" ".join(["W%s=\d+" % weight for weight in ["bd", "be", "bg", "bm", "db", "eb", "ed", "ee", "eg", "em", "gb", "gd", "gg", "gm", "mb", "md", "me", "mg", "mm"]])) -_router_desc_end_kws = ["r", "bandwidth-weights", "directory-footer", "directory-signature"] - -Flavour = stem.util.enum.Enum( - ("NONE", ""), - ("NS", "ns"), - ("MICRODESCRIPTOR", "microdesc"), -) - Flag = stem.util.enum.Enum( ("AUTHORITY", "Authority"), ("BADEXIT", "BadExit"), @@ -82,12 +74,13 @@ Flag = stem.util.enum.Enum( ("VALID", "Valid"), ) -def parse_file(document_file, validate = True, flavour = Flavour.NONE): +def parse_file(document_file, validate = True, is_microdescriptor = False): """ Parses a network status document and provides a NetworkStatusDocument object. :param file document_file: file with network status document content :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise + :param bool is_microdescriptor: True if this is for a microdescriptor consensus, False otherwise :returns: :class:`stem.descriptor.networkstatus.NetworkStatusDocument` object @@ -96,35 +89,52 @@ def parse_file(document_file, validate = True, flavour = Flavour.NONE): * IOError if the file can't be read """ - # parse until "r" - document_data = "".join(_read_until_keywords("r", document_file)) - # store offset - r_offset = document_file.tell() - # skip until end of router descriptors - _read_until_keywords(["bandwidth-weights", "directory-footer", "directory-signature"], document_file, skip = True) - # parse until end - document_data = document_data + document_file.read() - - if flavour == Flavour.NONE: - document = NetworkStatusDocument(document_data, validate) - document_file.seek(r_offset) - document.router_descriptors = _ns_router_desc_generator(document_file, document.vote_status == "vote", validate) - return document - elif flavour == Flavour.MICRODESCRIPTOR: - document = MicrodescriptorConsensus(document_data, validate) - document_file.seek(r_offset) - document.router_descriptors = _router_microdesc_generator(document_file, validate, document.known_flags) - return document - -def _ns_router_desc_generator(document_file, vote, validate): - while _peek_keyword(document_file) == "r": - desc_content = "".join(_read_until_keywords(_router_desc_end_kws, document_file, False, True)) - yield RouterDescriptor(desc_content, vote, validate) + if not is_microdescriptor: + document_type, router_type = NetworkStatusDocument, RouterDescriptor + else: + document_type, router_type = MicrodescriptorConsensus, RouterMicrodescriptor + + document, routers_start, routers_end = _get_document(document_file, validate, document_type) + document_file.seek(routers_start) + + while document_file.tell() < routers_end: + desc_content = "".join(_read_until_keywords("r", document_file, ignore_first = True, end_position = routers_end)) + yield router_type(desc_content, document, validate) -def _router_microdesc_generator(document_file, validate, known_flags): - while _peek_keyword(document_file) == "r": - desc_content = "".join(_read_until_keywords(_router_desc_end_kws, document_file, False, True)) - yield RouterMicrodescriptor(desc_content, validate, known_flags) +def _get_document(document_file, validate, document_type): + """ + Network status documents consist of three sections: header, router entries, + and the footer. This provides back a tuple with the following... + (NetworkStatusDocument, routers_start, routers_end) + + :param file document_file: file with network status document content + :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise + :param object document_type: consensus document class to construct + + :returns: tuple with the network status document and range that has the routers + + :raises: + * ValueError if the contents is malformed and validate is True + * IOError if the file can't be read + """ + + # parse until the first router record + + header = _read_until_keywords("r", document_file) + routers_start = document_file.tell() + + # figure out the network status version + + # TODO: we should pick either 'directory-footer' or 'directory-signature' + # based on the header's network-status-version + + _read_until_keywords(["directory-footer", "directory-signature"], document_file, skip = True) + routers_end = document_file.tell() + footer = document_file.readlines() + + document_data = "".join(header + footer) + + return (document_type(document_data, validate), routers_start, routers_end) class NetworkStatusDocument(stem.descriptor.Descriptor): """ @@ -147,7 +157,6 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :var list server_versions: list of recommended Tor server versions :var list known_flags: **\*** list of known router flags :var list params: dict of parameter(str) => value(int) mappings - :var list router_descriptors: **\*** iterator for RouterDescriptor objects defined in the document :var list directory_authorities: **\*** list of DirectoryAuthority objects that have generated this document :var dict bandwidth_weights: **~** dict of weight(str) => value(int) mappings :var list directory_signatures: **\*** list of signatures this document has @@ -169,7 +178,6 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): super(NetworkStatusDocument, self).__init__(raw_content) - self.router_descriptors = [] self.directory_authorities = [] self.directory_signatures = [] self.validated = validate @@ -192,11 +200,6 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): self._parse(raw_content) - def _router_desc_generator(self, document_file): - while _peek_keyword(document_file) == "r": - desc_content = "".join(_read_until_keywords(_router_desc_end_kws, document_file, False, True)) - yield RouterDescriptor(desc_content, self.vote_status == "vote", self.validated, self.known_flags) - def _validate_network_status_version(self): return self.network_status_version == "3" @@ -255,11 +258,6 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): dirauth_data = "".join(dirauth_data).rstrip() self.directory_authorities.append(DirectoryAuthority(dirauth_data, vote, validate)) - # router descriptors - if _peek_keyword(content) == "r": - router_descriptors_data = "".join(_read_until_keywords(["bandwidth-weights", "directory-footer", "directory-signature"], content, False, True)) - self.router_descriptors = self._router_desc_generator(StringIO(router_descriptors_data)) - # footer section if self.consensus_method > 9 or vote and filter(lambda x: x >= 9, self.consensus_methods): if _peek_keyword(content) == "directory-footer": @@ -396,6 +394,8 @@ class RouterDescriptor(stem.descriptor.Descriptor): Router descriptor object. Parses and stores router information in a router entry read from a v3 network status document. + :var NetworkStatusDocument document: **\*** document this descriptor came from + :var str nickname: **\*** router's nickname :var str identity: **\*** router's identity :var str digest: **\*** router's digest @@ -420,13 +420,13 @@ class RouterDescriptor(stem.descriptor.Descriptor): | exit_policy appears only in votes """ - def __init__(self, raw_contents, vote = True, validate = True, known_flags = Flag): + def __init__(self, raw_contents, document, validate = True, known_flags = Flag): """ Parse a router descriptor in a v3 network status document and provide a new RouterDescriptor object. :param str raw_content: router descriptor content to be parsed - :param bool vote: True if the descriptor is from a vote document + :param NetworkStatusDocument document: document this descriptor came from :param bool validate: whether the router descriptor should be validated :param bool known_flags: list of known router status flags @@ -435,6 +435,8 @@ class RouterDescriptor(stem.descriptor.Descriptor): super(RouterDescriptor, self).__init__(raw_contents) + self.document = document + self.nickname = None self.identity = None self.digest = None @@ -455,18 +457,18 @@ class RouterDescriptor(stem.descriptor.Descriptor): self.microdescriptor_hashes = [] - self._parse(raw_contents, vote, validate, known_flags) + self._parse(raw_contents, validate, known_flags) - def _parse(self, raw_content, vote, validate, known_flags): + def _parse(self, raw_content, validate, known_flags): """ :param dict raw_content: iptor contents to be applied - :param bool vote: True if the descriptor is from a vote document :param bool validate: checks the validity of descriptor content if True :param bool known_flags: list of known router status flags :raises: ValueError if an error occures in validation """ + vote = self.document.vote_status == "vote" content = StringIO(raw_content) seen_keywords = set() peek_check_kw = lambda keyword: keyword == _peek_keyword(content) @@ -576,7 +578,6 @@ class MicrodescriptorConsensus(NetworkStatusDocument): :var list server_versions: list of recommended Tor server versions :var list known_flags: **\*** list of known router flags :var list params: dict of parameter(str) => value(int) mappings - :var list router_descriptors: **\*** iterator for RouterDescriptor objects defined in the document :var list directory_authorities: **\*** list of DirectoryAuthority objects that have generated this document :var dict bandwidth_weights: **~** dict of weight(str) => value(int) mappings :var list directory_signatures: **\*** list of signatures this document has @@ -585,11 +586,6 @@ class MicrodescriptorConsensus(NetworkStatusDocument): | **~** attribute appears only in consensuses """ - def _router_desc_generator(self, document_file): - while _peek_keyword(document_file) == "r": - desc_content = "".join(_read_until_keywords(_router_desc_end_kws, document_file, False, True)) - yield RouterMicrodescriptor(desc_content, self.validated, self.known_flags) - def _validate_network_status_version(self): return self.network_status_version == "3 microdesc" @@ -598,6 +594,8 @@ class RouterMicrodescriptor(RouterDescriptor): Router microdescriptor object. Parses and stores router information in a router microdescriptor from a v3 microdescriptor consensus. + :var MicrodescriptorConsensus document: **\*** document this descriptor came from + :var str nickname: **\*** router's nickname :var str identity: **\*** router's identity :var datetime publication: **\*** router's publication @@ -618,21 +616,24 @@ class RouterMicrodescriptor(RouterDescriptor): | **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined """ - def __init__(self, raw_contents, validate = True, known_flags = Flag): + def __init__(self, raw_contents, document, validate = True, known_flags = Flag): """ Parse a router descriptor in a v3 microdescriptor consensus and provide a new RouterMicrodescriptor object. :param str raw_content: router descriptor content to be parsed + :param MicrodescriptorConsensus document: document this descriptor came from :param bool validate: whether the router descriptor should be validated :param bool known_flags: list of known router status flags :raises: ValueError if the descriptor data is invalid """ - super(RouterMicrodescriptor, self).__init__(raw_contents, False, validate, known_flags) + super(RouterMicrodescriptor, self).__init__(raw_contents, document, validate, known_flags) + + self.document = document - def _parse(self, raw_content, _, validate, known_flags): + def _parse(self, raw_content, validate, known_flags): """ :param dict raw_content: router descriptor contents to be parsed :param bool validate: checks the validity of descriptor content if True diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index 11bd3e3..f95270b 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -13,7 +13,6 @@ import stem.exit_policy import stem.version import stem.descriptor.networkstatus import test.integ.descriptor -from stem.descriptor.networkstatus import Flavour def _strptime(string): return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") @@ -39,7 +38,7 @@ class TestNetworkStatusDocument(unittest.TestCase): count = 0 with open(descriptor_path) as descriptor_file: - for desc in stem.descriptor.networkstatus.parse_file(descriptor_file).router_descriptors: + for desc in stem.descriptor.networkstatus.parse_file(descriptor_file): if resource.getrusage(resource.RUSAGE_SELF).ru_maxrss > 200000: # if we're using > 200 MB we should fail self.fail() @@ -75,7 +74,8 @@ class TestNetworkStatusDocument(unittest.TestCase): descriptor_path = test.integ.descriptor.get_resource("cached-consensus") descriptor_file = file(descriptor_path) - desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) + router1 = next(stem.descriptor.networkstatus.parse_file(descriptor_file)) + desc = router1.document descriptor_file.close() self.assertEquals(True, desc.validated) @@ -102,7 +102,7 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertEquals(set(desc.known_flags), set(["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", "Named", "Running", "Stable", "Unnamed", "V2Dir", "Valid"])) expected_params = {"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1} self.assertEquals(expected_params, desc.params) - router1 = next(desc.router_descriptors) + self.assertEquals("sumkledi", router1.nickname) self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router1.identity) self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router1.digest) @@ -167,7 +167,8 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= descriptor_path = test.integ.descriptor.get_resource("vote") descriptor_file = file(descriptor_path) - desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) + router1 = next(stem.descriptor.networkstatus.parse_file(descriptor_file)) + desc = router1.document descriptor_file.close() self.assertEquals(True, desc.validated) @@ -186,7 +187,7 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals(set(desc.known_flags), set(["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", "Running", "Stable", "V2Dir", "Valid"])) expected_params = {"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1} self.assertEquals(expected_params, desc.params) - router1 = next(desc.router_descriptors) + self.assertEquals("sumkledi", router1.nickname) self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router1.identity) self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router1.digest) @@ -272,7 +273,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= count = 0 with open(descriptor_path) as descriptor_file: - for desc in stem.descriptor.networkstatus.parse_file(descriptor_file, True, flavour = Flavour.MICRODESCRIPTOR).router_descriptors: + for desc in stem.descriptor.networkstatus.parse_file(descriptor_file, True, is_microdescriptor = True): assert desc.nickname # check that the router has a nickname count += 1

1 0

[stem/master] Renaming RouterDescriptor to RouterStatusEntry
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 393c4de2817a27c1cbdaa7bdf881e9cd08e61a76 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 19 16:35:42 2012 -0700 Renaming RouterDescriptor to RouterStatusEntry The dir-spec calls these "router status entries", so using that name in stem too. --- stem/descriptor/networkstatus.py | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index a91dcc5..7eab8da 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -4,7 +4,7 @@ status documents (both votes and consensuses). The network status documents also contain a list of router descriptors, directory authorities, signatures etc. If you only need the -:class:`stem.descriptor.networkstatus.RouterDescriptor` objects, use +:class:`stem.descriptor.networkstatus.RouterStatusEntry` objects, use :func:`stem.descriptor.parse_file`. Other information can be accessed by directly instantiating :class:`stem.descriptor.networkstatus.NetworkStatusDocument` objects. @@ -34,7 +34,7 @@ The documents can be obtained from any of the following sources... parse_file - parses a network status file and provides a NetworkStatusDocument NetworkStatusDocument - Tor v3 network status document +- MicrodescriptorConsensus - Microdescriptor flavoured consensus documents - RouterDescriptor - Router descriptor; contains information about a Tor relay + RouterStatusEntry - Router descriptor; contains information about a Tor relay +- RouterMicrodescriptor - Router microdescriptor; contains information that doesn't change frequently DirectorySignature - Network status document's directory signature DirectoryAuthority - Directory authority defined in a v3 network status document @@ -76,7 +76,7 @@ Flag = stem.util.enum.Enum( def parse_file(document_file, validate = True, is_microdescriptor = False): """ - Parses a network status and iterates over the RouterDescriptor or + Parses a network status and iterates over the RouterStatusEntry or RouterMicrodescriptor in it. The document that these instances reference have an empty 'rotuers' attribute to allow for limited memory usage. @@ -96,7 +96,7 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): if not is_microdescriptor: document = NetworkStatusDocument(document_data, validate) - router_type = RouterDescriptor + router_type = RouterStatusEntry else: document = MicrodescriptorConsensus(document_data, validate) router_type = RouterMicrodescriptor @@ -168,7 +168,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): This could be a v3 consensus or vote document. - :var tuple routers: RouterDescriptor contained in the document + :var tuple routers: RouterStatusEntry contained in the document :var bool validated: **\*** whether the document is validated :var str network_status_version: **\*** a document format version. For v3 documents this is "3" @@ -238,7 +238,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): self.routers = () def _get_router_type(self): - return RouterDescriptor + return RouterStatusEntry def _validate_network_status_version(self): return self.network_status_version == "3" @@ -429,7 +429,7 @@ class DirectorySignature(stem.descriptor.Descriptor): return self.unrecognized_lines -class RouterDescriptor(stem.descriptor.Descriptor): +class RouterStatusEntry(stem.descriptor.Descriptor): """ Router descriptor object. Parses and stores router information in a router entry read from a v3 network status document. @@ -463,7 +463,7 @@ class RouterDescriptor(stem.descriptor.Descriptor): def __init__(self, raw_contents, document, validate = True, known_flags = Flag): """ Parse a router descriptor in a v3 network status document and provide a new - RouterDescriptor object. + RouterStatusEntry object. :param str raw_content: router descriptor content to be parsed :param NetworkStatusDocument document: document this descriptor came from @@ -473,7 +473,7 @@ class RouterDescriptor(stem.descriptor.Descriptor): :raises: ValueError if the descriptor data is invalid """ - super(RouterDescriptor, self).__init__(raw_contents) + super(RouterStatusEntry, self).__init__(raw_contents) self.document = document @@ -632,7 +632,7 @@ class MicrodescriptorConsensus(NetworkStatusDocument): def _validate_network_status_version(self): return self.network_status_version == "3 microdesc" -class RouterMicrodescriptor(RouterDescriptor): +class RouterMicrodescriptor(RouterStatusEntry): """ Router microdescriptor object. Parses and stores router information in a router microdescriptor from a v3 microdescriptor consensus.

1 0

[stem/master] Error due to KEYWORD_LINE regex change
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 2b4ec7c1c6e1e1848ca3080d9da41cb02d51b0ea Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 19 18:11:49 2012 -0700 Error due to KEYWORD_LINE regex change The KEYWORD_LINE regex fix a few commits back broke one of the server descriptor unit tests because it caused lines with only a keyword (ex. "platform") to have a None value. Before the regex change they had an empty string value. I'm not really sure which is proper, but for now keeping the prior behavior. --- stem/descriptor/__init__.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 74b538c..fb81237 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -387,6 +387,7 @@ def _get_descriptor_components(raw_contents, validate, extra_keywords): raise ValueError("Line contains invalid characters: %s" % line) keyword, value = line_match.groups() + if value is None: value = '' if not first_keyword: first_keyword = keyword last_keyword = keyword

1 0

[stem/master] Dropping unknown_flags attribute
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 0752e699fcfb657eb27ed97e0b188376cd9368a1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 19 21:47:06 2012 -0700 Dropping unknown_flags attribute I suggested an unknown_flags attribute that would have flags which stem didn't recognize but, on reflection, this wasn't a very good idea. Splitting the attributes mean that users have multiple places to look for the relay's flags. We were also failing validation if there were any unknown flags and that is definitely wrong. New flags are perfectly valid and will happen in the future - those new flags should not break stem's parser. This was probably confusion from when I asked for the cached-consensus test to check for new flags. This is something separate and added it with this change. --- stem/descriptor/networkstatus.py | 11 ----------- test/integ/descriptor/networkstatus.py | 7 +++++++ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index b48c0df..b7a1b2f 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -447,7 +447,6 @@ class RouterStatusEntry(stem.descriptor.Descriptor): :var int dirport: **\*** router's DirPort :var list flags: **\*** list of status flags - :var list unknown_flags: **\*** list of unidentified status flags :var :class:`stem.version.Version`,str version: Version of the Tor protocol this router is running @@ -487,7 +486,6 @@ class RouterStatusEntry(stem.descriptor.Descriptor): self.dirport = None self.flags = [] - self.unknown_flags = [] self.version = None @@ -534,10 +532,6 @@ class RouterStatusEntry(stem.descriptor.Descriptor): # s Named Running Stable Valid #A series of space-separated status flags, in *lexical order* self.flags = line.split(" ") - - self.unknown_flags = filter(lambda f: not f in Flag, self.flags) - if validate and self.unknown_flags: - raise ValueError("Router contained unknown flags: %s", " ".join(self.unknown_flags)) elif peek_check_kw("v"): if "v" in seen_keywords: raise ValueError("Invalid router descriptor: 'v' line appears twice") @@ -647,7 +641,6 @@ class RouterMicrodescriptor(RouterStatusEntry): :var int dirport: **\*** router's DirPort :var list flags: **\*** list of status flags - :var list unknown_flags: **\*** list of unidentified status flags :var :class:`stem.version.Version`,str version: Version of the Tor protocol this router is running @@ -708,10 +701,6 @@ class RouterMicrodescriptor(RouterStatusEntry): # s Named Running Stable Valid #A series of space-separated status flags, in *lexical order* self.flags = line.split(" ") - - self.unknown_flags = filter(lambda f: not f in Flag, self.flags) - if validate and self.unknown_flags: - raise ValueError("Router contained unknown flags: %s", " ".join(self.unknown_flags)) elif peek_check_kw("v"): if "v" in seen_keywords: raise ValueError("Invalid router descriptor: 'v' line appears twice") diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index 621bb14..040a710 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -44,6 +44,13 @@ class TestNetworkStatus(unittest.TestCase): self.fail() assert desc.nickname # check that the router has a nickname count += 1 + + # check if there's any unknown flags + for flag in desc.flags: + if not flag in stem.descriptor.networkstatus.Flag: + # TODO: this should be a special 'new capability' check later + # rather than failing the tests + raise ValueError("Unrecognized flag type: %s, found on relay %s (%s)" % (flag, desc.fingerprint, desc.nickname)) assert count > 100 # sanity check - assuming atleast 100 relays in the Tor network

1 0

[stem/master] Asserting that _decode_fingerprint() output is a fingerprint
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit dd071270d705507f0131bfceac7358f7223ec32d Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 19 21:25:59 2012 -0700 Asserting that _decode_fingerprint() output is a fingerprint Checking that the _decode_fingerprint() function returns a fingerprint, throwing a ValueError if it doesn't. Also adding some tests for this. --- stem/descriptor/networkstatus.py | 6 ++++++ test/unit/descriptor/networkstatus.py | 4 ++++ 2 files changed, 10 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 485234f..d08f316 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -53,6 +53,7 @@ import stem.descriptor import stem.version import stem.exit_policy import stem.util.enum +import stem.util.tor_tools from stem.descriptor import _read_until_keywords, _peek_keyword, _strptime from stem.descriptor import _read_keyword_line, _read_keyword_line_str, _get_pseudo_pgp_block, _peek_line @@ -779,6 +780,8 @@ def _decode_fingerprint(identity): :param str identity: encoded fingerprint from the consensus :returns: str with the uppercase hex encoding of the relay's fingerprint + + :raises: ValueError if the result isn't a valid fingerprint """ # trailing equal signs were stripped from the identity @@ -799,5 +802,8 @@ def _decode_fingerprint(identity): fingerprint += hex(ord(char))[2:].zfill(2).upper() + if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): + raise ValueError("Decoded '%s' to be '%s', which isn't a valid fingerprint" % (identity, fingerprint)) + return fingerprint diff --git a/test/unit/descriptor/networkstatus.py b/test/unit/descriptor/networkstatus.py index ec69367..f14c2be 100644 --- a/test/unit/descriptor/networkstatus.py +++ b/test/unit/descriptor/networkstatus.py @@ -24,4 +24,8 @@ class TestNetworkStatus(unittest.TestCase): for arg, expected in test_values.items(): self.assertEqual(expected, networkstatus._decode_fingerprint(arg)) + + self.assertRaises(ValueError, networkstatus._decode_fingerprint, '') + self.assertRaises(ValueError, networkstatus._decode_fingerprint, '20wYcb') + self.assertRaises(ValueError, networkstatus._decode_fingerprint, '20wYcb' * 30)

1 0

[stem/master] Replacing RouterStatusEntry's identity with fingerprint
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit fd58bc5a565bca11d372188a4add167775f091c1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 19 17:55:25 2012 -0700 Replacing RouterStatusEntry's identity with fingerprint Truncated base64 encoding of a relay's fingerprint? Not useful. Common hex encoded fingerprint that's used everywhere else? Very useful. Decoding the RouterStatusEntry's identity value and adding a unit test for it. Unfortunately the networkstatus module didn't have any unit tests at all so this is gonna take a while to remedy... --- run_tests.py | 4 ++- stem/descriptor/networkstatus.py | 59 ++++++++++++++++++++++++++------ test/integ/descriptor/networkstatus.py | 14 ++++---- test/unit/descriptor/__init__.py | 2 +- test/unit/descriptor/networkstatus.py | 27 ++++++++++++++ 5 files changed, 86 insertions(+), 20 deletions(-) diff --git a/run_tests.py b/run_tests.py index 8d115f1..d2e4da7 100755 --- a/run_tests.py +++ b/run_tests.py @@ -20,6 +20,7 @@ import test.unit.descriptor.export import test.unit.descriptor.reader import test.unit.descriptor.server_descriptor import test.unit.descriptor.extrainfo_descriptor +import test.unit.descriptor.networkstatus import test.unit.response.control_line import test.unit.response.control_message import test.unit.response.getinfo @@ -114,6 +115,7 @@ UNIT_TESTS = ( test.unit.descriptor.reader.TestDescriptorReader, test.unit.descriptor.server_descriptor.TestServerDescriptor, test.unit.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, + test.unit.descriptor.networkstatus.TestNetworkStatus, test.unit.exit_policy.rule.TestExitPolicyRule, test.unit.exit_policy.policy.TestExitPolicy, test.unit.version.TestVersion, @@ -135,7 +137,7 @@ INTEG_TESTS = ( test.integ.descriptor.reader.TestDescriptorReader, test.integ.descriptor.server_descriptor.TestServerDescriptor, test.integ.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, - test.integ.descriptor.networkstatus.TestNetworkStatusDocument, + test.integ.descriptor.networkstatus.TestNetworkStatus, test.integ.version.TestVersion, test.integ.response.protocolinfo.TestProtocolInfo, test.integ.process.TestProcess, diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 7eab8da..485234f 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -41,6 +41,7 @@ The documents can be obtained from any of the following sources... """ import re +import base64 import datetime try: @@ -325,7 +326,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): documents. :var str nickname: directory authority's nickname - :var str identity: uppercase hex fingerprint of the authority's identity key + :var str fingerprint: uppercase hex fingerprint of the authority's identity key :var str address: hostname :var str ip: current IP address :var int dirport: current directory port @@ -352,13 +353,13 @@ class DirectoryAuthority(stem.descriptor.Descriptor): """ super(DirectoryAuthority, self).__init__(raw_content) - self.nickname, self.identity, self.address, self.ip = None, None, None, None + self.nickname, self.fingerprint, self.address, self.ip = None, None, None, None self.dirport, self.orport, self.legacy_dir_key = None, None, None self.key_certificate, self.contact, self.vote_digest = None, None, None content = StringIO(raw_content) dir_source = _read_keyword_line("dir-source", content, validate) - self.nickname, self.identity, self.address, self.ip, self.dirport, self.orport = dir_source.split(" ") + self.nickname, self.fingerprint, self.address, self.ip, self.dirport, self.orport = dir_source.split(" ") self.dirport = int(self.dirport) self.orport = int(self.orport) @@ -431,13 +432,13 @@ class DirectorySignature(stem.descriptor.Descriptor): class RouterStatusEntry(stem.descriptor.Descriptor): """ - Router descriptor object. Parses and stores router information in a router - entry read from a v3 network status document. + Information about an individual router stored within a network status + document. - :var NetworkStatusDocument document: **\*** document this descriptor came from + :var NetworkStatusDocument document: **\*** document that this descriptor came from :var str nickname: **\*** router's nickname - :var str identity: **\*** router's identity + :var str fingerprint: **\*** router's fingerprint :var str digest: **\*** router's digest :var datetime publication: **\*** router's publication :var str ip: **\*** router's IP address @@ -478,7 +479,7 @@ class RouterStatusEntry(stem.descriptor.Descriptor): self.document = document self.nickname = None - self.identity = None + self.fingerprint = None self.digest = None self.publication = None self.ip = None @@ -519,7 +520,7 @@ class RouterStatusEntry(stem.descriptor.Descriptor): if r: seen_keywords.add("r") values = r.split(" ") - self.nickname, self.identity, self.digest = values[0], values[1], values[2] + self.nickname, self.fingerprint, self.digest = values[0], _decode_fingerprint(values[1]), values[2] self.publication = _strptime(" ".join((values[3], values[4])), validate) self.ip, self.orport, self.dirport = values[5], int(values[6]), int(values[7]) if self.dirport == 0: self.dirport = None @@ -640,7 +641,7 @@ class RouterMicrodescriptor(RouterStatusEntry): :var MicrodescriptorConsensus document: **\*** document this descriptor came from :var str nickname: **\*** router's nickname - :var str identity: **\*** router's identity + :var str fingerprint: **\*** router's fingerprint :var datetime publication: **\*** router's publication :var str ip: **\*** router's IP address :var int orport: **\*** router's ORPort @@ -695,7 +696,7 @@ class RouterMicrodescriptor(RouterStatusEntry): if r: seen_keywords.add("r") values = r.split(" ") - self.nickname, self.identity = values[0], values[1] + self.nickname, self.fingerprint = values[0], _decode_fingerprint(values[1]) self.publication = _strptime(" ".join((values[2], values[3])), validate) self.ip, self.orport, self.dirport = values[4], int(values[5]), int(values[6]) if self.dirport == 0: self.dirport = None @@ -764,3 +765,39 @@ class RouterMicrodescriptor(RouterStatusEntry): """ return self.unrecognized_lines + +def _decode_fingerprint(identity): + """ + Decodes the 'identity' value found in consensuses into the more common hex + encoding of the relay's fingerprint. For example... + + :: + + >>> _decode_fingerprint('p1aag7VwarGxqctS7/fS0y5FU+s') + 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB' + + :param str identity: encoded fingerprint from the consensus + + :returns: str with the uppercase hex encoding of the relay's fingerprint + """ + + # trailing equal signs were stripped from the identity + missing_padding = 28 - len(identity) + identity += "=" * missing_padding + + fingerprint = "" + for char in base64.b64decode(identity): + # Individual characters are either standard ascii or hex encoded, and each + # represent two hex digits. For instnace... + # + # >>> ord('\n') + # 10 + # >>> hex(10) + # '0xa' + # >>> '0xa'[2:].zfill(2).upper() + # '0A' + + fingerprint += hex(ord(char))[2:].zfill(2).upper() + + return fingerprint + diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index d16929d..621bb14 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -17,7 +17,7 @@ import test.integ.descriptor def _strptime(string): return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") -class TestNetworkStatusDocument(unittest.TestCase): +class TestNetworkStatus(unittest.TestCase): def test_cached_consensus(self): """ Parses the cached-consensus file in our data directory. @@ -59,7 +59,7 @@ class TestNetworkStatusDocument(unittest.TestCase): router = next(descriptors) self.assertEquals("sumkledi", router.nickname) - self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router.identity) + self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router.fingerprint) self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router.digest) self.assertEquals(_strptime("2012-07-12 04:01:55"), router.publication) self.assertEquals("178.218.213.229", router.ip) @@ -104,7 +104,7 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertEquals(expected_params, desc.params) self.assertEquals("sumkledi", router1.nickname) - self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router1.identity) + self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router1.fingerprint) self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router1.digest) self.assertEquals(_strptime("2012-07-12 04:01:55"), router1.publication) self.assertEquals("178.218.213.229", router1.ip) @@ -114,7 +114,7 @@ class TestNetworkStatusDocument(unittest.TestCase): self.assertEquals(8, len(desc.directory_authorities)) self.assertEquals("tor26", desc.directory_authorities[0].nickname) - self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.directory_authorities[0].identity) + self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.directory_authorities[0].fingerprint) self.assertEquals("86.59.21.38", desc.directory_authorities[0].address) self.assertEquals("86.59.21.38", desc.directory_authorities[0].ip) self.assertEquals(80, desc.directory_authorities[0].dirport) @@ -152,7 +152,7 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= router = next(descriptors) self.assertEquals("sumkledi", router.nickname) - self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router.identity) + self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router.fingerprint) self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router.digest) self.assertEquals(_strptime("2012-07-11 04:22:53"), router.publication) self.assertEquals("178.218.213.229", router.ip) @@ -189,7 +189,7 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals(expected_params, desc.params) self.assertEquals("sumkledi", router1.nickname) - self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router1.identity) + self.assertEquals("0013D22389CD50D0B784A3E4061CB31E8CE8CEB5", router1.fingerprint) self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router1.digest) self.assertEquals(_strptime("2012-07-11 04:22:53"), router1.publication) self.assertEquals("178.218.213.229", router1.ip) @@ -198,7 +198,7 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals(1, len(desc.directory_authorities)) self.assertEquals("turtles", desc.directory_authorities[0].nickname) - self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_authorities[0].identity) + self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_authorities[0].fingerprint) self.assertEquals("76.73.17.194", desc.directory_authorities[0].address) self.assertEquals("76.73.17.194", desc.directory_authorities[0].ip) self.assertEquals(9030, desc.directory_authorities[0].dirport) diff --git a/test/unit/descriptor/__init__.py b/test/unit/descriptor/__init__.py index 1837a07..4793560 100644 --- a/test/unit/descriptor/__init__.py +++ b/test/unit/descriptor/__init__.py @@ -2,5 +2,5 @@ Unit tests for stem.descriptor. """ -__all__ = ["export", "reader", "extrainfo_descriptor", "server_descriptor"] +__all__ = ["export", "reader", "extrainfo_descriptor", "server_descriptor", "networkstatus"] diff --git a/test/unit/descriptor/networkstatus.py b/test/unit/descriptor/networkstatus.py new file mode 100644 index 0000000..ec69367 --- /dev/null +++ b/test/unit/descriptor/networkstatus.py @@ -0,0 +1,27 @@ +""" +Unit tests for stem.descriptor.networkstatus. +""" + +import unittest + +from stem.descriptor import networkstatus + +class TestNetworkStatus(unittest.TestCase): + def test_fingerprint_decoding(self): + """ + Tests for the _decode_fingerprint() helper. + """ + + # consensus identity field and fingerprint for caerSidi and Amunet1-5 + test_values = { + 'p1aag7VwarGxqctS7/fS0y5FU+s': 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB', + 'IbhGa8T+8tyy/MhxCk/qI+EI2LU': '21B8466BC4FEF2DCB2FCC8710A4FEA23E108D8B5', + '20wYcbFGwFfMktmuffYj6Z1RM9k': 'DB4C1871B146C057CC92D9AE7DF623E99D5133D9', + 'nTv9AG1cZeFW2hXiSIEAF6JLRJ4': '9D3BFD006D5C65E156DA15E248810017A24B449E', + '/UKsQiOSGPi/6es0/ha1prNTeDI': 'FD42AC42239218F8BFE9EB34FE16B5A6B3537832', + '/nHdqoKZ6bKZixxAPzYt9Qen+Is': 'FE71DDAA8299E9B2998B1C403F362DF507A7F88B', + } + + for arg, expected in test_values.items(): + self.assertEqual(expected, networkstatus._decode_fingerprint(arg)) +

1 0

[stem/master] Dropping known_flags arg from RouterStatusEntry
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit b1a1674cd5df2b5443699738b6880c6a0077cf0f Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 19 21:31:49 2012 -0700 Dropping known_flags arg from RouterStatusEntry The RouterStatusEntry accepts a known_flags argument and I'm not sure why. It's not used, and the known/unknown flag distinction is simply based on if stem knows about it (and hence is in the Flag enum) so I'm not sure how or why users would utilize it. --- stem/descriptor/networkstatus.py | 20 ++++++++------------ 1 files changed, 8 insertions(+), 12 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index d08f316..b48c0df 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -462,7 +462,7 @@ class RouterStatusEntry(stem.descriptor.Descriptor): | exit_policy appears only in votes """ - def __init__(self, raw_contents, document, validate = True, known_flags = Flag): + def __init__(self, raw_contents, document, validate = True): """ Parse a router descriptor in a v3 network status document and provide a new RouterStatusEntry object. @@ -470,7 +470,6 @@ class RouterStatusEntry(stem.descriptor.Descriptor): :param str raw_content: router descriptor content to be parsed :param NetworkStatusDocument document: document this descriptor came from :param bool validate: whether the router descriptor should be validated - :param bool known_flags: list of known router status flags :raises: ValueError if the descriptor data is invalid """ @@ -499,13 +498,12 @@ class RouterStatusEntry(stem.descriptor.Descriptor): self.microdescriptor_hashes = [] - self._parse(raw_contents, validate, known_flags) + self._parse(raw_contents, validate) - def _parse(self, raw_content, validate, known_flags): + def _parse(self, raw_content, validate): """ :param dict raw_content: iptor contents to be applied :param bool validate: checks the validity of descriptor content if True - :param bool known_flags: list of known router status flags :raises: ValueError if an error occures in validation """ @@ -537,7 +535,7 @@ class RouterStatusEntry(stem.descriptor.Descriptor): #A series of space-separated status flags, in *lexical order* self.flags = line.split(" ") - self.unknown_flags = filter(lambda f: not f in known_flags, self.flags) + self.unknown_flags = filter(lambda f: not f in Flag, self.flags) if validate and self.unknown_flags: raise ValueError("Router contained unknown flags: %s", " ".join(self.unknown_flags)) @@ -661,7 +659,7 @@ class RouterMicrodescriptor(RouterStatusEntry): | **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined """ - def __init__(self, raw_contents, document, validate = True, known_flags = Flag): + def __init__(self, raw_contents, document, validate = True): """ Parse a router descriptor in a v3 microdescriptor consensus and provide a new RouterMicrodescriptor object. @@ -669,20 +667,18 @@ class RouterMicrodescriptor(RouterStatusEntry): :param str raw_content: router descriptor content to be parsed :param MicrodescriptorConsensus document: document this descriptor came from :param bool validate: whether the router descriptor should be validated - :param bool known_flags: list of known router status flags :raises: ValueError if the descriptor data is invalid """ - super(RouterMicrodescriptor, self).__init__(raw_contents, document, validate, known_flags) + super(RouterMicrodescriptor, self).__init__(raw_contents, document, validate) self.document = document - def _parse(self, raw_content, validate, known_flags): + def _parse(self, raw_content, validate): """ :param dict raw_content: router descriptor contents to be parsed :param bool validate: checks the validity of descriptor content if True - :param bool known_flags: list of known router status flags :raises: ValueError if an error occures in validation """ @@ -713,7 +709,7 @@ class RouterMicrodescriptor(RouterStatusEntry): #A series of space-separated status flags, in *lexical order* self.flags = line.split(" ") - self.unknown_flags = filter(lambda f: not f in known_flags, self.flags) + self.unknown_flags = filter(lambda f: not f in Flag, self.flags) if validate and self.unknown_flags: raise ValueError("Router contained unknown flags: %s", " ".join(self.unknown_flags))

1 0

[stem/master] Setting zero ports to None in server descriptors
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit dc15e682be37ea450a9b5e45f05958f676eabded Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Aug 20 22:58:11 2012 -0700 Setting zero ports to None in server descriptors When writing the server descriptor parser I faithfully translated zero ports as zero because I wanted users to be able to differentiate between 'malformed' and 'zero' when validation was disabled. However, on reflection I don't really care about catoring to non-validation use cases. Ravi made the router status entries interpret a zero port as being None which makes perfect sense - zero means that the port doesn't exist. Doing this for the server descriptors too. --- stem/descriptor/server_descriptor.py | 6 +++--- test/integ/descriptor/server_descriptor.py | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 9b8ef0d..6eb3883 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -134,7 +134,7 @@ class ServerDescriptor(stem.descriptor.Descriptor): :var str address: **\*** IPv4 address of the relay :var int or_port: **\*** port used for relaying - :var int socks_port: **\*** port used as client (deprecated, always zero) + :var int socks_port: **\*** port used as client (deprecated, always None) :var int dir_port: **\*** port used for descriptor mirroring :var str platform: line with operating system and tor version @@ -340,8 +340,8 @@ class ServerDescriptor(stem.descriptor.Descriptor): self.nickname = router_comp[0] self.address = router_comp[1] self.or_port = int(router_comp[2]) - self.socks_port = int(router_comp[3]) - self.dir_port = int(router_comp[4]) + self.socks_port = None if router_comp[3] == '0' else int(router_comp[3]) + self.dir_port = None if router_comp[4] == '0' else int(router_comp[4]) elif keyword == "bandwidth": # "bandwidth" bandwidth-avg bandwidth-burst bandwidth-observed bandwidth_comp = value.split() diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 38d633c..55e6545 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -62,8 +62,8 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", desc.fingerprint) self.assertEquals("71.35.133.197", desc.address) self.assertEquals(9001, desc.or_port) - self.assertEquals(0, desc.socks_port) - self.assertEquals(0, desc.dir_port) + self.assertEquals(None, desc.socks_port) + self.assertEquals(None, desc.dir_port) self.assertEquals("Tor 0.2.1.30 on Linux x86_64", desc.platform) self.assertEquals(stem.version.Version("0.2.1.30"), desc.tor_version) self.assertEquals("Linux x86_64", desc.operating_system) @@ -105,8 +105,8 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals("3E2F63E2356F52318B536A12B6445373808A5D6C", desc.fingerprint) self.assertEquals("212.37.39.59", desc.address) self.assertEquals(8000, desc.or_port) - self.assertEquals(0, desc.socks_port) - self.assertEquals(0, desc.dir_port) + self.assertEquals(None, desc.socks_port) + self.assertEquals(None, desc.dir_port) self.assertEquals("Tor 0.1.0.14 on FreeBSD i386", desc.platform) self.assertEquals(stem.version.Version("0.1.0.14"), desc.tor_version) self.assertEquals("FreeBSD i386", desc.operating_system) @@ -161,7 +161,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(None, desc.read_history_end) self.assertEquals(None, desc.write_history_end) self.assertEquals(None, desc.eventdns) - self.assertEquals(0, desc.socks_port) + self.assertEquals(None, desc.socks_port) unrecognized_lines = desc.get_unrecognized_lines() @@ -193,8 +193,8 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals("FEBC7F992AC418BBE42BC13FE94EFCFE6549197E", desc.fingerprint) self.assertEquals("130.243.230.116", desc.address) self.assertEquals(9001, desc.or_port) - self.assertEquals(0, desc.socks_port) - self.assertEquals(0, desc.dir_port) + self.assertEquals(None, desc.socks_port) + self.assertEquals(None, desc.dir_port) self.assertEquals("Tor 0.2.2.35 (git-4f42b0a93422f70e) on Linux x86_64", desc.platform) self.assertEquals(stem.version.Version("0.2.2.35"), desc.tor_version) self.assertEquals("Linux x86_64", desc.operating_system) @@ -291,8 +291,8 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals("AE54E28ED069CDF45F3009F963EE3B3D6FA26A2E", desc.fingerprint) self.assertEquals("10.45.227.253", desc.address) self.assertEquals(9001, desc.or_port) - self.assertEquals(0, desc.socks_port) - self.assertEquals(0, desc.dir_port) + self.assertEquals(None, desc.socks_port) + self.assertEquals(None, desc.dir_port) self.assertEquals("Tor 0.2.3.12-alpha (git-800942b4176ca31c) on Linux x86_64", desc.platform) self.assertEquals(stem.version.Version("0.2.3.12-alpha"), desc.tor_version) self.assertEquals("Linux x86_64", desc.operating_system)

1 0