October 2012 - tor-commits - lists.torproject.org

[stem/master] stem.descriptor.networkstatus_descriptor => stem.descriptor.networkstatus
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 4e29ae8dfc69cb68a25c9cf159e56db4e943565b Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Tue Aug 7 12:09:25 2012 +0530 stem.descriptor.networkstatus_descriptor => stem.descriptor.networkstatus --- run_tests.py | 4 +- stem/descriptor/__init__.py | 2 +- stem/descriptor/networkstatus.py | 614 +++++++++++++++++++++ stem/descriptor/networkstatus_descriptor.py | 614 --------------------- test/integ/descriptor/networkstatus.py | 228 ++++++++ test/integ/descriptor/networkstatus_descriptor.py | 228 -------- 6 files changed, 845 insertions(+), 845 deletions(-) diff --git a/run_tests.py b/run_tests.py index 10aa24c..8d115f1 100755 --- a/run_tests.py +++ b/run_tests.py @@ -46,7 +46,7 @@ import test.integ.socket.control_socket import test.integ.descriptor.reader import test.integ.descriptor.server_descriptor import test.integ.descriptor.extrainfo_descriptor -import test.integ.descriptor.networkstatus_descriptor +import test.integ.descriptor.networkstatus import test.integ.response.protocolinfo import test.integ.util.conf import test.integ.util.proc @@ -135,7 +135,7 @@ INTEG_TESTS = ( test.integ.descriptor.reader.TestDescriptorReader, test.integ.descriptor.server_descriptor.TestServerDescriptor, test.integ.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, - test.integ.descriptor.networkstatus_descriptor.TestNetworkStatusDocument, + test.integ.descriptor.networkstatus.TestNetworkStatusDocument, test.integ.version.TestVersion, test.integ.response.protocolinfo.TestProtocolInfo, test.integ.process.TestProcess, diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 5afe214..903a877 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -17,7 +17,7 @@ __all__ = [ "reader", "extrainfo_descriptor", "server_descriptor", - "networkstatus_descriptor", + "networkstatus", "parse_file", "Descriptor", ] diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py new file mode 100644 index 0000000..8daf6f7 --- /dev/null +++ b/stem/descriptor/networkstatus.py @@ -0,0 +1,614 @@ +""" +Parsing for Tor network status documents. Currently supports parsing v3 network +status documents (both votes and consensus'). + +The network status documents also contain a list of router descriptors, +directory authorities, signatures etc. + +The votes and consensus' can be obtained from any of the following sources... + +* the 'cached-consensus' file in tor's data directory +* tor metrics, at https://metrics.torproject.org/data.html +* directory authorities and mirrors via their DirPort + +**Module Overview:** + +:: + + parse_file - parses a network status file and provides a NetworkStatusDocument + NetworkStatusDocument - Tor v3 network status document + +- MicrodescriptorConsensus - Tor microdescriptor consensus document + RouterDescriptor - Router descriptor; contains information about a Tor relay + +- RouterMicrodescriptor - Router microdescriptor; contains information that doesn't change often + DirectorySignature + DirectoryAuthority +""" + +import re +import base64 +import hashlib +import datetime + +import stem.prereq +import stem.descriptor +import stem.descriptor.extrainfo_descriptor +import stem.version +import stem.exit_policy +import stem.util.log as log +import stem.util.connection +import stem.util.tor_tools + +_bandwidth_weights_regex = re.compile(" ".join(["W%s=\d+" % weight for weight in ["bd", + "be", "bg", "bm", "db", "eb", "ed", "ee", "eg", "em", "gb", "gd", "gg", "gm", "mb", "md", "me", "mg", "mm"]])) + +def parse_file(document_file, validate = True): + """ + Iterates over the router descriptors in a network status document. + + :param file document_file: file with network status document content + :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise + + :returns: iterator for :class:`stem.descriptor.networkstatus_descriptor.RouterDescriptor` instances in the file + + :raises: + * ValueError if the contents is malformed and validate is True + * IOError if the file can't be read + """ + + document = NetworkStatusDocument(document_file.read(), validate) + return document.router_descriptors + +def _strptime(string, validate = True, optional = False): + try: + return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") + except ValueError, exc: + if validate or not optional: raise exc + +class NetworkStatusDocument(stem.descriptor.Descriptor): + """ + A v3 network status document. + + This could be a v3 consensus or vote document. + + :var bool validated: **\*** whether the document is validated + :var str network_status_version: **\*** a document format version. For v3 documents this is "3" + :var str vote_status: **\*** status of the vote (is either "vote" or "consensus") + :var list consensus_methods: A list of supported consensus generation methods (integers) + :var datetime published: time when the document was published + :var int consensus_method: consensus method used to generate a consensus + :var datetime valid_after: **\*** time when the consensus becomes valid + :var datetime fresh_until: **\*** time until when the consensus is considered to be fresh + :var datetime valid_until: **\*** time until when the consensus is valid + :var int vote_delay: **\*** number of seconds allowed for collecting votes from all authorities + :var int dist_delay: number of seconds allowed for collecting signatures from all authorities + :var list client_versions: list of recommended Tor client versions + :var list server_versions: list of recommended Tor server versions + :var list known_flags: **\*** list of known router flags + :var list params: dict of parameter(str) => value(int) mappings + :var list router_descriptors: **\*** list of RouterDescriptor objects defined in the document + :var list directory_authorities: **\*** list of DirectoryAuthority objects that have generated this document + :var dict bandwidth_weights: dict of weight(str) => value(int) mappings + :var list directory_signatures: **\*** list of signatures this document has + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, raw_content, validate = True): + """ + Parse a v3 network status document and provide a new NetworkStatusDocument object. + + :param str raw_content: raw network status document data + :param bool validate: True if the document is to be validated, False otherwise + + :raises: ValueError if the document is invalid + """ + + super(NetworkStatusDocument, self).__init__(raw_content) + + self.router_descriptors = [] + self.directory_authorities = [] + self.directory_signatures = [] + self.validated = validate + + self.network_status_version = None + self.vote_status = None + self.consensus_methods = [] + self.published = None + self.consensus_method = None + self.valid_after = None + self.fresh_until = None + self.valid_until = None + self.vote_delay = None + self.dist_delay = None + self.client_versions = [] + self.server_versions = [] + self.known_flags = [] + self.params = {} + self.bandwidth_weights = {} + + self._parse(raw_content) + + def _generate_router(self, raw_content, vote, validate): + return RouterDescriptor(raw_content, vote, validate) + + def _validate_network_status_version(self): + return self.network_status_version == "3" + + def get_unrecognized_lines(self): + """ + Returns any unrecognized trailing lines. + + :returns: a list of unrecognized trailing lines + """ + + return self._unrecognized_lines + + def _parse(self, raw_content): + # preamble + validate = self.validated + doc_parser = stem.descriptor.DescriptorParser(raw_content, validate) + + read_keyword_line = lambda keyword, optional = False: setattr(self, keyword.replace("-", "_"), doc_parser.read_keyword_line(keyword, optional)) + + map(read_keyword_line, ["network-status-version", "vote-status"]) + if validate and not self._validate_network_status_version(): + raise ValueError("Invalid network-status-version: %s" % self.network_status_version) + + if self.vote_status == "vote": vote = True + elif self.vote_status == "consensus": vote = False + elif validate: raise ValueError("Unrecognized document type specified in vote-status") + + if vote: + read_keyword_line("consensus-methods", True) + self.consensus_methods = [int(method) for method in self.consensus_methods.split(" ")] + self.published = _strptime(doc_parser.read_keyword_line("published", True), validate, True) + else: + self.consensus_method = int(doc_parser.read_keyword_line("consensus-method", True)) + + map(read_keyword_line, ["valid-after", "fresh-until", "valid-until"]) + self.valid_after = _strptime(self.valid_after, validate) + self.fresh_until = _strptime(self.fresh_until, validate) + self.valid_until = _strptime(self.valid_until, validate) + voting_delay = doc_parser.read_keyword_line("voting-delay") + self.vote_delay, self.dist_delay = [int(delay) for delay in voting_delay.split(" ")] + + client_versions = doc_parser.read_keyword_line("client-versions", True) + if client_versions: + self.client_versions = [stem.version.Version(version_string) for version_string in client_versions.split(",")] + server_versions = doc_parser.read_keyword_line("server-versions", True) + if server_versions: + self.server_versions = [stem.version.Version(version_string) for version_string in server_versions.split(",")] + self.known_flags = doc_parser.read_keyword_line("known-flags").split(" ") + read_keyword_line("params", True) + if self.params: + self.params = dict([(param.split("=")[0], int(param.split("=")[1])) for param in self.params.split(" ")]) + + # authority section + while doc_parser.line.startswith("dir-source "): + dirauth_data = doc_parser.read_until(["dir-source", "r"]) + self.directory_authorities.append(DirectoryAuthority(dirauth_data, vote, validate)) + + def _router_desc_generator(raw_content, vote, validate): + parser = stem.descriptor.DescriptorParser(raw_content, validate) + while parser.line != None: + descriptor = parser.read_until("r") + yield self._generate_router(descriptor, vote, validate) + + # router descriptors + if doc_parser.peek_keyword() == "r": + router_descriptors_data = doc_parser.read_until(["bandwidth-weights", "directory-footer", "directory-signature"]) + self.router_descriptors = _router_desc_generator(router_descriptors_data, vote, validate) + elif validate: + raise ValueError("No router descriptors found") + + # footer section + if self.consensus_method > 9 or vote and filter(lambda x: x >= 9, self.consensus_methods): + if doc_parser.line == "directory-footer": + doc_parser.read_line() + elif validate: + raise ValueError("Network status document missing directory-footer") + + if not vote: + read_keyword_line("bandwidth-weights", True) + if _bandwidth_weights_regex.match(self.bandwidth_weights): + self.bandwidth_weights = dict([(weight.split("=")[0], int(weight.split("=")[1])) for weight in self.bandwidth_weights.split(" ")]) + elif validate: + raise ValueError("Invalid bandwidth-weights line") + + while doc_parser.line.startswith("directory-signature "): + signature_data = doc_parser.read_until(["directory-signature"]) + self.directory_signatures.append(DirectorySignature(signature_data)) + + self._unrecognized_lines = doc_parser.remaining() + if validate and self._unrecognized_lines: raise ValueError("Unrecognized trailing data") + +class DirectoryAuthority(stem.descriptor.Descriptor): + """ + Contains directory authority information obtained from v3 network status + documents. + + :var str nickname: directory authority's nickname + :var str identity: uppercase hex fingerprint of the authority's identity key + :var str address: hostname + :var str ip: current IP address + :var int dirport: current directory port + :var int orport: current orport + :var str contact: directory authority's contact information + :var str legacy_dir_key: fingerprint of and obsolete identity key + :var :class:`stem.descriptor.networkstatus_descriptor.KeyCertificate` key_certificate: directory authority's current key certificate + :var str vote_digest: digest of the authority that contributed to the consensus + """ + + def __init__(self, raw_content, vote = True, validate = True): + """ + Parse a directory authority entry in a v3 network status document and + provide a DirectoryAuthority object. + + :param str raw_content: raw directory authority entry information + :param bool validate: True if the document is to be validated, False otherwise + + :raises: ValueError if the raw data is invalid + """ + + super(DirectoryAuthority, self).__init__(raw_content) + self.nickname, self.identity, self.address, self.ip = None, None, None, None + self.dirport, self.orport, self.legacy_dir_key = None, None, None + self.key_certificate, self.contact, self.vote_digest = None, None, None + parser = stem.descriptor.DescriptorParser(raw_content, validate) + + dir_source = parser.read_keyword_line("dir-source") + self.nickname, self.identity, self.address, self.ip, self.dirport, self.orport = dir_source.split(" ") + self.dirport = int(self.dirport) + self.orport = int(self.orport) + + self.contact = parser.read_keyword_line("contact") + if vote: + self.legacy_dir_key = parser.read_keyword_line("legacy-dir-key", True) + self.key_certificate = KeyCertificate("\n".join(parser.remaining()), validate) + else: + self.vote_digest = parser.read_keyword_line("vote-digest", True) + rmng = parser.remaining() + if rmng and validate: + raise ValueError("Unrecognized trailing data in directory authority information") + +class KeyCertificate(stem.descriptor.Descriptor): + """ + Directory key certificate. + + :var str key_certificate_version: **\*** version of the key certificate (Should be "3") + :var str ip: IP address on which the directory authority is listening + :var int port: port on which the directory authority is listening + :var str fingerprint: **\*** hex encoded fingerprint of the authority's identity key + :var str identity_key: **\*** long term authority identity key + :var datetime published: **\*** time (in GMT) when this document & the key were last generated + :var str expires: **\*** time (in GMT) after which this key becomes invalid + :var str signing_key: **\*** directory server's public signing key + :var str crosscert: signature made using certificate's signing key + :var str certification: **\*** signature of this key certificate signed with the identity key + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, raw_content, validate = True): + """ + Parse a key certificate entry and provide a KeyCertificate object. + + :param str raw_content: raw key certificate information + :param bool validate: True if the document is to be validated, False otherwise + + :raises: ValueError if the raw data is invalid + """ + + super(KeyCertificate, self).__init__(raw_content) + self.key_certificate_version, self.ip, self.port = None, None, None + self.fingerprint, self.identity_key, self.published = None, None, None + self.expires, self.signing_key, self.crosscert = None, None, None + self.certification = None + parser = stem.descriptor.DescriptorParser(raw_content, validate) + peek_check_kw = lambda keyword: keyword == parser.peek_keyword() + seen_keywords = set() + + self.key_certificate_version = parser.read_keyword_line("dir-key-certificate-version") + if validate and self.key_certificate_version != "3": raise ValueError("Unrecognized dir-key-certificate-version") + + def _read_keyword_line(keyword): + if validate and keyword in seen_keywords: + raise ValueError("Invalid key certificate: '%s' appears twice" % keyword) + seen_keywords.add(keyword) + return parser.read_keyword_line(keyword) + + while parser.line: + if peek_check_kw("dir-address"): + line = _read_keyword_line("dir-address") + try: + self.ip, self.port = line.rsplit(":", 1) + self.port = int(self.port) + except Exception: + if validate: raise ValueError("Invalid dir-address line: %s" % line) + + elif peek_check_kw("fingerprint"): + self.fingerprint = _read_keyword_line("fingerprint") + + elif peek_check_kw("dir-identity-key"): + _read_keyword_line("dir-identity-key") + self.identity_key = parser.read_block("RSA PUBLIC KEY") + + elif peek_check_kw("dir-key-published"): + self.published = _strptime(_read_keyword_line("dir-key-published")) + + elif peek_check_kw("dir-key-expires"): + self.expires = _strptime(_read_keyword_line("dir-key-expires")) + + elif peek_check_kw("dir-signing-key"): + _read_keyword_line("dir-signing-key") + self.signing_key = parser.read_block("RSA PUBLIC KEY") + + elif peek_check_kw("dir-key-crosscert"): + _read_keyword_line("dir-key-crosscert") + self.crosscert = parser.read_block("ID SIGNATURE") + + elif peek_check_kw("dir-key-certification"): + _read_keyword_line("dir-key-certification") + self.certification = parser.read_block("SIGNATURE") + break + + elif validate: + raise ValueError("Key certificate contains unrecognized lines: %s" % parser.line) + + else: + # ignore unrecognized lines if we aren't validating + self._unrecognized_lines.append(parser.read_line()) + + self._unrecognized_lines = parser.remaining() + if self._unrecognized_lines and validate: + raise ValueError("Unrecognized trailing data in key certificate") + + def get_unrecognized_lines(self): + """ + Returns any unrecognized lines. + + :returns: a list of unrecognized lines + """ + + return self._unrecognized_lines + +class DirectorySignature(stem.descriptor.Descriptor): + """ + Contains directory signature information described in a v3 network status + document. + + :var str identity: signature identity + :var str key_digest: signature key digest + :var str method: method used to generate the signature + :var str signature: the signature data + """ + + def __init__(self, raw_content, validate = True): + """ + Parse a directory signature entry in a v3 network status document and + provide a DirectorySignature object. + + :param str raw_content: raw directory signature entry information + :param bool validate: True if the document is to be validated, False otherwise + + :raises: ValueError if the raw data is invalid + """ + + super(DirectorySignature, self).__init__(raw_content) + self.identity, self.key_digest, self.method, self.signature = None, None, None, None + parser = stem.descriptor.DescriptorParser(raw_content, validate) + + signature_line = parser.read_keyword_line("directory-signature").split(" ") + + if len(signature_line) == 2: + self.identity, self.key_digest = signature_line + if len(signature_line) == 3: # for microdescriptor consensuses + self.method, self.identity, self.key_digest = signature_line + + self.signature = parser.read_block("SIGNATURE") + self._unrecognized_lines = parser.remaining() + if self._unrecognized_lines and validate: + raise ValueError("Unrecognized trailing data in directory signature") + +class RouterDescriptor(stem.descriptor.Descriptor): + """ + Router descriptor object. Parses and stores router information in a router + entry read from a v3 network status document. + + :var str nickname: **\*** router's nickname + :var str identity: **\*** router's identity + :var str digest: **\*** router's digest + :var datetime publication: **\*** router's publication + :var str ip: **\*** router's IP address + :var int orport: **\*** router's ORPort + :var int dirport: **\*** router's DirPort + + :var bool is_valid: **\*** router is valid + :var bool is_guard: **\*** router is suitable for use as an entry guard + :var bool is_named: **\*** router is named + :var bool is_unnamed: **\*** router is unnamed + :var bool is_running: **\*** router is running and currently usable + :var bool is_stable: **\*** router is stable, i.e., it's suitable for for long-lived circuits + :var bool is_exit: **\*** router is an exit router + :var bool is_fast: **\*** router is Fast, i.e., it's usable for high-bandwidth circuits + :var bool is_authority: **\*** router is a directory authority + :var bool supports_v2dir: **\*** router supports v2dir + :var bool supports_v3dir: **\*** router supports v3dir + :var bool is_hsdir: **\*** router is a hidden status + :var bool is_badexit: **\*** router is marked a bad exit + :var bool is_baddirectory: **\*** router is a bad directory + + :var :class:`stem.version.Version`,str version: Version of the Tor protocol this router is running + + :var int bandwidth: router's claimed bandwidth + :var int measured_bandwidth: router's measured bandwidth + + :var :class:`stem.exit_policy.MicrodescriptorExitPolicy` exitpolicy: router's exitpolicy + + :var str microdescriptor_hashes: a list of two-tuples with a list of consensus methods(int) that may produce the digest and a dict with algorithm(str) => digest(str) mappings. algorithm is the hashing algorithm (usually "sha256") that is used to produce digest (the base64 encoding of the hash of the router's microdescriptor with trailing =s omitted). + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, raw_contents, vote = True, validate = True): + """ + Parse a router descriptor in a v3 network status document and provide a new + RouterDescriptor object. + + :param str raw_content: router descriptor content to be parsed + :param bool validate: whether the router descriptor should be validated + """ + + super(RouterDescriptor, self).__init__(raw_contents) + + self.nickname = None + self.identity = None + self.digest = None + self.publication = None + self.ip = None + self.orport = None + self.dirport = None + + self.is_valid = False + self.is_guard = False + self.is_named = False + self.is_unnamed = False + self.is_running = False + self.is_stable = False + self.is_exit = False + self.is_fast = False + self.is_authority = False + self.supports_v2dir = False + self.supports_v3dir = False + self.is_hsdir = False + self.is_badexit = False + self.is_baddirectory = False + + self.version = None + + self.bandwidth = None + self.measured_bandwidth = None + + self.exit_policy = None + + self.microdescriptor_hashes = [] + + self._parse(raw_contents, vote, validate) + + def _parse(self, raw_content, vote, validate): + """ + :param dict raw_content: iptor contents to be applied + :param bool validate: checks the validity of descriptor content if True + + :raises: ValueError if an error occures in validation + """ + + parser = stem.descriptor.DescriptorParser(raw_content, validate) + seen_keywords = set() + peek_check_kw = lambda keyword: keyword == parser.peek_keyword() + + r = parser.read_keyword_line("r") + # r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0 + # "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort SP DirPort NL + seen_keywords.add("r") + if r: + values = r.split(" ") + self.nickname, self.identity, self.digest = values[0], values[1], values[2] + self.publication = _strptime(" ".join((values[3], values[4])), validate) + self.ip, self.orport, self.dirport = values[5], int(values[6]), int(values[7]) + if self.dirport == 0: self.dirport = None + elif validate: raise ValueError("Invalid router descriptor: empty 'r' line" ) + + while parser.line: + if peek_check_kw("s"): + if "s" in seen_keywords: raise ValueError("Invalid router descriptor: 's' line appears twice") + line = parser.read_keyword_line("s") + if not line: continue + seen_keywords.add("s") + # s Named Running Stable Valid + #A series of space-separated status flags, in *lexical order* + flags = line.split(" ") + flag_map = { + "Valid": "is_valid", + "Guard": "is_guard", + "Named": "is_named", + "Unnamed": "is_unnamed", + "Running": "is_running", + "Stable": "is_stable", + "Exit": "is_exit", + "Fast": "is_fast", + "Authority": "is_authority", + "V2Dir": "supports_v2dir", + "V3Dir": "supports_v3dir", + "HSDir": "is_hsdir", + "BadExit": "is_badexit", + "BadDirectory": "is_baddirectory", + } + map(lambda flag: setattr(self, flag_map[flag], True), flags) + + if self.is_unnamed: self.is_named = False + elif self.is_named: self.is_unnamed = False + + elif peek_check_kw("v"): + if "v" in seen_keywords: raise ValueError("Invalid router descriptor: 'v' line appears twice") + line = parser.read_keyword_line("v", True) + seen_keywords.add("v") + # v Tor 0.2.2.35 + if line: + if line.startswith("Tor "): + self.version = stem.version.Version(line[4:]) + else: + self.version = line + elif validate: raise ValueError("Invalid router descriptor: empty 'v' line" ) + + elif peek_check_kw("w"): + if "w" in seen_keywords: raise ValueError("Invalid router descriptor: 'w' line appears twice") + w = parser.read_keyword_line("w", True) + # "w" SP "Bandwidth=" INT [SP "Measured=" INT] NL + seen_keywords.add("w") + if w: + values = w.split(" ") + if len(values) <= 2 and len(values) > 0: + key, value = values[0].split("=") + if key == "Bandwidth": self.bandwidth = int(value) + elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Bandwidth, read " + key) + + if len(values) == 2: + key, value = values[1].split("=") + if key == "Measured": self.measured_bandwidth = int(value) + elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Measured, read " + key) + elif validate: raise ValueError("Router descriptor contains invalid 'w' line") + elif validate: raise ValueError("Router descriptor contains empty 'w' line") + + elif peek_check_kw("p"): + if "p" in seen_keywords: raise ValueError("Invalid router descriptor: 'p' line appears twice") + p = parser.read_keyword_line("p", True) + seen_keywords.add("p") + # "p" SP ("accept" / "reject") SP PortList NL + if p: + self.exit_policy = stem.exit_policy.MicrodescriptorExitPolicy(p) + #self.exit_policy = p + + elif vote and peek_check_kw("m"): + # microdescriptor hashes + m = parser.read_keyword_line("m", True) + methods, digests = m.split(" ", 1) + method_list = methods.split(",") + digest_dict = [digest.split("=", 1) for digest in digests.split(" ")] + self.microdescriptor_hashes.append((method_list, digest_dict)) + + elif validate: + raise ValueError("Router descriptor contains unrecognized trailing lines: %s" % parser.line) + + else: + self._unrecognized_lines.append(parser.read_line()) # ignore unrecognized lines if we aren't validating + + def get_unrecognized_lines(self): + """ + Returns any unrecognized lines. + + :returns: a list of unrecognized lines + """ + + return self._unrecognized_lines + diff --git a/stem/descriptor/networkstatus_descriptor.py b/stem/descriptor/networkstatus_descriptor.py deleted file mode 100644 index 8daf6f7..0000000 --- a/stem/descriptor/networkstatus_descriptor.py +++ /dev/null @@ -1,614 +0,0 @@ -""" -Parsing for Tor network status documents. Currently supports parsing v3 network -status documents (both votes and consensus'). - -The network status documents also contain a list of router descriptors, -directory authorities, signatures etc. - -The votes and consensus' can be obtained from any of the following sources... - -* the 'cached-consensus' file in tor's data directory -* tor metrics, at https://metrics.torproject.org/data.html -* directory authorities and mirrors via their DirPort - -**Module Overview:** - -:: - - parse_file - parses a network status file and provides a NetworkStatusDocument - NetworkStatusDocument - Tor v3 network status document - +- MicrodescriptorConsensus - Tor microdescriptor consensus document - RouterDescriptor - Router descriptor; contains information about a Tor relay - +- RouterMicrodescriptor - Router microdescriptor; contains information that doesn't change often - DirectorySignature - DirectoryAuthority -""" - -import re -import base64 -import hashlib -import datetime - -import stem.prereq -import stem.descriptor -import stem.descriptor.extrainfo_descriptor -import stem.version -import stem.exit_policy -import stem.util.log as log -import stem.util.connection -import stem.util.tor_tools - -_bandwidth_weights_regex = re.compile(" ".join(["W%s=\d+" % weight for weight in ["bd", - "be", "bg", "bm", "db", "eb", "ed", "ee", "eg", "em", "gb", "gd", "gg", "gm", "mb", "md", "me", "mg", "mm"]])) - -def parse_file(document_file, validate = True): - """ - Iterates over the router descriptors in a network status document. - - :param file document_file: file with network status document content - :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise - - :returns: iterator for :class:`stem.descriptor.networkstatus_descriptor.RouterDescriptor` instances in the file - - :raises: - * ValueError if the contents is malformed and validate is True - * IOError if the file can't be read - """ - - document = NetworkStatusDocument(document_file.read(), validate) - return document.router_descriptors - -def _strptime(string, validate = True, optional = False): - try: - return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") - except ValueError, exc: - if validate or not optional: raise exc - -class NetworkStatusDocument(stem.descriptor.Descriptor): - """ - A v3 network status document. - - This could be a v3 consensus or vote document. - - :var bool validated: **\*** whether the document is validated - :var str network_status_version: **\*** a document format version. For v3 documents this is "3" - :var str vote_status: **\*** status of the vote (is either "vote" or "consensus") - :var list consensus_methods: A list of supported consensus generation methods (integers) - :var datetime published: time when the document was published - :var int consensus_method: consensus method used to generate a consensus - :var datetime valid_after: **\*** time when the consensus becomes valid - :var datetime fresh_until: **\*** time until when the consensus is considered to be fresh - :var datetime valid_until: **\*** time until when the consensus is valid - :var int vote_delay: **\*** number of seconds allowed for collecting votes from all authorities - :var int dist_delay: number of seconds allowed for collecting signatures from all authorities - :var list client_versions: list of recommended Tor client versions - :var list server_versions: list of recommended Tor server versions - :var list known_flags: **\*** list of known router flags - :var list params: dict of parameter(str) => value(int) mappings - :var list router_descriptors: **\*** list of RouterDescriptor objects defined in the document - :var list directory_authorities: **\*** list of DirectoryAuthority objects that have generated this document - :var dict bandwidth_weights: dict of weight(str) => value(int) mappings - :var list directory_signatures: **\*** list of signatures this document has - - **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined - """ - - def __init__(self, raw_content, validate = True): - """ - Parse a v3 network status document and provide a new NetworkStatusDocument object. - - :param str raw_content: raw network status document data - :param bool validate: True if the document is to be validated, False otherwise - - :raises: ValueError if the document is invalid - """ - - super(NetworkStatusDocument, self).__init__(raw_content) - - self.router_descriptors = [] - self.directory_authorities = [] - self.directory_signatures = [] - self.validated = validate - - self.network_status_version = None - self.vote_status = None - self.consensus_methods = [] - self.published = None - self.consensus_method = None - self.valid_after = None - self.fresh_until = None - self.valid_until = None - self.vote_delay = None - self.dist_delay = None - self.client_versions = [] - self.server_versions = [] - self.known_flags = [] - self.params = {} - self.bandwidth_weights = {} - - self._parse(raw_content) - - def _generate_router(self, raw_content, vote, validate): - return RouterDescriptor(raw_content, vote, validate) - - def _validate_network_status_version(self): - return self.network_status_version == "3" - - def get_unrecognized_lines(self): - """ - Returns any unrecognized trailing lines. - - :returns: a list of unrecognized trailing lines - """ - - return self._unrecognized_lines - - def _parse(self, raw_content): - # preamble - validate = self.validated - doc_parser = stem.descriptor.DescriptorParser(raw_content, validate) - - read_keyword_line = lambda keyword, optional = False: setattr(self, keyword.replace("-", "_"), doc_parser.read_keyword_line(keyword, optional)) - - map(read_keyword_line, ["network-status-version", "vote-status"]) - if validate and not self._validate_network_status_version(): - raise ValueError("Invalid network-status-version: %s" % self.network_status_version) - - if self.vote_status == "vote": vote = True - elif self.vote_status == "consensus": vote = False - elif validate: raise ValueError("Unrecognized document type specified in vote-status") - - if vote: - read_keyword_line("consensus-methods", True) - self.consensus_methods = [int(method) for method in self.consensus_methods.split(" ")] - self.published = _strptime(doc_parser.read_keyword_line("published", True), validate, True) - else: - self.consensus_method = int(doc_parser.read_keyword_line("consensus-method", True)) - - map(read_keyword_line, ["valid-after", "fresh-until", "valid-until"]) - self.valid_after = _strptime(self.valid_after, validate) - self.fresh_until = _strptime(self.fresh_until, validate) - self.valid_until = _strptime(self.valid_until, validate) - voting_delay = doc_parser.read_keyword_line("voting-delay") - self.vote_delay, self.dist_delay = [int(delay) for delay in voting_delay.split(" ")] - - client_versions = doc_parser.read_keyword_line("client-versions", True) - if client_versions: - self.client_versions = [stem.version.Version(version_string) for version_string in client_versions.split(",")] - server_versions = doc_parser.read_keyword_line("server-versions", True) - if server_versions: - self.server_versions = [stem.version.Version(version_string) for version_string in server_versions.split(",")] - self.known_flags = doc_parser.read_keyword_line("known-flags").split(" ") - read_keyword_line("params", True) - if self.params: - self.params = dict([(param.split("=")[0], int(param.split("=")[1])) for param in self.params.split(" ")]) - - # authority section - while doc_parser.line.startswith("dir-source "): - dirauth_data = doc_parser.read_until(["dir-source", "r"]) - self.directory_authorities.append(DirectoryAuthority(dirauth_data, vote, validate)) - - def _router_desc_generator(raw_content, vote, validate): - parser = stem.descriptor.DescriptorParser(raw_content, validate) - while parser.line != None: - descriptor = parser.read_until("r") - yield self._generate_router(descriptor, vote, validate) - - # router descriptors - if doc_parser.peek_keyword() == "r": - router_descriptors_data = doc_parser.read_until(["bandwidth-weights", "directory-footer", "directory-signature"]) - self.router_descriptors = _router_desc_generator(router_descriptors_data, vote, validate) - elif validate: - raise ValueError("No router descriptors found") - - # footer section - if self.consensus_method > 9 or vote and filter(lambda x: x >= 9, self.consensus_methods): - if doc_parser.line == "directory-footer": - doc_parser.read_line() - elif validate: - raise ValueError("Network status document missing directory-footer") - - if not vote: - read_keyword_line("bandwidth-weights", True) - if _bandwidth_weights_regex.match(self.bandwidth_weights): - self.bandwidth_weights = dict([(weight.split("=")[0], int(weight.split("=")[1])) for weight in self.bandwidth_weights.split(" ")]) - elif validate: - raise ValueError("Invalid bandwidth-weights line") - - while doc_parser.line.startswith("directory-signature "): - signature_data = doc_parser.read_until(["directory-signature"]) - self.directory_signatures.append(DirectorySignature(signature_data)) - - self._unrecognized_lines = doc_parser.remaining() - if validate and self._unrecognized_lines: raise ValueError("Unrecognized trailing data") - -class DirectoryAuthority(stem.descriptor.Descriptor): - """ - Contains directory authority information obtained from v3 network status - documents. - - :var str nickname: directory authority's nickname - :var str identity: uppercase hex fingerprint of the authority's identity key - :var str address: hostname - :var str ip: current IP address - :var int dirport: current directory port - :var int orport: current orport - :var str contact: directory authority's contact information - :var str legacy_dir_key: fingerprint of and obsolete identity key - :var :class:`stem.descriptor.networkstatus_descriptor.KeyCertificate` key_certificate: directory authority's current key certificate - :var str vote_digest: digest of the authority that contributed to the consensus - """ - - def __init__(self, raw_content, vote = True, validate = True): - """ - Parse a directory authority entry in a v3 network status document and - provide a DirectoryAuthority object. - - :param str raw_content: raw directory authority entry information - :param bool validate: True if the document is to be validated, False otherwise - - :raises: ValueError if the raw data is invalid - """ - - super(DirectoryAuthority, self).__init__(raw_content) - self.nickname, self.identity, self.address, self.ip = None, None, None, None - self.dirport, self.orport, self.legacy_dir_key = None, None, None - self.key_certificate, self.contact, self.vote_digest = None, None, None - parser = stem.descriptor.DescriptorParser(raw_content, validate) - - dir_source = parser.read_keyword_line("dir-source") - self.nickname, self.identity, self.address, self.ip, self.dirport, self.orport = dir_source.split(" ") - self.dirport = int(self.dirport) - self.orport = int(self.orport) - - self.contact = parser.read_keyword_line("contact") - if vote: - self.legacy_dir_key = parser.read_keyword_line("legacy-dir-key", True) - self.key_certificate = KeyCertificate("\n".join(parser.remaining()), validate) - else: - self.vote_digest = parser.read_keyword_line("vote-digest", True) - rmng = parser.remaining() - if rmng and validate: - raise ValueError("Unrecognized trailing data in directory authority information") - -class KeyCertificate(stem.descriptor.Descriptor): - """ - Directory key certificate. - - :var str key_certificate_version: **\*** version of the key certificate (Should be "3") - :var str ip: IP address on which the directory authority is listening - :var int port: port on which the directory authority is listening - :var str fingerprint: **\*** hex encoded fingerprint of the authority's identity key - :var str identity_key: **\*** long term authority identity key - :var datetime published: **\*** time (in GMT) when this document & the key were last generated - :var str expires: **\*** time (in GMT) after which this key becomes invalid - :var str signing_key: **\*** directory server's public signing key - :var str crosscert: signature made using certificate's signing key - :var str certification: **\*** signature of this key certificate signed with the identity key - - **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined - """ - - def __init__(self, raw_content, validate = True): - """ - Parse a key certificate entry and provide a KeyCertificate object. - - :param str raw_content: raw key certificate information - :param bool validate: True if the document is to be validated, False otherwise - - :raises: ValueError if the raw data is invalid - """ - - super(KeyCertificate, self).__init__(raw_content) - self.key_certificate_version, self.ip, self.port = None, None, None - self.fingerprint, self.identity_key, self.published = None, None, None - self.expires, self.signing_key, self.crosscert = None, None, None - self.certification = None - parser = stem.descriptor.DescriptorParser(raw_content, validate) - peek_check_kw = lambda keyword: keyword == parser.peek_keyword() - seen_keywords = set() - - self.key_certificate_version = parser.read_keyword_line("dir-key-certificate-version") - if validate and self.key_certificate_version != "3": raise ValueError("Unrecognized dir-key-certificate-version") - - def _read_keyword_line(keyword): - if validate and keyword in seen_keywords: - raise ValueError("Invalid key certificate: '%s' appears twice" % keyword) - seen_keywords.add(keyword) - return parser.read_keyword_line(keyword) - - while parser.line: - if peek_check_kw("dir-address"): - line = _read_keyword_line("dir-address") - try: - self.ip, self.port = line.rsplit(":", 1) - self.port = int(self.port) - except Exception: - if validate: raise ValueError("Invalid dir-address line: %s" % line) - - elif peek_check_kw("fingerprint"): - self.fingerprint = _read_keyword_line("fingerprint") - - elif peek_check_kw("dir-identity-key"): - _read_keyword_line("dir-identity-key") - self.identity_key = parser.read_block("RSA PUBLIC KEY") - - elif peek_check_kw("dir-key-published"): - self.published = _strptime(_read_keyword_line("dir-key-published")) - - elif peek_check_kw("dir-key-expires"): - self.expires = _strptime(_read_keyword_line("dir-key-expires")) - - elif peek_check_kw("dir-signing-key"): - _read_keyword_line("dir-signing-key") - self.signing_key = parser.read_block("RSA PUBLIC KEY") - - elif peek_check_kw("dir-key-crosscert"): - _read_keyword_line("dir-key-crosscert") - self.crosscert = parser.read_block("ID SIGNATURE") - - elif peek_check_kw("dir-key-certification"): - _read_keyword_line("dir-key-certification") - self.certification = parser.read_block("SIGNATURE") - break - - elif validate: - raise ValueError("Key certificate contains unrecognized lines: %s" % parser.line) - - else: - # ignore unrecognized lines if we aren't validating - self._unrecognized_lines.append(parser.read_line()) - - self._unrecognized_lines = parser.remaining() - if self._unrecognized_lines and validate: - raise ValueError("Unrecognized trailing data in key certificate") - - def get_unrecognized_lines(self): - """ - Returns any unrecognized lines. - - :returns: a list of unrecognized lines - """ - - return self._unrecognized_lines - -class DirectorySignature(stem.descriptor.Descriptor): - """ - Contains directory signature information described in a v3 network status - document. - - :var str identity: signature identity - :var str key_digest: signature key digest - :var str method: method used to generate the signature - :var str signature: the signature data - """ - - def __init__(self, raw_content, validate = True): - """ - Parse a directory signature entry in a v3 network status document and - provide a DirectorySignature object. - - :param str raw_content: raw directory signature entry information - :param bool validate: True if the document is to be validated, False otherwise - - :raises: ValueError if the raw data is invalid - """ - - super(DirectorySignature, self).__init__(raw_content) - self.identity, self.key_digest, self.method, self.signature = None, None, None, None - parser = stem.descriptor.DescriptorParser(raw_content, validate) - - signature_line = parser.read_keyword_line("directory-signature").split(" ") - - if len(signature_line) == 2: - self.identity, self.key_digest = signature_line - if len(signature_line) == 3: # for microdescriptor consensuses - self.method, self.identity, self.key_digest = signature_line - - self.signature = parser.read_block("SIGNATURE") - self._unrecognized_lines = parser.remaining() - if self._unrecognized_lines and validate: - raise ValueError("Unrecognized trailing data in directory signature") - -class RouterDescriptor(stem.descriptor.Descriptor): - """ - Router descriptor object. Parses and stores router information in a router - entry read from a v3 network status document. - - :var str nickname: **\*** router's nickname - :var str identity: **\*** router's identity - :var str digest: **\*** router's digest - :var datetime publication: **\*** router's publication - :var str ip: **\*** router's IP address - :var int orport: **\*** router's ORPort - :var int dirport: **\*** router's DirPort - - :var bool is_valid: **\*** router is valid - :var bool is_guard: **\*** router is suitable for use as an entry guard - :var bool is_named: **\*** router is named - :var bool is_unnamed: **\*** router is unnamed - :var bool is_running: **\*** router is running and currently usable - :var bool is_stable: **\*** router is stable, i.e., it's suitable for for long-lived circuits - :var bool is_exit: **\*** router is an exit router - :var bool is_fast: **\*** router is Fast, i.e., it's usable for high-bandwidth circuits - :var bool is_authority: **\*** router is a directory authority - :var bool supports_v2dir: **\*** router supports v2dir - :var bool supports_v3dir: **\*** router supports v3dir - :var bool is_hsdir: **\*** router is a hidden status - :var bool is_badexit: **\*** router is marked a bad exit - :var bool is_baddirectory: **\*** router is a bad directory - - :var :class:`stem.version.Version`,str version: Version of the Tor protocol this router is running - - :var int bandwidth: router's claimed bandwidth - :var int measured_bandwidth: router's measured bandwidth - - :var :class:`stem.exit_policy.MicrodescriptorExitPolicy` exitpolicy: router's exitpolicy - - :var str microdescriptor_hashes: a list of two-tuples with a list of consensus methods(int) that may produce the digest and a dict with algorithm(str) => digest(str) mappings. algorithm is the hashing algorithm (usually "sha256") that is used to produce digest (the base64 encoding of the hash of the router's microdescriptor with trailing =s omitted). - - **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined - """ - - def __init__(self, raw_contents, vote = True, validate = True): - """ - Parse a router descriptor in a v3 network status document and provide a new - RouterDescriptor object. - - :param str raw_content: router descriptor content to be parsed - :param bool validate: whether the router descriptor should be validated - """ - - super(RouterDescriptor, self).__init__(raw_contents) - - self.nickname = None - self.identity = None - self.digest = None - self.publication = None - self.ip = None - self.orport = None - self.dirport = None - - self.is_valid = False - self.is_guard = False - self.is_named = False - self.is_unnamed = False - self.is_running = False - self.is_stable = False - self.is_exit = False - self.is_fast = False - self.is_authority = False - self.supports_v2dir = False - self.supports_v3dir = False - self.is_hsdir = False - self.is_badexit = False - self.is_baddirectory = False - - self.version = None - - self.bandwidth = None - self.measured_bandwidth = None - - self.exit_policy = None - - self.microdescriptor_hashes = [] - - self._parse(raw_contents, vote, validate) - - def _parse(self, raw_content, vote, validate): - """ - :param dict raw_content: iptor contents to be applied - :param bool validate: checks the validity of descriptor content if True - - :raises: ValueError if an error occures in validation - """ - - parser = stem.descriptor.DescriptorParser(raw_content, validate) - seen_keywords = set() - peek_check_kw = lambda keyword: keyword == parser.peek_keyword() - - r = parser.read_keyword_line("r") - # r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0 - # "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort SP DirPort NL - seen_keywords.add("r") - if r: - values = r.split(" ") - self.nickname, self.identity, self.digest = values[0], values[1], values[2] - self.publication = _strptime(" ".join((values[3], values[4])), validate) - self.ip, self.orport, self.dirport = values[5], int(values[6]), int(values[7]) - if self.dirport == 0: self.dirport = None - elif validate: raise ValueError("Invalid router descriptor: empty 'r' line" ) - - while parser.line: - if peek_check_kw("s"): - if "s" in seen_keywords: raise ValueError("Invalid router descriptor: 's' line appears twice") - line = parser.read_keyword_line("s") - if not line: continue - seen_keywords.add("s") - # s Named Running Stable Valid - #A series of space-separated status flags, in *lexical order* - flags = line.split(" ") - flag_map = { - "Valid": "is_valid", - "Guard": "is_guard", - "Named": "is_named", - "Unnamed": "is_unnamed", - "Running": "is_running", - "Stable": "is_stable", - "Exit": "is_exit", - "Fast": "is_fast", - "Authority": "is_authority", - "V2Dir": "supports_v2dir", - "V3Dir": "supports_v3dir", - "HSDir": "is_hsdir", - "BadExit": "is_badexit", - "BadDirectory": "is_baddirectory", - } - map(lambda flag: setattr(self, flag_map[flag], True), flags) - - if self.is_unnamed: self.is_named = False - elif self.is_named: self.is_unnamed = False - - elif peek_check_kw("v"): - if "v" in seen_keywords: raise ValueError("Invalid router descriptor: 'v' line appears twice") - line = parser.read_keyword_line("v", True) - seen_keywords.add("v") - # v Tor 0.2.2.35 - if line: - if line.startswith("Tor "): - self.version = stem.version.Version(line[4:]) - else: - self.version = line - elif validate: raise ValueError("Invalid router descriptor: empty 'v' line" ) - - elif peek_check_kw("w"): - if "w" in seen_keywords: raise ValueError("Invalid router descriptor: 'w' line appears twice") - w = parser.read_keyword_line("w", True) - # "w" SP "Bandwidth=" INT [SP "Measured=" INT] NL - seen_keywords.add("w") - if w: - values = w.split(" ") - if len(values) <= 2 and len(values) > 0: - key, value = values[0].split("=") - if key == "Bandwidth": self.bandwidth = int(value) - elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Bandwidth, read " + key) - - if len(values) == 2: - key, value = values[1].split("=") - if key == "Measured": self.measured_bandwidth = int(value) - elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Measured, read " + key) - elif validate: raise ValueError("Router descriptor contains invalid 'w' line") - elif validate: raise ValueError("Router descriptor contains empty 'w' line") - - elif peek_check_kw("p"): - if "p" in seen_keywords: raise ValueError("Invalid router descriptor: 'p' line appears twice") - p = parser.read_keyword_line("p", True) - seen_keywords.add("p") - # "p" SP ("accept" / "reject") SP PortList NL - if p: - self.exit_policy = stem.exit_policy.MicrodescriptorExitPolicy(p) - #self.exit_policy = p - - elif vote and peek_check_kw("m"): - # microdescriptor hashes - m = parser.read_keyword_line("m", True) - methods, digests = m.split(" ", 1) - method_list = methods.split(",") - digest_dict = [digest.split("=", 1) for digest in digests.split(" ")] - self.microdescriptor_hashes.append((method_list, digest_dict)) - - elif validate: - raise ValueError("Router descriptor contains unrecognized trailing lines: %s" % parser.line) - - else: - self._unrecognized_lines.append(parser.read_line()) # ignore unrecognized lines if we aren't validating - - def get_unrecognized_lines(self): - """ - Returns any unrecognized lines. - - :returns: a list of unrecognized lines - """ - - return self._unrecognized_lines - diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py new file mode 100644 index 0000000..e65d960 --- /dev/null +++ b/test/integ/descriptor/networkstatus.py @@ -0,0 +1,228 @@ +""" +Integration tests for stem.descriptor.server_descriptor. +""" + +from __future__ import with_statement + +import os +import resource +import datetime +import unittest + +import stem.exit_policy +import stem.version +import stem.descriptor.networkstatus +import test.integ.descriptor + +def _strptime(string): + return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") + +class TestNetworkStatusDocument(unittest.TestCase): + def test_metrics_consensus(self): + """ + Checks if consensus documents from Metrics are parsed properly. + """ + + descriptor_path = test.integ.descriptor.get_resource("metrics_consensus") + + with file(descriptor_path) as descriptor_file: + desc = stem.descriptor.parse_file(descriptor_path, descriptor_file) + + router = next(desc) + self.assertEquals("sumkledi", router.nickname) + self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router.identity) + self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router.digest) + self.assertEquals(_strptime("2012-07-12 04:01:55"), router.publication) + self.assertEquals("178.218.213.229", router.ip) + self.assertEquals(80, router.orport) + self.assertEquals(None, router.dirport) + + def test_cached_consensus(self): + """ + Parses the cached-consensus file in our data directory. + """ + + # lengthy test and uneffected by targets, so only run once + if test.runner.only_run_once(self, "test_cached_consensus"): return + + descriptor_path = test.runner.get_runner().get_test_dir("cached-consensus") + + if not os.path.exists(descriptor_path): + test.runner.skip(self, "(no cached-consensus)") + + if stem.util.system.is_windows(): + # might hog memory and hang the system + # and we aren't checking for memory usage in windows, so, skip. + test.runner.skip(self, "(unavailable on windows)") + + count = 0 + with open(descriptor_path) as descriptor_file: + for desc in stem.descriptor.networkstatus.parse_file(descriptor_file): + if resource.getrusage(resource.RUSAGE_SELF).ru_maxrss > 100000: + # if we're using > 100 MB we should fail + self.fail() + assert desc.nickname # check that the router has a nickname + count += 1 + + assert count > 100 # sanity check - assuming atleast 100 relays in the Tor network + + def test_consensus(self): + """ + Checks that consensus documents are properly parsed. + """ + + descriptor_path = test.integ.descriptor.get_resource("consensus") + + descriptor_file = file(descriptor_path) + desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) + descriptor_file.close() + + self.assertEquals(True, desc.validated) + self.assertEquals("3", desc.network_status_version) + self.assertEquals("consensus", desc.vote_status) + self.assertEquals([], desc.consensus_methods) + self.assertEquals(None, desc.published) + self.assertEquals(12, desc.consensus_method) + self.assertEquals(_strptime("2012-07-12 10:00:00"), desc.valid_after) + self.assertEquals(_strptime("2012-07-12 11:00:00"), desc.fresh_until) + self.assertEquals(_strptime("2012-07-12 13:00:00"), desc.valid_until) + self.assertEquals(300, desc.vote_delay) + self.assertEquals(300, desc.dist_delay) + expected_client_versions = [stem.version.Version(version_string) for version_string in ["0.2.2.35", + "0.2.2.36", "0.2.2.37", "0.2.3.10-alpha", "0.2.3.11-alpha", "0.2.3.12-alpha", + "0.2.3.13-alpha", "0.2.3.14-alpha", "0.2.3.15-alpha", "0.2.3.16-alpha", "0.2.3.17-beta", + "0.2.3.18-rc", "0.2.3.19-rc"]] + expected_server_versions = [stem.version.Version(version_string) for version_string in ["0.2.2.35", + "0.2.2.36", "0.2.2.37", "0.2.3.10-alpha", "0.2.3.11-alpha", "0.2.3.12-alpha", + "0.2.3.13-alpha", "0.2.3.14-alpha", "0.2.3.15-alpha", "0.2.3.16-alpha", "0.2.3.17-beta", + "0.2.3.18-rc", "0.2.3.19-rc"]] + self.assertEquals(expected_client_versions, desc.client_versions) + self.assertEquals(expected_server_versions, desc.server_versions) + known_flags = ["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", "Named", "Running", "Stable", "Unnamed", "V2Dir", "Valid"] + self.assertEquals(known_flags, desc.known_flags) + expected_params = {"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1} + self.assertEquals(expected_params, desc.params) + router1 = next(desc.router_descriptors) + self.assertEquals("sumkledi", router1.nickname) + self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router1.identity) + self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router1.digest) + self.assertEquals(_strptime("2012-07-12 04:01:55"), router1.publication) + self.assertEquals("178.218.213.229", router1.ip) + self.assertEquals(80, router1.orport) + self.assertEquals(None, router1.dirport) + + self.assertEquals(8, len(desc.directory_authorities)) + self.assertEquals("tor26", desc.directory_authorities[0].nickname) + self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.directory_authorities[0].identity) + self.assertEquals("86.59.21.38", desc.directory_authorities[0].address) + self.assertEquals("86.59.21.38", desc.directory_authorities[0].ip) + self.assertEquals(80, desc.directory_authorities[0].dirport) + self.assertEquals(443, desc.directory_authorities[0].orport) + self.assertEquals("Peter Palfrader", desc.directory_authorities[0].contact) + self.assertEquals(None, desc.directory_authorities[0].legacy_dir_key) + self.assertEquals(None, desc.directory_authorities[0].key_certificate) + self.assertEquals("0B6D1E9A300B895AA2D0B427F92917B6995C3C1C", desc.directory_authorities[0].vote_digest) + expected_bandwidth_weights = { + "Wbd": 3335, "Wbe": 0, "Wbg": 3536, "Wbm": 10000, "Wdb": 10000, "Web": 10000, + "Wed": 3329, "Wee": 10000, "Weg": 3329, "Wem": 10000, "Wgb": 10000, "Wgd": 3335, + "Wgg": 6464, "Wgm": 6464, "Wmb": 10000, "Wmd": 3335, "Wme": 0, "Wmg": 3536, "Wmm": 10000 + } + self.assertEquals(expected_bandwidth_weights, desc.bandwidth_weights) + + expected_signature = """HFXB4497LzESysYJ/4jJY83E5vLjhv+igIxD9LU6lf6ftkGeF+lNmIAIEKaMts8H +mfWcW0b+jsrXcJoCxV5IrwCDF3u1aC3diwZY6yiG186pwWbOwE41188XI2DeYPwE +I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY=""" + self.assertEquals(8, len(desc.directory_signatures)) + self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.directory_signatures[0].identity) + self.assertEquals("BF112F1C6D5543CFD0A32215ACABD4197B5279AD", desc.directory_signatures[0].key_digest) + self.assertEquals(expected_signature, desc.directory_signatures[0].signature) + + def test_vote(self): + """ + Checks that vote documents are properly parsed. + """ + + descriptor_path = test.integ.descriptor.get_resource("vote") + + descriptor_file = file(descriptor_path) + desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) + descriptor_file.close() + + self.assertEquals(True, desc.validated) + self.assertEquals("3", desc.network_status_version) + self.assertEquals("vote", desc.vote_status) + self.assertEquals(range(1, 13), desc.consensus_methods) + self.assertEquals(_strptime("2012-07-11 23:50:01"), desc.published) + self.assertEquals(None, desc.consensus_method) + self.assertEquals(_strptime("2012-07-12 00:00:00"), desc.valid_after) + self.assertEquals(_strptime("2012-07-12 01:00:00"), desc.fresh_until) + self.assertEquals(_strptime("2012-07-12 03:00:00"), desc.valid_until) + self.assertEquals(300, desc.vote_delay) + self.assertEquals(300, desc.dist_delay) + self.assertEquals([], desc.client_versions) + self.assertEquals([], desc.server_versions) + known_flags = ["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", "Running", "Stable", "V2Dir", "Valid"] + self.assertEquals(known_flags, desc.known_flags) + expected_params = {"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1} + self.assertEquals(expected_params, desc.params) + router1 = next(desc.router_descriptors) + self.assertEquals("sumkledi", router1.nickname) + self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router1.identity) + self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router1.digest) + self.assertEquals(_strptime("2012-07-11 04:22:53"), router1.publication) + self.assertEquals("178.218.213.229", router1.ip) + self.assertEquals(80, router1.orport) + self.assertEquals(None, router1.dirport) + + self.assertEquals(1, len(desc.directory_authorities)) + self.assertEquals("turtles", desc.directory_authorities[0].nickname) + self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_authorities[0].identity) + self.assertEquals("76.73.17.194", desc.directory_authorities[0].address) + self.assertEquals("76.73.17.194", desc.directory_authorities[0].ip) + self.assertEquals(9030, desc.directory_authorities[0].dirport) + self.assertEquals(9090, desc.directory_authorities[0].orport) + self.assertEquals("Mike Perry <email>", desc.directory_authorities[0].contact) + self.assertEquals(None, desc.directory_authorities[0].legacy_dir_key) + + expected_identity_key = """MIIBigKCAYEA6uSmsoxj2MiJ3qyZq0qYXlRoG8o82SNqg+22m+t1c7MlQOZWPJYn +XeMcBCt8xrTeIt2ZI+Q/Kt2QJSeD9WZRevTKk/kn5Tg2+xXPogalUU47y5tUohGz ++Q8+CxtRSXpDxBHL2P8rLHvGrI69wbNHGoQkce/7gJy9vw5Ie2qzbyXk1NG6V8Fb +pr6A885vHo6TbhUnolz2Wqt/kN+UorjLkN2H3fV+iGcQFv42SyHYGDLa0WwL3PJJ +r/veu36S3VaHBrfhutfioi+d3d4Ya0bKwiWi5Lm2CHuuRTgMpHLU9vlci8Hunuxq +HsULe2oMsr4VEic7sW5SPC5Obpx6hStHdNv1GxoSEm3/vIuPM8pINpU5ZYAyH9yO +Ef22ZHeiVMMKmpV9TtFyiFqvlI6GpQn3mNbsQqF1y3XCA3Q4vlRAkpgJVUSvTxFP +2bNDobOyVCpCM/rwxU1+RCNY5MFJ/+oktUY+0ydvTen3gFdZdgNqCYjKPLfBNm9m +RGL7jZunMUNvAgMBAAE=""" + expected_signing_key = """MIGJAoGBAJ5itcJRYNEM3Qf1OVWLRkwjqf84oXPc2ZusaJ5zOe7TVvBMra9GNyc0 +NM9y6zVkHCAePAjr4KbW/8P1olA6FUE2LV9bozaU1jFf6K8B2OELKs5FUEW+n+ic +GM0x6MhngyXonWOcKt5Gj+mAu5lrno9tpNbPkz2Utr/Pi0nsDhWlAgMBAAE=""" + expected_key_crosscert = """RHYImGTwg36wmEdAn7qaRg2sAfql7ZCtPIL/O3lU5OIdXXp0tNn/K00Bamqohjk+ +Tz4FKsKXGDlbGv67PQcZPOK6NF0GRkNh4pk89prrDO4XwtEn7rkHHdBH6/qQ7IRG +GdDZHtZ1a69oFZvPWD3hUaB50xeIe7GoKdKIfdNNJ+8=""" + expected_key_certification = """fasWOGyUZ3iMCYpDfJ+0JcMiTH25sXPWzvlHorEOyOMbaMqRYpZU4GHzt1jLgdl6 +AAoR6KdamsLg5VE8xzst48a4UFuzHFlklZ5O8om2rcvDd5DhSnWWYZnYJecqB+bo +dNisPmaIVSAWb29U8BpNRj4GMC9KAgGYUj8aE/KtutAeEekFfFEHTfWZ2fFp4j3m +9rY8FWraqyiF+Emq1T8pAAgMQ+79R3oZxq0TXS42Z4Anhms735ccauKhI3pDKjbl +tD5vAzIHOyjAOXj7a6jY/GrnaBNuJ4qe/4Hf9UmzK/jKKwG95BPJtPTT4LoFwEB0 +KG2OUeQUNoCck4nDpsZwFqPlrWCHcHfTV2iDYFV1HQWDTtZz/qf+GtB8NXsq+I1w +brADmvReM2BD6p/13h0QURCI5hq7ZYlIKcKrBa0jn1d9cduULl7vgKsRCJDls/ID +emBZ6pUxMpBmV0v+PrA3v9w4DlE7GHAq61FF/zju2kpqj6MInbEvI/E+e438sWsL""" + self.assertEquals("3", desc.directory_authorities[0].key_certificate.key_certificate_version) + self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_authorities[0].key_certificate.fingerprint) + self.assertEquals(_strptime("2011-11-28 21:51:04"), desc.directory_authorities[0].key_certificate.published) + self.assertEquals(_strptime("2012-11-28 21:51:04"), desc.directory_authorities[0].key_certificate.expires) + self.assertEquals(expected_identity_key, desc.directory_authorities[0].key_certificate.identity_key) + self.assertEquals(expected_signing_key, desc.directory_authorities[0].key_certificate.signing_key) + self.assertEquals(expected_key_crosscert, desc.directory_authorities[0].key_certificate.crosscert) + self.assertEquals(expected_key_certification, desc.directory_authorities[0].key_certificate.certification) + self.assertEquals(None, desc.directory_authorities[0].vote_digest) + self.assertEquals({}, desc.bandwidth_weights) + + expected_signature = """fskXN84wB3mXfo+yKGSt0AcDaaPuU3NwMR3ROxWgLN0KjAaVi2eV9PkPCsQkcgw3 +JZ/1HL9sHyZfo6bwaC6YSM9PNiiY6L7rnGpS7UkHiFI+M96VCMorvjm5YPs3FioJ +DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w=""" + self.assertEquals(1, len(desc.directory_signatures)) + self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_signatures[0].identity) + self.assertEquals("D5C30C15BB3F1DA27669C2D88439939E8F418FCF", desc.directory_signatures[0].key_digest) + self.assertEquals(expected_signature, desc.directory_signatures[0].signature) + diff --git a/test/integ/descriptor/networkstatus_descriptor.py b/test/integ/descriptor/networkstatus_descriptor.py deleted file mode 100644 index 46019ea..0000000 --- a/test/integ/descriptor/networkstatus_descriptor.py +++ /dev/null @@ -1,228 +0,0 @@ -""" -Integration tests for stem.descriptor.server_descriptor. -""" - -from __future__ import with_statement - -import os -import resource -import datetime -import unittest - -import stem.exit_policy -import stem.version -import stem.descriptor.networkstatus_descriptor -import test.integ.descriptor - -def _strptime(string): - return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") - -class TestNetworkStatusDocument(unittest.TestCase): - def test_metrics_consensus(self): - """ - Checks if consensus documents from Metrics are parsed properly. - """ - - descriptor_path = test.integ.descriptor.get_resource("metrics_consensus") - - with file(descriptor_path) as descriptor_file: - desc = stem.descriptor.parse_file(descriptor_path, descriptor_file) - - router = next(desc) - self.assertEquals("sumkledi", router.nickname) - self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router.identity) - self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router.digest) - self.assertEquals(_strptime("2012-07-12 04:01:55"), router.publication) - self.assertEquals("178.218.213.229", router.ip) - self.assertEquals(80, router.orport) - self.assertEquals(None, router.dirport) - - def test_cached_consensus(self): - """ - Parses the cached-consensus file in our data directory. - """ - - # lengthy test and uneffected by targets, so only run once - if test.runner.only_run_once(self, "test_cached_consensus"): return - - descriptor_path = test.runner.get_runner().get_test_dir("cached-consensus") - - if not os.path.exists(descriptor_path): - test.runner.skip(self, "(no cached-consensus)") - - if stem.util.system.is_windows(): - # might hog memory and hang the system - # and we aren't checking for memory usage in windows, so, skip. - test.runner.skip(self, "(unavailable on windows)") - - count = 0 - with open(descriptor_path) as descriptor_file: - for desc in stem.descriptor.networkstatus_descriptor.parse_file(descriptor_file): - if resource.getrusage(resource.RUSAGE_SELF).ru_maxrss > 100000: - # if we're using > 100 MB we should fail - self.fail() - assert desc.nickname # check that the router has a nickname - count += 1 - - assert count > 100 # sanity check - assuming atleast 100 relays in the Tor network - - def test_consensus(self): - """ - Checks that consensus documents are properly parsed. - """ - - descriptor_path = test.integ.descriptor.get_resource("consensus") - - descriptor_file = file(descriptor_path) - desc = stem.descriptor.networkstatus_descriptor.NetworkStatusDocument(descriptor_file.read()) - descriptor_file.close() - - self.assertEquals(True, desc.validated) - self.assertEquals("3", desc.network_status_version) - self.assertEquals("consensus", desc.vote_status) - self.assertEquals([], desc.consensus_methods) - self.assertEquals(None, desc.published) - self.assertEquals(12, desc.consensus_method) - self.assertEquals(_strptime("2012-07-12 10:00:00"), desc.valid_after) - self.assertEquals(_strptime("2012-07-12 11:00:00"), desc.fresh_until) - self.assertEquals(_strptime("2012-07-12 13:00:00"), desc.valid_until) - self.assertEquals(300, desc.vote_delay) - self.assertEquals(300, desc.dist_delay) - expected_client_versions = [stem.version.Version(version_string) for version_string in ["0.2.2.35", - "0.2.2.36", "0.2.2.37", "0.2.3.10-alpha", "0.2.3.11-alpha", "0.2.3.12-alpha", - "0.2.3.13-alpha", "0.2.3.14-alpha", "0.2.3.15-alpha", "0.2.3.16-alpha", "0.2.3.17-beta", - "0.2.3.18-rc", "0.2.3.19-rc"]] - expected_server_versions = [stem.version.Version(version_string) for version_string in ["0.2.2.35", - "0.2.2.36", "0.2.2.37", "0.2.3.10-alpha", "0.2.3.11-alpha", "0.2.3.12-alpha", - "0.2.3.13-alpha", "0.2.3.14-alpha", "0.2.3.15-alpha", "0.2.3.16-alpha", "0.2.3.17-beta", - "0.2.3.18-rc", "0.2.3.19-rc"]] - self.assertEquals(expected_client_versions, desc.client_versions) - self.assertEquals(expected_server_versions, desc.server_versions) - known_flags = ["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", "Named", "Running", "Stable", "Unnamed", "V2Dir", "Valid"] - self.assertEquals(known_flags, desc.known_flags) - expected_params = {"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1} - self.assertEquals(expected_params, desc.params) - router1 = next(desc.router_descriptors) - self.assertEquals("sumkledi", router1.nickname) - self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router1.identity) - self.assertEquals("8mCr8Sl7RF4ENU4jb0FZFA/3do8", router1.digest) - self.assertEquals(_strptime("2012-07-12 04:01:55"), router1.publication) - self.assertEquals("178.218.213.229", router1.ip) - self.assertEquals(80, router1.orport) - self.assertEquals(None, router1.dirport) - - self.assertEquals(8, len(desc.directory_authorities)) - self.assertEquals("tor26", desc.directory_authorities[0].nickname) - self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.directory_authorities[0].identity) - self.assertEquals("86.59.21.38", desc.directory_authorities[0].address) - self.assertEquals("86.59.21.38", desc.directory_authorities[0].ip) - self.assertEquals(80, desc.directory_authorities[0].dirport) - self.assertEquals(443, desc.directory_authorities[0].orport) - self.assertEquals("Peter Palfrader", desc.directory_authorities[0].contact) - self.assertEquals(None, desc.directory_authorities[0].legacy_dir_key) - self.assertEquals(None, desc.directory_authorities[0].key_certificate) - self.assertEquals("0B6D1E9A300B895AA2D0B427F92917B6995C3C1C", desc.directory_authorities[0].vote_digest) - expected_bandwidth_weights = { - "Wbd": 3335, "Wbe": 0, "Wbg": 3536, "Wbm": 10000, "Wdb": 10000, "Web": 10000, - "Wed": 3329, "Wee": 10000, "Weg": 3329, "Wem": 10000, "Wgb": 10000, "Wgd": 3335, - "Wgg": 6464, "Wgm": 6464, "Wmb": 10000, "Wmd": 3335, "Wme": 0, "Wmg": 3536, "Wmm": 10000 - } - self.assertEquals(expected_bandwidth_weights, desc.bandwidth_weights) - - expected_signature = """HFXB4497LzESysYJ/4jJY83E5vLjhv+igIxD9LU6lf6ftkGeF+lNmIAIEKaMts8H -mfWcW0b+jsrXcJoCxV5IrwCDF3u1aC3diwZY6yiG186pwWbOwE41188XI2DeYPwE -I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY=""" - self.assertEquals(8, len(desc.directory_signatures)) - self.assertEquals("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4", desc.directory_signatures[0].identity) - self.assertEquals("BF112F1C6D5543CFD0A32215ACABD4197B5279AD", desc.directory_signatures[0].key_digest) - self.assertEquals(expected_signature, desc.directory_signatures[0].signature) - - def test_vote(self): - """ - Checks that vote documents are properly parsed. - """ - - descriptor_path = test.integ.descriptor.get_resource("vote") - - descriptor_file = file(descriptor_path) - desc = stem.descriptor.networkstatus_descriptor.NetworkStatusDocument(descriptor_file.read()) - descriptor_file.close() - - self.assertEquals(True, desc.validated) - self.assertEquals("3", desc.network_status_version) - self.assertEquals("vote", desc.vote_status) - self.assertEquals(range(1, 13), desc.consensus_methods) - self.assertEquals(_strptime("2012-07-11 23:50:01"), desc.published) - self.assertEquals(None, desc.consensus_method) - self.assertEquals(_strptime("2012-07-12 00:00:00"), desc.valid_after) - self.assertEquals(_strptime("2012-07-12 01:00:00"), desc.fresh_until) - self.assertEquals(_strptime("2012-07-12 03:00:00"), desc.valid_until) - self.assertEquals(300, desc.vote_delay) - self.assertEquals(300, desc.dist_delay) - self.assertEquals([], desc.client_versions) - self.assertEquals([], desc.server_versions) - known_flags = ["Authority", "BadExit", "Exit", "Fast", "Guard", "HSDir", "Running", "Stable", "V2Dir", "Valid"] - self.assertEquals(known_flags, desc.known_flags) - expected_params = {"CircuitPriorityHalflifeMsec": 30000, "bwauthpid": 1} - self.assertEquals(expected_params, desc.params) - router1 = next(desc.router_descriptors) - self.assertEquals("sumkledi", router1.nickname) - self.assertEquals("ABPSI4nNUNC3hKPkBhyzHozozrU", router1.identity) - self.assertEquals("B5n4BiALAF8B5AqafxohyYiuj7E", router1.digest) - self.assertEquals(_strptime("2012-07-11 04:22:53"), router1.publication) - self.assertEquals("178.218.213.229", router1.ip) - self.assertEquals(80, router1.orport) - self.assertEquals(None, router1.dirport) - - self.assertEquals(1, len(desc.directory_authorities)) - self.assertEquals("turtles", desc.directory_authorities[0].nickname) - self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_authorities[0].identity) - self.assertEquals("76.73.17.194", desc.directory_authorities[0].address) - self.assertEquals("76.73.17.194", desc.directory_authorities[0].ip) - self.assertEquals(9030, desc.directory_authorities[0].dirport) - self.assertEquals(9090, desc.directory_authorities[0].orport) - self.assertEquals("Mike Perry <email>", desc.directory_authorities[0].contact) - self.assertEquals(None, desc.directory_authorities[0].legacy_dir_key) - - expected_identity_key = """MIIBigKCAYEA6uSmsoxj2MiJ3qyZq0qYXlRoG8o82SNqg+22m+t1c7MlQOZWPJYn -XeMcBCt8xrTeIt2ZI+Q/Kt2QJSeD9WZRevTKk/kn5Tg2+xXPogalUU47y5tUohGz -+Q8+CxtRSXpDxBHL2P8rLHvGrI69wbNHGoQkce/7gJy9vw5Ie2qzbyXk1NG6V8Fb -pr6A885vHo6TbhUnolz2Wqt/kN+UorjLkN2H3fV+iGcQFv42SyHYGDLa0WwL3PJJ -r/veu36S3VaHBrfhutfioi+d3d4Ya0bKwiWi5Lm2CHuuRTgMpHLU9vlci8Hunuxq -HsULe2oMsr4VEic7sW5SPC5Obpx6hStHdNv1GxoSEm3/vIuPM8pINpU5ZYAyH9yO -Ef22ZHeiVMMKmpV9TtFyiFqvlI6GpQn3mNbsQqF1y3XCA3Q4vlRAkpgJVUSvTxFP -2bNDobOyVCpCM/rwxU1+RCNY5MFJ/+oktUY+0ydvTen3gFdZdgNqCYjKPLfBNm9m -RGL7jZunMUNvAgMBAAE=""" - expected_signing_key = """MIGJAoGBAJ5itcJRYNEM3Qf1OVWLRkwjqf84oXPc2ZusaJ5zOe7TVvBMra9GNyc0 -NM9y6zVkHCAePAjr4KbW/8P1olA6FUE2LV9bozaU1jFf6K8B2OELKs5FUEW+n+ic -GM0x6MhngyXonWOcKt5Gj+mAu5lrno9tpNbPkz2Utr/Pi0nsDhWlAgMBAAE=""" - expected_key_crosscert = """RHYImGTwg36wmEdAn7qaRg2sAfql7ZCtPIL/O3lU5OIdXXp0tNn/K00Bamqohjk+ -Tz4FKsKXGDlbGv67PQcZPOK6NF0GRkNh4pk89prrDO4XwtEn7rkHHdBH6/qQ7IRG -GdDZHtZ1a69oFZvPWD3hUaB50xeIe7GoKdKIfdNNJ+8=""" - expected_key_certification = """fasWOGyUZ3iMCYpDfJ+0JcMiTH25sXPWzvlHorEOyOMbaMqRYpZU4GHzt1jLgdl6 -AAoR6KdamsLg5VE8xzst48a4UFuzHFlklZ5O8om2rcvDd5DhSnWWYZnYJecqB+bo -dNisPmaIVSAWb29U8BpNRj4GMC9KAgGYUj8aE/KtutAeEekFfFEHTfWZ2fFp4j3m -9rY8FWraqyiF+Emq1T8pAAgMQ+79R3oZxq0TXS42Z4Anhms735ccauKhI3pDKjbl -tD5vAzIHOyjAOXj7a6jY/GrnaBNuJ4qe/4Hf9UmzK/jKKwG95BPJtPTT4LoFwEB0 -KG2OUeQUNoCck4nDpsZwFqPlrWCHcHfTV2iDYFV1HQWDTtZz/qf+GtB8NXsq+I1w -brADmvReM2BD6p/13h0QURCI5hq7ZYlIKcKrBa0jn1d9cduULl7vgKsRCJDls/ID -emBZ6pUxMpBmV0v+PrA3v9w4DlE7GHAq61FF/zju2kpqj6MInbEvI/E+e438sWsL""" - self.assertEquals("3", desc.directory_authorities[0].key_certificate.key_certificate_version) - self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_authorities[0].key_certificate.fingerprint) - self.assertEquals(_strptime("2011-11-28 21:51:04"), desc.directory_authorities[0].key_certificate.published) - self.assertEquals(_strptime("2012-11-28 21:51:04"), desc.directory_authorities[0].key_certificate.expires) - self.assertEquals(expected_identity_key, desc.directory_authorities[0].key_certificate.identity_key) - self.assertEquals(expected_signing_key, desc.directory_authorities[0].key_certificate.signing_key) - self.assertEquals(expected_key_crosscert, desc.directory_authorities[0].key_certificate.crosscert) - self.assertEquals(expected_key_certification, desc.directory_authorities[0].key_certificate.certification) - self.assertEquals(None, desc.directory_authorities[0].vote_digest) - self.assertEquals({}, desc.bandwidth_weights) - - expected_signature = """fskXN84wB3mXfo+yKGSt0AcDaaPuU3NwMR3ROxWgLN0KjAaVi2eV9PkPCsQkcgw3 -JZ/1HL9sHyZfo6bwaC6YSM9PNiiY6L7rnGpS7UkHiFI+M96VCMorvjm5YPs3FioJ -DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w=""" - self.assertEquals(1, len(desc.directory_signatures)) - self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_signatures[0].identity) - self.assertEquals("D5C30C15BB3F1DA27669C2D88439939E8F418FCF", desc.directory_signatures[0].key_digest) - self.assertEquals(expected_signature, desc.directory_signatures[0].signature) -

1 0

[stem/master] Fix cached-consensus parsing
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 847d1caec38e110f7d75ae1db526a85803094d13 Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Tue Aug 7 14:43:42 2012 +0530 Fix cached-consensus parsing --- stem/descriptor/__init__.py | 2 +- test/integ/descriptor/data/cached-consensus | 130 +++++++++++++++++++++++++++ test/integ/descriptor/data/consensus | 130 --------------------------- test/integ/descriptor/networkstatus.py | 2 +- test/integ/descriptor/reader.py | 2 +- 5 files changed, 133 insertions(+), 133 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 72d2275..5f9d144 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -64,7 +64,7 @@ def parse_file(path, descriptor_file): elif filename == "cached-extrainfo": file_parser = stem.descriptor.extrainfo_descriptor.parse_file elif filename == "cached-consensus": - file_parser = stem.descriptor.extrainfo_descriptor.parse_file + file_parser = stem.descriptor.networkstatus.parse_file if file_parser: for desc in file_parser(descriptor_file): diff --git a/test/integ/descriptor/data/cached-consensus b/test/integ/descriptor/data/cached-consensus new file mode 100644 index 0000000..adcf13b --- /dev/null +++ b/test/integ/descriptor/data/cached-consensus @@ -0,0 +1,130 @@ +network-status-version 3 +vote-status consensus +consensus-method 12 +valid-after 2012-07-12 10:00:00 +fresh-until 2012-07-12 11:00:00 +valid-until 2012-07-12 13:00:00 +voting-delay 300 300 +client-versions 0.2.2.35,0.2.2.36,0.2.2.37,0.2.3.10-alpha,0.2.3.11-alpha,0.2.3.12-alpha,0.2.3.13-alpha,0.2.3.14-alpha,0.2.3.15-alpha,0.2.3.16-alpha,0.2.3.17-beta,0.2.3.18-rc,0.2.3.19-rc +server-versions 0.2.2.35,0.2.2.36,0.2.2.37,0.2.3.10-alpha,0.2.3.11-alpha,0.2.3.12-alpha,0.2.3.13-alpha,0.2.3.14-alpha,0.2.3.15-alpha,0.2.3.16-alpha,0.2.3.17-beta,0.2.3.18-rc,0.2.3.19-rc +known-flags Authority BadExit Exit Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid +params CircuitPriorityHalflifeMsec=30000 bwauthpid=1 +dir-source tor26 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 86.59.21.38 86.59.21.38 80 443 +contact Peter Palfrader +vote-digest 0B6D1E9A300B895AA2D0B427F92917B6995C3C1C +dir-source turtles 27B6B5996C426270A5C95488AA5BCEB6BCC86956 76.73.17.194 76.73.17.194 9030 9090 +contact Mike Perry <mikeperryTAfsckedTODorg> +vote-digest 904B1974B9879D02B4ADFB81D7E9B4E07D768A5A +dir-source maatuska 49015F787433103580E3B66A1707A00E60F2D15B 171.25.193.9 171.25.193.9 443 80 +contact 4096R/23291265 Linus Nordberg <linus(a)nordberg.se> +vote-digest A8839355BAC373320B8CEDD0A0A09DAAA1637E3A +dir-source dannenberg 585769C78764D58426B8B52B6651A5A71137189A dannenberg.ccc.de 193.23.244.244 80 443 +contact Andreas Lehner <anonymizer(a)ccc.de> +vote-digest 416B73C49E717B0A5D61A4F634DCCF94611802E7 +dir-source urras 80550987E1D626E3EBA5E5E75A458DE0626D088C 208.83.223.34 208.83.223.34 443 80 +contact 4096R/E012B42D Jacob Appelbaum <jacob(a)appelbaum.net> +vote-digest 08B1F8E4910F136E7FB7DFD52ABB2A9EDE939F0B +dir-source moria1 D586D18309DED4CD6D57C18FDB97EFA96D330566 128.31.0.34 128.31.0.34 9131 9101 +contact 1024D/28988BF5 arma mit edu +vote-digest 5006931FB78F7AE42B602697591DBA82AACEF533 +dir-source dizum E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 194.109.206.212 194.109.206.212 80 443 +contact 1024R/8D56913D Alex de Joode <adejoode(a)sabotage.org> +vote-digest 3F1F1E071EC5F54115CB8EA9723E30A9386AB8CA +dir-source gabelmoo ED03BB616EB2F60BEC80151114BB25CEF515B226 212.112.245.170 212.112.245.170 80 443 +contact 4096R/C5AA446D Sebastian Hahn <tor(a)sebastianhahn.net> +vote-digest DF2EC9AD207831DED1D01BB889A9C4478DE2CFB9 +r sumkledi ABPSI4nNUNC3hKPkBhyzHozozrU 8mCr8Sl7RF4ENU4jb0FZFA/3do8 2012-07-12 04:01:55 178.218.213.229 80 0 +s Exit Fast Named Running Valid +v Tor 0.2.2.35 +w Bandwidth=38 +p accept 80,443 +r Unnamed AEXri4INxBAZeyi0wvJZoC58nZs 9t4U465paxGASh0x5Tds8a8YiKo 2012-07-12 04:48:09 79.139.135.90 443 22 +s Fast HSDir Running V2Dir Valid +v Tor 0.2.2.37 +w Bandwidth=35 +p reject 1-65535 +r ANONIONROUTER AHhuQ8zFQJdT8l42Axxc6m6kNwI uI7+jQ/T3kFVnl7H7TYE/7WJxi4 2012-07-12 04:40:31 93.128.55.236 24051 24052 +s Fast Named Running V2Dir Valid +v Tor 0.2.2.37 +w Bandwidth=108 +p reject 1-65535 +r ph3x AMLCoWrttR1eX7fWFo/GazQ9gi8 ZJJnmKK6+9B2KOUSIPV49+Vprxs 2012-07-11 19:44:22 86.59.119.83 443 80 +s Fast Guard HSDir Named Running Stable V2Dir Valid +v Tor 0.2.3.18-rc +w Bandwidth=55300 +p reject 1-65535 +r nargothrond ANi/r5RGhUxfZ3simlDXFrf2O68 DsP6REKOns/vAUYNp3rfkCOSJFM 2012-07-11 18:25:37 173.11.83.10 9001 0 +s Fast Guard Named Running Stable Valid +v Tor 0.2.3.18-rc +w Bandwidth=543 +p reject 1-65535 +r default AN1sc6ymJ4WcSJ95VITqL0B5wDQ I9HQ2zph5Nuvf4FKANoKDf5vPV8 2012-07-11 18:48:22 82.243.60.52 443 9030 +s Fast Running V2Dir Valid +v Tor 0.2.2.35 +w Bandwidth=92 +p reject 1-65535 +r catfesh AOTNBUkB8Lob/wiz7h9gtuDoT2Q 0Ycp54MgG+Ns+oEd3BIubFJdGGw 2012-07-12 08:26:51 80.177.151.82 9001 9030 +s Fast HSDir Running V2Dir Valid +v Tor 0.2.2.37 +w Bandwidth=61 +p reject 1-65535 +r 111111 AO70G2tKrAacjjIITtSmzAFrSs8 l/yzMBbM4crBHivPKh69pDXuTRM 2012-07-12 04:03:06 178.170.144.91 80 0 +s Exit Fast Named Running Valid +v Tor 0.2.2.34 +w Bandwidth=29 +p accept 80,443 +r Unnamed AP1onm4+6g+gIQMs1u9r6CeLX80 HY8Ud5ffEX28pglH+Vqvfle1xDQ 2012-07-12 09:09:15 189.41.71.79 443 0 +s Running Valid +v Tor 0.2.2.37 +w Bandwidth=0 +p reject 1-65535 +directory-footer +bandwidth-weights Wbd=3335 Wbe=0 Wbg=3536 Wbm=10000 Wdb=10000 Web=10000 Wed=3329 Wee=10000 Weg=3329 Wem=10000 Wgb=10000 Wgd=3335 Wgg=6464 Wgm=6464 Wmb=10000 Wmd=3335 Wme=0 Wmg=3536 Wmm=10000 +directory-signature 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 BF112F1C6D5543CFD0A32215ACABD4197B5279AD +-----BEGIN SIGNATURE----- +HFXB4497LzESysYJ/4jJY83E5vLjhv+igIxD9LU6lf6ftkGeF+lNmIAIEKaMts8H +mfWcW0b+jsrXcJoCxV5IrwCDF3u1aC3diwZY6yiG186pwWbOwE41188XI2DeYPwE +I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= +-----END SIGNATURE----- +directory-signature 27B6B5996C426270A5C95488AA5BCEB6BCC86956 D5C30C15BB3F1DA27669C2D88439939E8F418FCF +-----BEGIN SIGNATURE----- +VAL+VWqcJiJtTZjFDz5/rS4WLfh8dOSnU2HYUb1ZgqM8PR1rFsoxpvaK9USrtkx9 +Byctu/flD3YOqGg+GpYQwU8w9tm7BGelD+dqg97DkJXmlPaXe/Z0nKW1UnCN9m93 +svyWCAqglEzxlK4H7ZfMlQbkMu7EFjXGzrn1gRVGOwg= +-----END SIGNATURE----- +directory-signature 49015F787433103580E3B66A1707A00E60F2D15B 1C915B9493589F97BAC764D1885A34BFC18C7E26 +-----BEGIN SIGNATURE----- +fHAC5vdqotMtTVIxfqoNrlob2jAi3PP/urvsVA0xmaOzgYtJFIjY2iEWrrU4fRwe +0M1vyCw+oztBrPKYukedkefE9ly/R30KVW2ezo5WpOO4y6oZpelb/jRKFoSRfbyB +WdKsHSe2xlXPA0ySu1klpuMOZiQ8wgxh4x3oLGXnL5Q= +-----END SIGNATURE----- +directory-signature 585769C78764D58426B8B52B6651A5A71137189A 499D7CE5A1356045D629F43271EBF600D6F2CC9C +-----BEGIN SIGNATURE----- +IDOUDykw+tdCyyVmPSGUDahIeEEPMWxarEoH2gPuyExDqZkUc0ah6Eh736rVSD5Z +R4nCjDNTQNr5byDfJk6cMDN9A/5P8uz421pnmLfs9SasLUjTdJt921jxJnSvSBeF +hSZPNi5wl++Uw3j2zeclOXvAkkAEGi9Pi5Zf6QNlWFI= +-----END SIGNATURE----- +directory-signature 80550987E1D626E3EBA5E5E75A458DE0626D088C 2B9B419BB44728A5BE01651D6D1553FD14B6CFFB +-----BEGIN SIGNATURE----- +D2wVGni7cYHyXNqt9RvW/CUd8r7TgkfEp9OAJKojUyweiGMJOMEqDBE01e4Ov9Pd +O9D46fjxWYGE9fN72xvD8CGoNcQgTtLpvypEfB96tKM3JYr5j4MCsdcOrQBkKGp7 +qf1Qfiw7aXahk8IfbgvmAvROlAMAxln7nVE0qenQWu4= +-----END SIGNATURE----- +directory-signature D586D18309DED4CD6D57C18FDB97EFA96D330566 8F0DEA35D1732D867FB94F5130364927DBCCBB8E +-----BEGIN SIGNATURE----- +cmrV1VUfCo9Smlc8BUblkBuSFqJdQkX/gd3ROOnpdRfogbsylq6xA7srCBGp1Z39 +To5Vk71AI0PIy031S6gKfOLgn9E5Bl0Agr60diFxjNn0ejR49MKJTjoDI+DmBlv4 +do+Bej+D8afl27LNH/QIYyzSkOl0eXSGtOEEuCQg/3A= +-----END SIGNATURE----- +directory-signature E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 9BE9A4CF520B6880EB42C5702BC521F03A6CF0FC +-----BEGIN SIGNATURE----- +UVXzEFkkjCpszLmoqQxAxSU83IS+fqrkIC4DCQZCEjRcXEvx3c56HUyTsew5WTFR +XANCJn+V3DaxYLuL6L8xW7r9xOQNU970nGwocuJckxyDcLHloL8E226vIAn6mLmt +a1Z6y8NzaQpv4fhdqhT7ETJo+chmf8bSX8qLLmaCIac= +-----END SIGNATURE----- +directory-signature ED03BB616EB2F60BEC80151114BB25CEF515B226 845CF1D0B370CA443A8579D18E7987E7E532F639 +-----BEGIN SIGNATURE----- +DILsRCrtn6rDbNo3DF+L1/VVAd+V86PdZKg3Q9QooqVOGgU/7HrspV/K4lFbWcTT +Zm+quRQfuKmB4xljwXpeRlABQR5eainlZBtrTFg056/dDrJqYXSwV/C391tAIDZs +2TANs/4uLi94q6Ov+zE9zYUiF8jwnyXl/q/jKOYM8bE= +-----END SIGNATURE----- diff --git a/test/integ/descriptor/data/consensus b/test/integ/descriptor/data/consensus deleted file mode 100644 index adcf13b..0000000 --- a/test/integ/descriptor/data/consensus +++ /dev/null @@ -1,130 +0,0 @@ -network-status-version 3 -vote-status consensus -consensus-method 12 -valid-after 2012-07-12 10:00:00 -fresh-until 2012-07-12 11:00:00 -valid-until 2012-07-12 13:00:00 -voting-delay 300 300 -client-versions 0.2.2.35,0.2.2.36,0.2.2.37,0.2.3.10-alpha,0.2.3.11-alpha,0.2.3.12-alpha,0.2.3.13-alpha,0.2.3.14-alpha,0.2.3.15-alpha,0.2.3.16-alpha,0.2.3.17-beta,0.2.3.18-rc,0.2.3.19-rc -server-versions 0.2.2.35,0.2.2.36,0.2.2.37,0.2.3.10-alpha,0.2.3.11-alpha,0.2.3.12-alpha,0.2.3.13-alpha,0.2.3.14-alpha,0.2.3.15-alpha,0.2.3.16-alpha,0.2.3.17-beta,0.2.3.18-rc,0.2.3.19-rc -known-flags Authority BadExit Exit Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid -params CircuitPriorityHalflifeMsec=30000 bwauthpid=1 -dir-source tor26 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 86.59.21.38 86.59.21.38 80 443 -contact Peter Palfrader -vote-digest 0B6D1E9A300B895AA2D0B427F92917B6995C3C1C -dir-source turtles 27B6B5996C426270A5C95488AA5BCEB6BCC86956 76.73.17.194 76.73.17.194 9030 9090 -contact Mike Perry <mikeperryTAfsckedTODorg> -vote-digest 904B1974B9879D02B4ADFB81D7E9B4E07D768A5A -dir-source maatuska 49015F787433103580E3B66A1707A00E60F2D15B 171.25.193.9 171.25.193.9 443 80 -contact 4096R/23291265 Linus Nordberg <linus(a)nordberg.se> -vote-digest A8839355BAC373320B8CEDD0A0A09DAAA1637E3A -dir-source dannenberg 585769C78764D58426B8B52B6651A5A71137189A dannenberg.ccc.de 193.23.244.244 80 443 -contact Andreas Lehner <anonymizer(a)ccc.de> -vote-digest 416B73C49E717B0A5D61A4F634DCCF94611802E7 -dir-source urras 80550987E1D626E3EBA5E5E75A458DE0626D088C 208.83.223.34 208.83.223.34 443 80 -contact 4096R/E012B42D Jacob Appelbaum <jacob(a)appelbaum.net> -vote-digest 08B1F8E4910F136E7FB7DFD52ABB2A9EDE939F0B -dir-source moria1 D586D18309DED4CD6D57C18FDB97EFA96D330566 128.31.0.34 128.31.0.34 9131 9101 -contact 1024D/28988BF5 arma mit edu -vote-digest 5006931FB78F7AE42B602697591DBA82AACEF533 -dir-source dizum E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 194.109.206.212 194.109.206.212 80 443 -contact 1024R/8D56913D Alex de Joode <adejoode(a)sabotage.org> -vote-digest 3F1F1E071EC5F54115CB8EA9723E30A9386AB8CA -dir-source gabelmoo ED03BB616EB2F60BEC80151114BB25CEF515B226 212.112.245.170 212.112.245.170 80 443 -contact 4096R/C5AA446D Sebastian Hahn <tor(a)sebastianhahn.net> -vote-digest DF2EC9AD207831DED1D01BB889A9C4478DE2CFB9 -r sumkledi ABPSI4nNUNC3hKPkBhyzHozozrU 8mCr8Sl7RF4ENU4jb0FZFA/3do8 2012-07-12 04:01:55 178.218.213.229 80 0 -s Exit Fast Named Running Valid -v Tor 0.2.2.35 -w Bandwidth=38 -p accept 80,443 -r Unnamed AEXri4INxBAZeyi0wvJZoC58nZs 9t4U465paxGASh0x5Tds8a8YiKo 2012-07-12 04:48:09 79.139.135.90 443 22 -s Fast HSDir Running V2Dir Valid -v Tor 0.2.2.37 -w Bandwidth=35 -p reject 1-65535 -r ANONIONROUTER AHhuQ8zFQJdT8l42Axxc6m6kNwI uI7+jQ/T3kFVnl7H7TYE/7WJxi4 2012-07-12 04:40:31 93.128.55.236 24051 24052 -s Fast Named Running V2Dir Valid -v Tor 0.2.2.37 -w Bandwidth=108 -p reject 1-65535 -r ph3x AMLCoWrttR1eX7fWFo/GazQ9gi8 ZJJnmKK6+9B2KOUSIPV49+Vprxs 2012-07-11 19:44:22 86.59.119.83 443 80 -s Fast Guard HSDir Named Running Stable V2Dir Valid -v Tor 0.2.3.18-rc -w Bandwidth=55300 -p reject 1-65535 -r nargothrond ANi/r5RGhUxfZ3simlDXFrf2O68 DsP6REKOns/vAUYNp3rfkCOSJFM 2012-07-11 18:25:37 173.11.83.10 9001 0 -s Fast Guard Named Running Stable Valid -v Tor 0.2.3.18-rc -w Bandwidth=543 -p reject 1-65535 -r default AN1sc6ymJ4WcSJ95VITqL0B5wDQ I9HQ2zph5Nuvf4FKANoKDf5vPV8 2012-07-11 18:48:22 82.243.60.52 443 9030 -s Fast Running V2Dir Valid -v Tor 0.2.2.35 -w Bandwidth=92 -p reject 1-65535 -r catfesh AOTNBUkB8Lob/wiz7h9gtuDoT2Q 0Ycp54MgG+Ns+oEd3BIubFJdGGw 2012-07-12 08:26:51 80.177.151.82 9001 9030 -s Fast HSDir Running V2Dir Valid -v Tor 0.2.2.37 -w Bandwidth=61 -p reject 1-65535 -r 111111 AO70G2tKrAacjjIITtSmzAFrSs8 l/yzMBbM4crBHivPKh69pDXuTRM 2012-07-12 04:03:06 178.170.144.91 80 0 -s Exit Fast Named Running Valid -v Tor 0.2.2.34 -w Bandwidth=29 -p accept 80,443 -r Unnamed AP1onm4+6g+gIQMs1u9r6CeLX80 HY8Ud5ffEX28pglH+Vqvfle1xDQ 2012-07-12 09:09:15 189.41.71.79 443 0 -s Running Valid -v Tor 0.2.2.37 -w Bandwidth=0 -p reject 1-65535 -directory-footer -bandwidth-weights Wbd=3335 Wbe=0 Wbg=3536 Wbm=10000 Wdb=10000 Web=10000 Wed=3329 Wee=10000 Weg=3329 Wem=10000 Wgb=10000 Wgd=3335 Wgg=6464 Wgm=6464 Wmb=10000 Wmd=3335 Wme=0 Wmg=3536 Wmm=10000 -directory-signature 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 BF112F1C6D5543CFD0A32215ACABD4197B5279AD ------BEGIN SIGNATURE----- -HFXB4497LzESysYJ/4jJY83E5vLjhv+igIxD9LU6lf6ftkGeF+lNmIAIEKaMts8H -mfWcW0b+jsrXcJoCxV5IrwCDF3u1aC3diwZY6yiG186pwWbOwE41188XI2DeYPwE -I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= ------END SIGNATURE----- -directory-signature 27B6B5996C426270A5C95488AA5BCEB6BCC86956 D5C30C15BB3F1DA27669C2D88439939E8F418FCF ------BEGIN SIGNATURE----- -VAL+VWqcJiJtTZjFDz5/rS4WLfh8dOSnU2HYUb1ZgqM8PR1rFsoxpvaK9USrtkx9 -Byctu/flD3YOqGg+GpYQwU8w9tm7BGelD+dqg97DkJXmlPaXe/Z0nKW1UnCN9m93 -svyWCAqglEzxlK4H7ZfMlQbkMu7EFjXGzrn1gRVGOwg= ------END SIGNATURE----- -directory-signature 49015F787433103580E3B66A1707A00E60F2D15B 1C915B9493589F97BAC764D1885A34BFC18C7E26 ------BEGIN SIGNATURE----- -fHAC5vdqotMtTVIxfqoNrlob2jAi3PP/urvsVA0xmaOzgYtJFIjY2iEWrrU4fRwe -0M1vyCw+oztBrPKYukedkefE9ly/R30KVW2ezo5WpOO4y6oZpelb/jRKFoSRfbyB -WdKsHSe2xlXPA0ySu1klpuMOZiQ8wgxh4x3oLGXnL5Q= ------END SIGNATURE----- -directory-signature 585769C78764D58426B8B52B6651A5A71137189A 499D7CE5A1356045D629F43271EBF600D6F2CC9C ------BEGIN SIGNATURE----- -IDOUDykw+tdCyyVmPSGUDahIeEEPMWxarEoH2gPuyExDqZkUc0ah6Eh736rVSD5Z -R4nCjDNTQNr5byDfJk6cMDN9A/5P8uz421pnmLfs9SasLUjTdJt921jxJnSvSBeF -hSZPNi5wl++Uw3j2zeclOXvAkkAEGi9Pi5Zf6QNlWFI= ------END SIGNATURE----- -directory-signature 80550987E1D626E3EBA5E5E75A458DE0626D088C 2B9B419BB44728A5BE01651D6D1553FD14B6CFFB ------BEGIN SIGNATURE----- -D2wVGni7cYHyXNqt9RvW/CUd8r7TgkfEp9OAJKojUyweiGMJOMEqDBE01e4Ov9Pd -O9D46fjxWYGE9fN72xvD8CGoNcQgTtLpvypEfB96tKM3JYr5j4MCsdcOrQBkKGp7 -qf1Qfiw7aXahk8IfbgvmAvROlAMAxln7nVE0qenQWu4= ------END SIGNATURE----- -directory-signature D586D18309DED4CD6D57C18FDB97EFA96D330566 8F0DEA35D1732D867FB94F5130364927DBCCBB8E ------BEGIN SIGNATURE----- -cmrV1VUfCo9Smlc8BUblkBuSFqJdQkX/gd3ROOnpdRfogbsylq6xA7srCBGp1Z39 -To5Vk71AI0PIy031S6gKfOLgn9E5Bl0Agr60diFxjNn0ejR49MKJTjoDI+DmBlv4 -do+Bej+D8afl27LNH/QIYyzSkOl0eXSGtOEEuCQg/3A= ------END SIGNATURE----- -directory-signature E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 9BE9A4CF520B6880EB42C5702BC521F03A6CF0FC ------BEGIN SIGNATURE----- -UVXzEFkkjCpszLmoqQxAxSU83IS+fqrkIC4DCQZCEjRcXEvx3c56HUyTsew5WTFR -XANCJn+V3DaxYLuL6L8xW7r9xOQNU970nGwocuJckxyDcLHloL8E226vIAn6mLmt -a1Z6y8NzaQpv4fhdqhT7ETJo+chmf8bSX8qLLmaCIac= ------END SIGNATURE----- -directory-signature ED03BB616EB2F60BEC80151114BB25CEF515B226 845CF1D0B370CA443A8579D18E7987E7E532F639 ------BEGIN SIGNATURE----- -DILsRCrtn6rDbNo3DF+L1/VVAd+V86PdZKg3Q9QooqVOGgU/7HrspV/K4lFbWcTT -Zm+quRQfuKmB4xljwXpeRlABQR5eainlZBtrTFg056/dDrJqYXSwV/C391tAIDZs -2TANs/4uLi94q6Ov+zE9zYUiF8jwnyXl/q/jKOYM8bE= ------END SIGNATURE----- diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index a634e54..67992c7 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -71,7 +71,7 @@ class TestNetworkStatusDocument(unittest.TestCase): Checks that consensus documents are properly parsed. """ - descriptor_path = test.integ.descriptor.get_resource("consensus") + descriptor_path = test.integ.descriptor.get_resource("cached-consensus") descriptor_file = file(descriptor_path) desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index 03fb8fe..6ec538d 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -355,7 +355,7 @@ class TestDescriptorReader(unittest.TestCase): for skip_path, skip_exception in skip_listener.results: if skip_path.endswith(".swp"): continue # skip vim temp files - if not os.path.basename(skip_path) in ("riddle", "tiny.png"): + if not os.path.basename(skip_path) in ("riddle", "tiny.png", "vote"): self.fail("Unexpected non-descriptor content: %s" % skip_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))

1 0

[stem/master] Remove trivial comment
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 9df15326fecd223bf7d4e5a40465a2fa80069025 Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Tue Aug 7 15:01:49 2012 +0530 Remove trivial comment --- stem/descriptor/networkstatus.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 8daf6f7..3ab5739 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -587,7 +587,6 @@ class RouterDescriptor(stem.descriptor.Descriptor): # "p" SP ("accept" / "reject") SP PortList NL if p: self.exit_policy = stem.exit_policy.MicrodescriptorExitPolicy(p) - #self.exit_policy = p elif vote and peek_check_kw("m"): # microdescriptor hashes

1 0

[stem/master] Fix incorrect docstring and reduce 1 LoC elsewhere
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 9e6fb6d9d99144832b78e5cf1b28fc4232d17c15 Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Tue Aug 7 13:25:43 2012 +0530 Fix incorrect docstring and reduce 1 LoC elsewhere --- stem/descriptor/networkstatus.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 988c71c..a9b23e2 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -62,15 +62,14 @@ def parse_file(document_file, validate = True): :param file document_file: file with network status document content :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise - :returns: iterator for :class:`stem.descriptor.networkstatus_descriptor.RouterDescriptor` instances in the file + :returns: iterator for :class:`stem.descriptor.networkstatus.RouterDescriptor` instances in the file :raises: * ValueError if the contents is malformed and validate is True * IOError if the file can't be read """ - document = NetworkStatusDocument(document_file.read(), validate) - return document.router_descriptors + return NetworkStatusDocument(document_file.read(), validate).router_descriptors def _strptime(string, validate = True, optional = False): try:

1 0

[stem/master] Update comments
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit a288e1e43f2cecc55b1b46cbfab6734aee1a5395 Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Tue Aug 7 13:10:49 2012 +0530 Update comments --- stem/descriptor/networkstatus.py | 33 +++++++++++++++++++++++---------- 1 files changed, 23 insertions(+), 10 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 3ab5739..876acd9 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -1,27 +1,41 @@ """ Parsing for Tor network status documents. Currently supports parsing v3 network -status documents (both votes and consensus'). +status documents (both votes and consensuses). The network status documents also contain a list of router descriptors, -directory authorities, signatures etc. +directory authorities, signatures etc. If you only need the +:class:`stem.descriptor.networkstatus.RouterDescriptor` objects, use +:func:`stem.descriptor.parse_file`. Other information can be accessed by +directly instantiating :class:`stem.descriptor.networkstatus.NetworkStatusDocument` +objects. -The votes and consensus' can be obtained from any of the following sources... +The documents can be obtained from any of the following sources... * the 'cached-consensus' file in tor's data directory * tor metrics, at https://metrics.torproject.org/data.html * directory authorities and mirrors via their DirPort +:: + + import stem.descriptor.networkstatus + + nsdoc_file = open("/home/neena/.tor/cached-consensus") + try: + consensus = stem.descriptor.networkstatus.NetworkStatusDocument(nsdoc_file.read()) + except ValueError: + print "Invalid cached-consensus file" + + print "Consensus was valid between %s and %s" % (str(consensus.valid_after), str(consensus.valid_until)) + **Module Overview:** :: parse_file - parses a network status file and provides a NetworkStatusDocument NetworkStatusDocument - Tor v3 network status document - +- MicrodescriptorConsensus - Tor microdescriptor consensus document RouterDescriptor - Router descriptor; contains information about a Tor relay - +- RouterMicrodescriptor - Router microdescriptor; contains information that doesn't change often - DirectorySignature - DirectoryAuthority + DirectorySignature - Network status document's directory signature + DirectoryAuthority - Directory authority defined in a v3 network status document """ import re @@ -85,7 +99,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :var list server_versions: list of recommended Tor server versions :var list known_flags: **\*** list of known router flags :var list params: dict of parameter(str) => value(int) mappings - :var list router_descriptors: **\*** list of RouterDescriptor objects defined in the document + :var list router_descriptors: **\*** iterator for RouterDescriptor objects defined in the document :var list directory_authorities: **\*** list of DirectoryAuthority objects that have generated this document :var dict bandwidth_weights: dict of weight(str) => value(int) mappings :var list directory_signatures: **\*** list of signatures this document has @@ -374,8 +388,7 @@ class KeyCertificate(stem.descriptor.Descriptor): class DirectorySignature(stem.descriptor.Descriptor): """ - Contains directory signature information described in a v3 network status - document. + Contains directory signatures in a v3 network status document. :var str identity: signature identity :var str key_digest: signature key digest

1 0

[stem/master] Fix incorrect stem.descriptor.networkstatus import
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit cba98e26f16214925f7f5e6d61b51767da32b66e Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Tue Aug 7 21:24:30 2012 +0530 Fix incorrect stem.descriptor.networkstatus import --- stem/descriptor/__init__.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index f5da97a..8193380 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -48,7 +48,7 @@ def parse_file(path, descriptor_file): import stem.descriptor.server_descriptor import stem.descriptor.extrainfo_descriptor - import stem.descriptor.networkstatus_descriptor + import stem.descriptor.networkstatus # The tor descriptor specifications do not provide a reliable method for # identifying a descriptor file's type and version so we need to guess @@ -98,7 +98,7 @@ def parse_file(path, descriptor_file): desc = stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(descriptor_file.read()) elif desc_type in ("network-status-consensus-3", "network-status-vote-3") and major_version == 1: - desc = stem.descriptor.networkstatus_descriptor.NetworkStatusDocument(descriptor_file.read()) + desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read()) for desc in desc.router_descriptors: desc._set_path(path) yield desc

1 0

[stem/master] More documentation
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit e7cc83b4e382d209dda4660b5d30b9691f3fd713 Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Tue Aug 7 13:48:24 2012 +0530 More documentation --- stem/descriptor/networkstatus.py | 51 ++++++++++++++++++++++++++++--------- 1 files changed, 38 insertions(+), 13 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index a9b23e2..c9b86a1 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -62,7 +62,7 @@ def parse_file(document_file, validate = True): :param file document_file: file with network status document content :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise - :returns: iterator for :class:`stem.descriptor.networkstatus.RouterDescriptor` instances in the file + :returns: iterator for :class:`stem.descriptor.networkstatus.RouterDescriptor` instances in the file :raises: * ValueError if the contents is malformed and validate is True @@ -86,9 +86,9 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :var bool validated: **\*** whether the document is validated :var str network_status_version: **\*** a document format version. For v3 documents this is "3" :var str vote_status: **\*** status of the vote (is either "vote" or "consensus") - :var list consensus_methods: A list of supported consensus generation methods (integers) - :var datetime published: time when the document was published - :var int consensus_method: consensus method used to generate a consensus + :var list consensus_methods: **^** A list of supported consensus generation methods (integers) + :var datetime published: **^** time when the document was published + :var int consensus_method: **~** consensus method used to generate a consensus :var datetime valid_after: **\*** time when the consensus becomes valid :var datetime fresh_until: **\*** time until when the consensus is considered to be fresh :var datetime valid_until: **\*** time until when the consensus is valid @@ -100,10 +100,12 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): :var list params: dict of parameter(str) => value(int) mappings :var list router_descriptors: **\*** iterator for RouterDescriptor objects defined in the document :var list directory_authorities: **\*** list of DirectoryAuthority objects that have generated this document - :var dict bandwidth_weights: dict of weight(str) => value(int) mappings + :var dict bandwidth_weights: **~** dict of weight(str) => value(int) mappings :var list directory_signatures: **\*** list of signatures this document has - **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + | **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + | **^** attribute appears only in votes + | **~** attribute appears only in consensuses """ def __init__(self, raw_content, validate = True): @@ -247,9 +249,13 @@ class DirectoryAuthority(stem.descriptor.Descriptor): :var int dirport: current directory port :var int orport: current orport :var str contact: directory authority's contact information - :var str legacy_dir_key: fingerprint of and obsolete identity key - :var :class:`stem.descriptor.KeyCertificate` key_certificate: directory authority's current key certificate - :var str vote_digest: digest of the authority that contributed to the consensus + :var str legacy_dir_key: **^** fingerprint of and obsolete identity key + :var :class:`stem.descriptor.KeyCertificate` key_certificate: **^** directory authority's current key certificate + :var str vote_digest: **~** digest of the authority that contributed to the consensus + + | **^** attribute appears only in votes + | **~** attribute appears only in consensuses + | legacy_dir_key is the only optional attribute """ def __init__(self, raw_content, vote = True, validate = True): @@ -280,9 +286,18 @@ class DirectoryAuthority(stem.descriptor.Descriptor): self.key_certificate = stem.descriptor.KeyCertificate("\n".join(parser.remaining()), validate) else: self.vote_digest = parser.read_keyword_line("vote-digest", True) - rmng = parser.remaining() - if rmng and validate: + self._unrecognized_lines = parser.remaining() + if self._unrecognized_lines and validate: raise ValueError("Unrecognized trailing data in directory authority information") + + def get_unrecognized_lines(self): + """ + Returns any unrecognized lines. + + :returns: a list of unrecognized lines + """ + + return self._unrecognized_lines class DirectorySignature(stem.descriptor.Descriptor): """ @@ -320,6 +335,15 @@ class DirectorySignature(stem.descriptor.Descriptor): self._unrecognized_lines = parser.remaining() if self._unrecognized_lines and validate: raise ValueError("Unrecognized trailing data in directory signature") + + def get_unrecognized_lines(self): + """ + Returns any unrecognized lines. + + :returns: a list of unrecognized lines + """ + + return self._unrecognized_lines class RouterDescriptor(stem.descriptor.Descriptor): """ @@ -354,11 +378,12 @@ class RouterDescriptor(stem.descriptor.Descriptor): :var int bandwidth: router's claimed bandwidth :var int measured_bandwidth: router's measured bandwidth - :var :class:`stem.exit_policy.MicrodescriptorExitPolicy` exitpolicy: router's exitpolicy + :var :class:`stem.exit_policy.MicrodescriptorExitPolicy` exit_policy: router's exit policy :var str microdescriptor_hashes: a list of two-tuples with a list of consensus methods(int) that may produce the digest and a dict with algorithm(str) => digest(str) mappings. algorithm is the hashing algorithm (usually "sha256") that is used to produce digest (the base64 encoding of the hash of the router's microdescriptor with trailing =s omitted). - **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + | **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + | exit_policy appears only in votes """ def __init__(self, raw_contents, vote = True, validate = True):

1 0

[stem/master] stem.descriptor.networkstatus.KeyCertificate => stem.descriptor.KeyCertificate
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 92f46907b3d84c53ab10c8f19851d65be86f7b9f Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Tue Aug 7 13:19:57 2012 +0530 stem.descriptor.networkstatus.KeyCertificate => stem.descriptor.KeyCertificate From section 4.7 the dir-spec, A concatenated set of all the current key certificates should be available at: http://<hostname>/tor/keys/all.z The key certificate for this server (if it is an authority) should be available at: http://<hostname>/tor/keys/authority.z ... we might want to parse these seperately at some point, so, moving it out of stem.descriptor.networkstatus --- stem/descriptor/__init__.py | 108 ++++++++++++++++++++++++++++++++++++++ stem/descriptor/networkstatus.py | 105 +------------------------------------ 2 files changed, 110 insertions(+), 103 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 5f9d144..f5da97a 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -24,6 +24,7 @@ __all__ = [ import os import re +import datetime KEYWORD_CHAR = "a-zA-Z0-9-" WHITESPACE = " \t" @@ -283,6 +284,12 @@ def _get_descriptor_components(raw_contents, validate, extra_keywords): return entries, first_keyword, last_keyword, extra_entries +def _strptime(string, validate = True, optional = False): + try: + return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") + except ValueError, exc: + if validate or not optional: raise exc + class DescriptorParser: """ Helper class to parse documents. @@ -423,3 +430,104 @@ class DescriptorParser: else: return [] +class KeyCertificate(Descriptor): + """ + Directory key certificate. + + :var str key_certificate_version: **\*** version of the key certificate (Should be "3") + :var str ip: IP address on which the directory authority is listening + :var int port: port on which the directory authority is listening + :var str fingerprint: **\*** hex encoded fingerprint of the authority's identity key + :var str identity_key: **\*** long term authority identity key + :var datetime published: **\*** time (in GMT) when this document & the key were last generated + :var str expires: **\*** time (in GMT) after which this key becomes invalid + :var str signing_key: **\*** directory server's public signing key + :var str crosscert: signature made using certificate's signing key + :var str certification: **\*** signature of this key certificate signed with the identity key + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, raw_content, validate = True): + """ + Parse a key certificate entry and provide a KeyCertificate object. + + :param str raw_content: raw key certificate information + :param bool validate: True if the document is to be validated, False otherwise + + :raises: ValueError if the raw data is invalid + """ + + super(KeyCertificate, self).__init__(raw_content) + self.key_certificate_version, self.ip, self.port = None, None, None + self.fingerprint, self.identity_key, self.published = None, None, None + self.expires, self.signing_key, self.crosscert = None, None, None + self.certification = None + parser = DescriptorParser(raw_content, validate) + peek_check_kw = lambda keyword: keyword == parser.peek_keyword() + seen_keywords = set() + + self.key_certificate_version = parser.read_keyword_line("dir-key-certificate-version") + if validate and self.key_certificate_version != "3": raise ValueError("Unrecognized dir-key-certificate-version") + + def _read_keyword_line(keyword): + if validate and keyword in seen_keywords: + raise ValueError("Invalid key certificate: '%s' appears twice" % keyword) + seen_keywords.add(keyword) + return parser.read_keyword_line(keyword) + + while parser.line: + if peek_check_kw("dir-address"): + line = _read_keyword_line("dir-address") + try: + self.ip, self.port = line.rsplit(":", 1) + self.port = int(self.port) + except Exception: + if validate: raise ValueError("Invalid dir-address line: %s" % line) + + elif peek_check_kw("fingerprint"): + self.fingerprint = _read_keyword_line("fingerprint") + + elif peek_check_kw("dir-identity-key"): + _read_keyword_line("dir-identity-key") + self.identity_key = parser.read_block("RSA PUBLIC KEY") + + elif peek_check_kw("dir-key-published"): + self.published = _strptime(_read_keyword_line("dir-key-published")) + + elif peek_check_kw("dir-key-expires"): + self.expires = _strptime(_read_keyword_line("dir-key-expires")) + + elif peek_check_kw("dir-signing-key"): + _read_keyword_line("dir-signing-key") + self.signing_key = parser.read_block("RSA PUBLIC KEY") + + elif peek_check_kw("dir-key-crosscert"): + _read_keyword_line("dir-key-crosscert") + self.crosscert = parser.read_block("ID SIGNATURE") + + elif peek_check_kw("dir-key-certification"): + _read_keyword_line("dir-key-certification") + self.certification = parser.read_block("SIGNATURE") + break + + elif validate: + raise ValueError("Key certificate contains unrecognized lines: %s" % parser.line) + + else: + # ignore unrecognized lines if we aren't validating + self._unrecognized_lines.append(parser.read_line()) + + self._unrecognized_lines = parser.remaining() + if self._unrecognized_lines and validate: + raise ValueError("Unrecognized trailing data in key certificate") + + def get_unrecognized_lines(self): + """ + Returns any unrecognized lines. + + :returns: a list of unrecognized lines + """ + + return self._unrecognized_lines + diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 876acd9..988c71c 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -249,7 +249,7 @@ class DirectoryAuthority(stem.descriptor.Descriptor): :var int orport: current orport :var str contact: directory authority's contact information :var str legacy_dir_key: fingerprint of and obsolete identity key - :var :class:`stem.descriptor.networkstatus_descriptor.KeyCertificate` key_certificate: directory authority's current key certificate + :var :class:`stem.descriptor.KeyCertificate` key_certificate: directory authority's current key certificate :var str vote_digest: digest of the authority that contributed to the consensus """ @@ -278,114 +278,13 @@ class DirectoryAuthority(stem.descriptor.Descriptor): self.contact = parser.read_keyword_line("contact") if vote: self.legacy_dir_key = parser.read_keyword_line("legacy-dir-key", True) - self.key_certificate = KeyCertificate("\n".join(parser.remaining()), validate) + self.key_certificate = stem.descriptor.KeyCertificate("\n".join(parser.remaining()), validate) else: self.vote_digest = parser.read_keyword_line("vote-digest", True) rmng = parser.remaining() if rmng and validate: raise ValueError("Unrecognized trailing data in directory authority information") -class KeyCertificate(stem.descriptor.Descriptor): - """ - Directory key certificate. - - :var str key_certificate_version: **\*** version of the key certificate (Should be "3") - :var str ip: IP address on which the directory authority is listening - :var int port: port on which the directory authority is listening - :var str fingerprint: **\*** hex encoded fingerprint of the authority's identity key - :var str identity_key: **\*** long term authority identity key - :var datetime published: **\*** time (in GMT) when this document & the key were last generated - :var str expires: **\*** time (in GMT) after which this key becomes invalid - :var str signing_key: **\*** directory server's public signing key - :var str crosscert: signature made using certificate's signing key - :var str certification: **\*** signature of this key certificate signed with the identity key - - **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined - """ - - def __init__(self, raw_content, validate = True): - """ - Parse a key certificate entry and provide a KeyCertificate object. - - :param str raw_content: raw key certificate information - :param bool validate: True if the document is to be validated, False otherwise - - :raises: ValueError if the raw data is invalid - """ - - super(KeyCertificate, self).__init__(raw_content) - self.key_certificate_version, self.ip, self.port = None, None, None - self.fingerprint, self.identity_key, self.published = None, None, None - self.expires, self.signing_key, self.crosscert = None, None, None - self.certification = None - parser = stem.descriptor.DescriptorParser(raw_content, validate) - peek_check_kw = lambda keyword: keyword == parser.peek_keyword() - seen_keywords = set() - - self.key_certificate_version = parser.read_keyword_line("dir-key-certificate-version") - if validate and self.key_certificate_version != "3": raise ValueError("Unrecognized dir-key-certificate-version") - - def _read_keyword_line(keyword): - if validate and keyword in seen_keywords: - raise ValueError("Invalid key certificate: '%s' appears twice" % keyword) - seen_keywords.add(keyword) - return parser.read_keyword_line(keyword) - - while parser.line: - if peek_check_kw("dir-address"): - line = _read_keyword_line("dir-address") - try: - self.ip, self.port = line.rsplit(":", 1) - self.port = int(self.port) - except Exception: - if validate: raise ValueError("Invalid dir-address line: %s" % line) - - elif peek_check_kw("fingerprint"): - self.fingerprint = _read_keyword_line("fingerprint") - - elif peek_check_kw("dir-identity-key"): - _read_keyword_line("dir-identity-key") - self.identity_key = parser.read_block("RSA PUBLIC KEY") - - elif peek_check_kw("dir-key-published"): - self.published = _strptime(_read_keyword_line("dir-key-published")) - - elif peek_check_kw("dir-key-expires"): - self.expires = _strptime(_read_keyword_line("dir-key-expires")) - - elif peek_check_kw("dir-signing-key"): - _read_keyword_line("dir-signing-key") - self.signing_key = parser.read_block("RSA PUBLIC KEY") - - elif peek_check_kw("dir-key-crosscert"): - _read_keyword_line("dir-key-crosscert") - self.crosscert = parser.read_block("ID SIGNATURE") - - elif peek_check_kw("dir-key-certification"): - _read_keyword_line("dir-key-certification") - self.certification = parser.read_block("SIGNATURE") - break - - elif validate: - raise ValueError("Key certificate contains unrecognized lines: %s" % parser.line) - - else: - # ignore unrecognized lines if we aren't validating - self._unrecognized_lines.append(parser.read_line()) - - self._unrecognized_lines = parser.remaining() - if self._unrecognized_lines and validate: - raise ValueError("Unrecognized trailing data in key certificate") - - def get_unrecognized_lines(self): - """ - Returns any unrecognized lines. - - :returns: a list of unrecognized lines - """ - - return self._unrecognized_lines - class DirectorySignature(stem.descriptor.Descriptor): """ Contains directory signatures in a v3 network status document.

1 0

[stem/master] Remove unnecessary imports
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 5f61f4fe91014dbe44e5acc8facd69e0c3467797 Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Tue Aug 7 21:39:57 2012 +0530 Remove unnecessary imports --- stem/descriptor/networkstatus.py | 7 ------- 1 files changed, 0 insertions(+), 7 deletions(-) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index c9b86a1..e4cfda1 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -39,18 +39,11 @@ The documents can be obtained from any of the following sources... """ import re -import base64 -import hashlib import datetime -import stem.prereq import stem.descriptor -import stem.descriptor.extrainfo_descriptor import stem.version import stem.exit_policy -import stem.util.log as log -import stem.util.connection -import stem.util.tor_tools _bandwidth_weights_regex = re.compile(" ".join(["W%s=\d+" % weight for weight in ["bd", "be", "bg", "bm", "db", "eb", "ed", "ee", "eg", "em", "gb", "gd", "gg", "gm", "mb", "md", "me", "mg", "mm"]]))

1 0

[stem/master] Implement lazy router descriptor reading
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit eb0e424ed9459188b70f33ff401e23e9fd89138b Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Wed Aug 8 12:39:55 2012 +0530 Implement lazy router descriptor reading --- stem/descriptor/__init__.py | 60 +++++++++++++++++++++++++++++++++++++- stem/descriptor/networkstatus.py | 36 ++++++++++++++++------- 2 files changed, 84 insertions(+), 12 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index b1f3ab6..168b357 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -148,7 +148,31 @@ class Descriptor(object): def __str__(self): return self._raw_contents -def _read_until_keywords(keywords, descriptor_file, inclusive = False): +def _peek_keyword(descriptor_file): + """ + Returns the keyword at the current offset of descriptor_file. Respects the + "opt" keyword and returns the next keyword instead. + + :param file descriptor_file: file with the descriptor content + + :returns: keyword at the current offset of descriptor_file + """ + + last_position = descriptor_file.tell() + line = descriptor_file.readline() + if not line: return None + + if " " in line: + keyword = line.split(" ", 1)[0] + if keyword == "opt": + keyword = line.split(" ", 2)[1] + else: keyword = line.strip() + + descriptor_file.seek(last_position) + + return keyword + +def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False): """ Reads from the descriptor file until we get to one of the given keywords or reach the end of the file. @@ -156,6 +180,7 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False): :param str,list keywords: keyword(s) we want to read until :param file descriptor_file: file with the descriptor content :param bool inclusive: includes the line with the keyword if True + :param bool ignore_first: doesn't check if the first line read has one of the given keywords :returns: list with the lines until we find one of the keywords """ @@ -163,6 +188,10 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False): content = [] if type(keywords) == str: keywords = (keywords,) + if ignore_first: + content.append(descriptor_file.readline()) + if content == [None]: return [] + while True: last_position = descriptor_file.tell() line = descriptor_file.readline() @@ -181,6 +210,35 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False): return content +def _skip_until_keywords(keywords, descriptor_file, inclusive = False): + """ + Reads and discards lines of data from the descriptor file until we get to one + of the given keywords or reach the end of the file. + + :param str,list keywords: keyword(s) we want to skip until + :param file descriptor_file: file with the descriptor content + :param bool inclusive: includes the line with the keyword if True + + :returns: descriptor_file with the new offset + """ + + if type(keywords) == str: keywords = (keywords,) + + while True: + last_position = descriptor_file.tell() + line = descriptor_file.readline() + if not line: break # EOF + + if " " in line: line_keyword = line.split(" ", 1)[0] + else: line_keyword = line.strip() + + if line_keyword in keywords: + if not inclusive: descriptor_file.seek(last_position) + + break + + return descriptor_file + def _get_pseudo_pgp_block(remaining_contents): """ Checks if given contents begins with a pseudo-Open-PGP-style block and, if diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index e4cfda1..a51fcbd 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -40,14 +40,19 @@ The documents can be obtained from any of the following sources... import re import datetime +from StringIO import StringIO import stem.descriptor import stem.version import stem.exit_policy +from stem.descriptor import _read_until_keywords, _skip_until_keywords, _peek_keyword + _bandwidth_weights_regex = re.compile(" ".join(["W%s=\d+" % weight for weight in ["bd", "be", "bg", "bm", "db", "eb", "ed", "ee", "eg", "em", "gb", "gd", "gg", "gm", "mb", "md", "me", "mg", "mm"]])) +_router_desc_end_kws = ["r", "bandwidth-weights", "directory-footer", "directory-signature"] + def parse_file(document_file, validate = True): """ Iterates over the router descriptors in a network status document. @@ -62,13 +67,30 @@ def parse_file(document_file, validate = True): * IOError if the file can't be read """ - return NetworkStatusDocument(document_file.read(), validate).router_descriptors + # parse until "r" + document_data = "".join(_read_until_keywords("r", document_file)) + # store offset + r_offset = document_file.tell() + # skip until end of router descriptors + _skip_until_keywords(["bandwidth-weights", "directory-footer", "directory-signature"], document_file) + # parse until end + document_data = document_data + document_file.read() + document = NetworkStatusDocument(document_data, validate) + document_file.seek(r_offset) + document.router_descriptors = _router_desc_generator(document_file, document.vote_status == "vote", validate) + return document.router_descriptors def _strptime(string, validate = True, optional = False): try: return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") except ValueError, exc: if validate or not optional: raise exc + else: return None + +def _router_desc_generator(document_file, vote, validate): + while _peek_keyword(document_file) == "r": + desc_content = "".join(_read_until_keywords(_router_desc_end_kws, document_file, False, True)) + yield RouterDescriptor(desc_content, vote, validate) class NetworkStatusDocument(stem.descriptor.Descriptor): """ @@ -193,21 +215,13 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): # authority section while doc_parser.line.startswith("dir-source "): - dirauth_data = doc_parser.read_until(["dir-source", "r"]) + dirauth_data = doc_parser.read_until(["dir-source", "r", "directory-footer", "directory-signature", "bandwidth-weights"]) self.directory_authorities.append(DirectoryAuthority(dirauth_data, vote, validate)) - def _router_desc_generator(raw_content, vote, validate): - parser = stem.descriptor.DescriptorParser(raw_content, validate) - while parser.line != None: - descriptor = parser.read_until("r") - yield self._generate_router(descriptor, vote, validate) - # router descriptors if doc_parser.peek_keyword() == "r": router_descriptors_data = doc_parser.read_until(["bandwidth-weights", "directory-footer", "directory-signature"]) - self.router_descriptors = _router_desc_generator(router_descriptors_data, vote, validate) - elif validate: - raise ValueError("No router descriptors found") + self.router_descriptors = _router_desc_generator(StringIO(router_descriptors_data), vote, validate) # footer section if self.consensus_method > 9 or vote and filter(lambda x: x >= 9, self.consensus_methods):

1 0