commit b8ca825a7f6dcde9f7689bca41a0938598749b4f Author: Damian Johnson <atagar@torproject.org> Date: Sun Oct 7 17:59:08 2012 -0700 Module for V2, V3 and Microdescriptor router status entries Moving the router status entries to their own module and adding classes for V2 entries and microdescriptors (both still completely untested). The abstraction is based on the ServerDescriptor breakdown but with more liberal use of helpers for the actual parsing. --- run_tests.py | 4 +- stem/descriptor/__init__.py | 1 + stem/descriptor/networkstatus.py | 328 +-------------- stem/descriptor/router_status_entry.py | 538 ++++++++++++++++++++++++ stem/descriptor/server_descriptor.py | 22 +- test/mocking.py | 15 +- test/unit/descriptor/__init__.py | 9 +- test/unit/descriptor/networkstatus/__init__.py | 2 +- test/unit/descriptor/networkstatus/document.py | 15 +- test/unit/descriptor/networkstatus/entry.py | 425 ------------------- test/unit/descriptor/router_status_entry.py | 425 +++++++++++++++++++ 11 files changed, 1009 insertions(+), 775 deletions(-) diff --git a/run_tests.py b/run_tests.py index 8f76dd3..c4db524 100755 --- a/run_tests.py +++ b/run_tests.py @@ -20,7 +20,7 @@ import test.unit.descriptor.export import test.unit.descriptor.reader import test.unit.descriptor.server_descriptor import test.unit.descriptor.extrainfo_descriptor -import test.unit.descriptor.networkstatus.entry +import test.unit.descriptor.router_status_entry import test.unit.descriptor.networkstatus.directory_authority import test.unit.descriptor.networkstatus.key_certificate import test.unit.descriptor.networkstatus.document @@ -118,7 +118,7 @@ UNIT_TESTS = ( test.unit.descriptor.reader.TestDescriptorReader, test.unit.descriptor.server_descriptor.TestServerDescriptor, test.unit.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, - test.unit.descriptor.networkstatus.entry.TestRouterStatusEntry, + test.unit.descriptor.router_status_entry.TestRouterStatusEntry, test.unit.descriptor.networkstatus.directory_authority.TestDirectoryAuthority, test.unit.descriptor.networkstatus.key_certificate.TestKeyCertificate, test.unit.descriptor.networkstatus.document.TestNetworkStatusDocument, diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index ea69300..ef3d558 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -18,6 +18,7 @@ __all__ = [ "extrainfo_descriptor", "server_descriptor", "networkstatus", + "router_status_entry", "parse_file", "Descriptor", ] diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 59e616a..38207b8 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -4,7 +4,7 @@ status documents (both votes and consensuses). The network status documents also contain a list of router descriptors, directory authorities, signatures etc. If you only need the -:class:`stem.descriptor.networkstatus.RouterStatusEntry` objects, use +:class:`stem.descriptor.router_status_entry.RouterStatusEntry` objects, use :func:`stem.descriptor.parse_file`. Other information can be accessed by directly instantiating :class:`stem.descriptor.networkstatus.NetworkStatusDocument` objects. @@ -34,14 +34,11 @@ The documents can be obtained from any of the following sources... parse_file - parses a network status file and provides a NetworkStatusDocument NetworkStatusDocument - Tor v3 network status document +- MicrodescriptorConsensus - Microdescriptor flavoured consensus documents - RouterStatusEntry - Router descriptor; contains information about a Tor relay - +- RouterMicrodescriptor - Router microdescriptor; contains information that doesn't change frequently DocumentSignature - Signature of a document by a directory authority DirectoryAuthority - Directory authority defined in a v3 network status document """ import re -import base64 import datetime try: @@ -50,6 +47,7 @@ except: from StringIO import StringIO import stem.descriptor +import stem.descriptor.router_status_entry import stem.version import stem.exit_policy import stem.util.tor_tools @@ -131,9 +129,9 @@ BANDWIDTH_WEIGHT_ENTRIES = ( def parse_file(document_file, validate = True, is_microdescriptor = False): """ - Parses a network status and iterates over the RouterStatusEntry or - RouterMicrodescriptor in it. The document that these instances reference have - an empty 'rotuers' attribute to allow for limited memory usage. + Parses a network status and iterates over the RouterStatusEntry in it. The + document that these instances reference have an empty 'rotuers' attribute to + allow for limited memory usage. :param file document_file: file with network status document content :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise @@ -159,7 +157,7 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): if not is_microdescriptor: document = NetworkStatusDocument(document_content, validate) - router_type = RouterStatusEntry + router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3 else: document = MicrodescriptorConsensus(document_content, validate) router_type = RouterMicrodescriptor @@ -221,7 +219,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): """ Version 3 network status document. This could be either a vote or consensus. - :var tuple routers: RouterStatusEntry contained in the document + :var tuple routers: RouterStatusEntryV3 contained in the document :var str version: **\*** document version :var bool is_consensus: **\*** true if the document is a consensus @@ -294,7 +292,7 @@ class NetworkStatusDocument(stem.descriptor.Descriptor): self._unrecognized_lines += value def _get_router_type(self): - return RouterStatusEntry + return stem.descriptor.router_status_entry.RouterStatusEntryV3 def meets_consensus_method(self, method): """ @@ -1036,265 +1034,6 @@ class DocumentSignature(object): return 0 -class RouterStatusEntry(stem.descriptor.Descriptor): - """ - Information about an individual router stored within a network status - document. - - :var NetworkStatusDocument document: **\*** document that this descriptor came from - - :var str nickname: **\*** router's nickname - :var str fingerprint: **\*** router's fingerprint - :var str digest: **\*** router's digest - :var datetime published: **\*** router's publication - :var str address: **\*** router's IP address - :var int or_port: **\*** router's ORPort - :var int dir_port: **\*** router's DirPort - :var list flags: **\*** list of status flags - - :var stem.version.Version version: parsed version of tor, this is None if the relay's using a new versioning scheme - :var str version_line: versioning information reported by the relay - - :var int bandwidth: bandwidth claimed by the relay (in kb/s) - :var int measured: bandwith measured to be available by the relay - :var list unrecognized_bandwidth_entries: **\*** bandwidth weighting information that isn't yet recognized - - :var stem.exit_policy.MicrodescriptorExitPolicy exit_policy: router's exit policy - - :var list microdescriptor_hashes: tuples of two values, the list of consensus methods for generting a set of digests and the 'algorithm => digest' mappings - - **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined - """ - - def __init__(self, raw_contents, validate = True, document = None): - """ - Parse a router descriptor in a v3 network status document. - - :param str raw_content: router descriptor content to be parsed - :param NetworkStatusDocument document: document this descriptor came from - :param bool validate: checks the validity of the content if True, skips these checks otherwise - - :raises: ValueError if the descriptor data is invalid - """ - - super(RouterStatusEntry, self).__init__(raw_contents) - - self.document = document - - self.nickname = None - self.fingerprint = None - self.digest = None - self.published = None - self.address = None - self.or_port = None - self.dir_port = None - - self.flags = None - - self.version_line = None - self.version = None - - self.bandwidth = None - self.measured = None - self.unrecognized_bandwidth_entries = [] - - self.exit_policy = None - self.microdescriptor_hashes = None - self._unrecognized_lines = [] - - self._parse(raw_contents, validate) - - def _parse(self, content, validate): - """ - Parses the given content and applies the attributes. - - :param str content: descriptor content - :param bool validate: checks validity if True - - :raises: ValueError if a validity check fails - """ - - entries, first_keyword, _, _ = stem.descriptor._get_descriptor_components(content, validate) - - if validate and first_keyword != 'r': - raise ValueError("Router status entries are expected to start with a 'r' line:\n%s" % (content)) - - # check that we have mandatory fields - if validate: - for keyword in ('r', 's'): - if not keyword in entries: - raise ValueError("Router status entries must have a '%s' line:\n%s" % (keyword, content)) - - for keyword, values in entries.items(): - value, block_contents = values[0] - line = "%s %s" % (keyword, value) - - # most attributes can only appear at most once - if validate and len(values) > 1 and keyword in ('r', 's', 'v', 'w', 'p'): - raise ValueError("Router status entries can only have a single '%s' line, got %i:\n%s" % (keyword, len(values), content)) - - if keyword == 'r': - # "r" nickname identity digest publication IP ORPort DirPort - # r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0 - - r_comp = value.split(" ") - - if len(r_comp) < 8: - if not validate: continue - raise ValueError("Router status entry's 'r' line must have eight values: %s" % line) - - if validate: - if not stem.util.tor_tools.is_valid_nickname(r_comp[0]): - raise ValueError("Router status entry's nickname isn't valid: %s" % r_comp[0]) - elif not stem.util.connection.is_valid_ip_address(r_comp[5]): - raise ValueError("Router status entry's address isn't a valid IPv4 address: %s" % r_comp[5]) - elif not stem.util.connection.is_valid_port(r_comp[6]): - raise ValueError("Router status entry's ORPort is invalid: %s" % r_comp[6]) - elif not stem.util.connection.is_valid_port(r_comp[7], allow_zero = True): - raise ValueError("Router status entry's DirPort is invalid: %s" % r_comp[7]) - elif not (r_comp[6].isdigit() and r_comp[7].isdigit()): - continue - - self.nickname = r_comp[0] - self.fingerprint = _decode_fingerprint(r_comp[1], validate) - self.digest = r_comp[2] - self.address = r_comp[5] - self.or_port = int(r_comp[6]) - self.dir_port = None if r_comp[7] == '0' else int(r_comp[7]) - - try: - published = "%s %s" % (r_comp[3], r_comp[4]) - self.published = datetime.datetime.strptime(published, "%Y-%m-%d %H:%M:%S") - except ValueError: - if validate: - raise ValueError("Publication time time wasn't parseable: %s" % line) - elif keyword == 's': - # "s" Flags - # s Named Running Stable Valid - - if value == "": - self.flags = [] - else: - self.flags = value.split(" ") - - if validate: - for flag in self.flags: - if self.flags.count(flag) > 1: - raise ValueError("Router status entry had duplicate flags: %s" % line) - elif flag == "": - raise ValueError("Router status entry had extra whitespace on its 's' line: %s" % line) - elif keyword == 'v': - # "v" version - # v Tor 0.2.2.35 - # - # The spec says that if this starts with "Tor " then what follows is a - # tor version. If not then it has "upgraded to a more sophisticated - # protocol versioning system". - - self.version_line = value - - if value.startswith("Tor "): - try: - self.version = stem.version.Version(value[4:]) - except ValueError, exc: - if validate: - raise ValueError("Router status entry has a malformed tor version (%s): %s" % (exc, line)) - elif keyword == 'w': - # "w" "Bandwidth=" INT ["Measured=" INT] - # w Bandwidth=7980 - - w_comp = value.split(" ") - - if len(w_comp) < 1: - if not validate: continue - raise ValueError("Router status entry's 'w' line is blank: %s" % line) - elif not w_comp[0].startswith("Bandwidth="): - if not validate: continue - raise ValueError("Router status entry's 'w' line needs to start with a 'Bandwidth=' entry: %s" % line) - - for w_entry in w_comp: - if '=' in w_entry: - w_key, w_value = w_entry.split('=', 1) - else: - w_key, w_value = w_entry, None - - if w_key == "Bandwidth": - if not (w_value and w_value.isdigit()): - if not validate: continue - raise ValueError("Router status entry's 'Bandwidth=' entry needs to have a numeric value: %s" % line) - - self.bandwidth = int(w_value) - elif w_key == "Measured": - if not (w_value and w_value.isdigit()): - if not validate: continue - raise ValueError("Router status entry's 'Measured=' entry needs to have a numeric value: %s" % line) - - self.measured = int(w_value) - else: - self.unrecognized_bandwidth_entries.append(w_entry) - elif keyword == 'p': - # "p" ("accept" / "reject") PortList - # p reject 1-65535 - # p accept 80,110,143,443,993,995,6660-6669,6697,7000-7001 - - try: - self.exit_policy = stem.exit_policy.MicrodescriptorExitPolicy(value) - except ValueError, exc: - if not validate: continue - raise ValueError("Router status entry's exit policy is malformed (%s): %s" % (exc, line)) - elif keyword == 'm': - # "m" methods 1*(algorithm "=" digest) - # m 8,9,10,11,12 sha256=g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs - - m_comp = value.split(" ") - - if not (self.document and self.document.is_vote): - if not validate: continue - - vote_status = "vote" if self.document else "<undefined document>" - raise ValueError("Router status entry's 'm' line should only appear in votes (appeared in a %s): %s" % (vote_status, line)) - elif len(m_comp) < 1: - if not validate: continue - raise ValueError("Router status entry's 'm' line needs to start with a series of methods: %s" % line) - - try: - methods = [int(entry) for entry in m_comp[0].split(",")] - except ValueError: - if not validate: continue - raise ValueError("Router status entry's microdescriptor methods should be a series of comma separated integers: %s" % line) - - hashes = {} - - for entry in m_comp[1:]: - if not '=' in entry: - if not validate: continue - raise ValueError("Router status entry's can only have a series of 'algorithm=digest' mappings after the methods: %s" % line) - - hash_name, digest = entry.split('=', 1) - hashes[hash_name] = digest - - if self.microdescriptor_hashes is None: - self.microdescriptor_hashes = [] - - self.microdescriptor_hashes.append((methods, hashes)) - else: - self._unrecognized_lines.append(line) - - def get_unrecognized_lines(self): - """ - Provides any unrecognized lines. - - :returns: list of unrecognized lines - """ - - return list(self._unrecognized_lines) - - def __cmp__(self, other): - if not isinstance(other, RouterStatusEntry): - return 1 - - return str(self) > str(other) - class MicrodescriptorConsensus(NetworkStatusDocument): """ A v3 microdescriptor consensus. @@ -1326,7 +1065,7 @@ class MicrodescriptorConsensus(NetworkStatusDocument): def _validate_network_status_version(self): return self.version == "3 microdesc" -class RouterMicrodescriptor(RouterStatusEntry): +class RouterMicrodescriptor(stem.descriptor.router_status_entry.RouterStatusEntry): """ Router microdescriptor object. Parses and stores router information in a router microdescriptor from a v3 microdescriptor consensus. @@ -1452,52 +1191,3 @@ class RouterMicrodescriptor(RouterStatusEntry): return self.unrecognized_lines -def _decode_fingerprint(identity, validate): - """ - Decodes the 'identity' value found in consensuses into the more common hex - encoding of the relay's fingerprint. For example... - - :: - - >>> _decode_fingerprint('p1aag7VwarGxqctS7/fS0y5FU+s') - 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB' - - :param str identity: encoded fingerprint from the consensus - :param bool validate: checks validity if True - - :returns: str with the uppercase hex encoding of the relay's fingerprint - - :raises: ValueError if the result isn't a valid fingerprint - """ - - # trailing equal signs were stripped from the identity - missing_padding = 28 - len(identity) - identity += "=" * missing_padding - - fingerprint = "" - - try: - identity_decoded = base64.b64decode(identity) - except TypeError, exc: - if not validate: return None - raise ValueError("Unable to decode identity string '%s'" % identity) - - for char in identity_decoded: - # Individual characters are either standard ascii or hex encoded, and each - # represent two hex digits. For instnace... - # - # >>> ord('\n') - # 10 - # >>> hex(10) - # '0xa' - # >>> '0xa'[2:].zfill(2).upper() - # '0A' - - fingerprint += hex(ord(char))[2:].zfill(2).upper() - - if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): - if not validate: return None - raise ValueError("Decoded '%s' to be '%s', which isn't a valid fingerprint" % (identity, fingerprint)) - - return fingerprint - diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py new file mode 100644 index 0000000..50f0825 --- /dev/null +++ b/stem/descriptor/router_status_entry.py @@ -0,0 +1,538 @@ +""" +Parsing for router status entries, the information for individual routers +within a network status document. This information is provided from a few +sources... + +* control port via 'GETINFO ns/*' and 'GETINFO md/*' queries +* router entries in a network status document, like the cached-consensus + +**Module Overview:** + +:: + + RouterStatusEntry - Common parent for router status entries. + |- RouterStatusEntryV2 - Entry for a network status v2 document. + |- RouterStatusEntryV3 - Entry for a network status v3 document. + +- RouterStatusEntryMicroV3 - Entry for a microdescriptor flavored v3 document. +""" + +import base64 +import datetime + +import stem.descriptor +import stem.exit_policy + +class RouterStatusEntry(stem.descriptor.Descriptor): + """ + Information about an individual router stored within a network status + document. This is the common parent for concrete status entry types. + + :var NetworkStatusDocument document: **\*** document that this descriptor came from + + :var str nickname: **\*** router's nickname + :var str fingerprint: **\*** router's fingerprint + :var datetime published: **\*** router's publication + :var str address: **\*** router's IP address + :var int or_port: **\*** router's ORPort + :var int dir_port: **\*** router's DirPort + + :var list flags: **\*** list of status flags + + :var stem.version.Version version: parsed version of tor, this is None if the relay's using a new versioning scheme + :var str version_line: versioning information reported by the relay + """ + + def __init__(self, content, validate, document): + """ + Parse a router descriptor in a network status document. + + :param str content: router descriptor content to be parsed + :param NetworkStatusDocument document: document this descriptor came from + :param bool validate: checks the validity of the content if True, skips these checks otherwise + + :raises: ValueError if the descriptor data is invalid + """ + + super(RouterStatusEntry, self).__init__(content) + + self.document = document + + self.nickname = None + self.fingerprint = None + self.published = None + self.address = None + self.or_port = None + self.dir_port = None + + self.flags = None + + self.version_line = None + self.version = None + + self._unrecognized_lines = [] + + entries, first_keyword, _, _ = stem.descriptor._get_descriptor_components(content, validate) + if validate: self._check_constraints(entries, first_keyword) + self._parse(entries, validate) + + def _parse(self, entries, validate): + """ + Parses the given content and applies the attributes. + + :param dict entries: keyword => (value, pgp key) entries + :param bool validate: checks validity if True + + :raises: ValueError if a validity check fails + """ + + for keyword, values in entries.items(): + value, _ = values[0] + + if keyword == 's': + _parse_s_line(self, value, validate) + elif keyword == 'v': + _parse_v_line(self, value, validate) + else: + self._unrecognized_lines.append("%s %s" % (keyword, value)) + + def _check_constraints(self, entries, first_keyword): + """ + Does a basic check that the entries conform to this descriptor type's + constraints. + + :param dict entries: keyword => (value, pgp key) entries + :param str first_keyword: keyword of the first line + + :raises: ValueError if an issue arises in validation + """ + + for keyword in self._required_fields(): + if not keyword in entries: + raise ValueError("%s must have a '%s' line:\n%s" % (self._name(True), keyword, str(self))) + + for keyword in self._single_fields(): + if keyword in entries and len(entries[keyword]) > 1: + raise ValueError("%s can only have a single '%s' line, got %i:\n%s" % (self._name(True), keyword, len(entries[keyword]), str(self))) + + if first_keyword != 'r': + raise ValueError("%s are expected to start with a 'r' line:\n%s" % (self._name(True), str(self))) + + def _name(self, is_plural = False): + """ + Name for this descriptor type. + """ + + if is_plural: + return "Router status entries" + else: + return "Router status entry" + + def _required_fields(self): + """ + Provides lines that must appear in the descriptor. + """ + + return () + + def _single_fields(self): + """ + Provides lines that can only appear in the descriptor once. + """ + + return () + + def get_unrecognized_lines(self): + """ + Provides any unrecognized lines. + + :returns: list of unrecognized lines + """ + + return list(self._unrecognized_lines) + + def __cmp__(self, other): + if not isinstance(other, RouterStatusEntry): + return 1 + + return str(self) > str(other) + +class RouterStatusEntryV2(RouterStatusEntry): + """ + Information about an individual router stored within a version 2 network + status document. + + :var str digest: **\*** router's digest + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, content, validate = True, document = None): + self.digest = None + super(RouterStatusEntryV2, self).__init__(content, validate, document) + + def _parse(self, entries, validate): + for keyword, values in entries.items(): + value, _ = values[0] + + if keyword == 'r': + _parse_r_line(self, value, validate, True) + del entries['r'] + + RouterStatusEntry._parse(self, entries, validate) + + def _name(self, is_plural = False): + if is_plural: + return "Router status entries (v2)" + else: + return "Router status entry (v2)" + + def _required_fields(self): + return ('r') + + def _single_fields(self): + return ('r', 's', 'v') + + def __cmp__(self, other): + if not isinstance(other, RouterStatusEntryV2): + return 1 + + return str(self) > str(other) + +class RouterStatusEntryV3(RouterStatusEntry): + """ + Information about an individual router stored within a version 3 network + status document. + + :var str digest: **\*** router's digest + + :var int bandwidth: bandwidth claimed by the relay (in kb/s) + :var int measured: bandwith measured to be available by the relay + :var list unrecognized_bandwidth_entries: **\*** bandwidth weighting information that isn't yet recognized + + :var stem.exit_policy.MicrodescriptorExitPolicy exit_policy: router's exit policy + + :var list microdescriptor_hashes: tuples of two values, the list of consensus methods for generting a set of digests and the 'algorithm => digest' mappings + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, content, validate = True, document = None): + self.digest = None + + self.bandwidth = None + self.measured = None + self.unrecognized_bandwidth_entries = [] + + self.exit_policy = None + self.microdescriptor_hashes = None + + super(RouterStatusEntryV3, self).__init__(content, validate, document) + + def _parse(self, entries, validate): + for keyword, values in entries.items(): + value, _ = values[0] + + if keyword == 'r': + _parse_r_line(self, value, validate, True) + del entries['r'] + elif keyword == 'w': + _parse_w_line(self, value, validate) + del entries['w'] + elif keyword == 'p': + _parse_p_line(self, value, validate) + del entries['p'] + elif keyword == 'm': + _parse_m_line(self, value, validate) + del entries['m'] + + RouterStatusEntry._parse(self, entries, validate) + + def _name(self, is_plural = False): + if is_plural: + return "Router status entries (v3)" + else: + return "Router status entry (v3)" + + def _required_fields(self): + return ('r', 's') + + def _single_fields(self): + return ('r', 's', 'v', 'w', 'p') + + def __cmp__(self, other): + if not isinstance(other, RouterStatusEntryV3): + return 1 + + return str(self) > str(other) + +class RouterStatusEntryMicroV3(RouterStatusEntry): + """ + Information about an individual router stored within a microdescriptor + flavored network status document. + + :var int bandwidth: bandwidth claimed by the relay (in kb/s) + :var int measured: bandwith measured to be available by the relay + :var list unrecognized_bandwidth_entries: **\*** bandwidth weighting information that isn't yet recognized + + :var str digest: **\*** router's base64 encoded router microdescriptor digest + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, content, validate = True, document = None): + self.version_line = None + self.version = None + + self.digest = None + + super(RouterStatusEntryMicroV3, self).__init__(content) + + def _parse(self, entries, validate): + for keyword, values in entries.items(): + value, _ = values[0] + + if keyword == 'r': + _parse_r_line(self, value, validate, False) + del entries['r'] + elif keyword == 'w': + _parse_w_line(self, value, validate) + del entries['w'] + elif keyword == 'm': + # "m" digest + # example: m aiUklwBrua82obG5AsTX+iEpkjQA2+AQHxZ7GwMfY70 + + self.digest = value + del entries['m'] + + RouterStatusEntry._parse(self, entries, validate) + + def _name(self, is_plural = False): + if is_plural: + return "Router status entries (micro v3)" + else: + return "Router status entry (micro v3)" + + def _required_fields(self): + return ('r', 's', 'm') + + def _single_fields(self): + return ('r', 's', 'v', 'w', 'm') + + def __cmp__(self, other): + if not isinstance(other, RouterStatusEntryMicroV3): + return 1 + + return str(self) > str(other) + +def _parse_r_line(desc, value, validate, include_digest = True): + # Parses a RouterStatusEntry's 'r' line. They're very nearly identical for + # all current entry types (v2, v3, and microdescriptor v3) with one little + # wrinkle: only the microdescriptor flavor excludes a 'digest' field. + # + # For v2 and v3 router status entries: + # "r" nickname identity digest publication IP ORPort DirPort + # example: r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0 + # + # For v3 microdescriptor router status entries: + # "r" nickname identity publication IP ORPort DirPort + # example: r Konata ARIJF2zbqirB9IwsW0mQznccWww 2012-09-24 13:40:40 69.64.48.168 9001 9030 + + r_comp = value.split(" ") + + # inject a None for the digest to normalize the field positioning + if not include_digest: + r_comp.insert(2, None) + + if len(r_comp) < 8: + if not validate: return + + expected_field_count = 'eight' if include_digest else 'seven' + raise ValueError("%s 'r' line must have %s values: r %s" % (desc._name(), expected_field_count, value)) + + if validate: + if not stem.util.tor_tools.is_valid_nickname(r_comp[0]): + raise ValueError("%s nickname isn't valid: %s" % (desc._name(), r_comp[0])) + elif not stem.util.connection.is_valid_ip_address(r_comp[5]): + raise ValueError("%s address isn't a valid IPv4 address: %s" % (desc._name(), r_comp[5])) + elif not stem.util.connection.is_valid_port(r_comp[6]): + raise ValueError("%s ORPort is invalid: %s" % (desc._name(), r_comp[6])) + elif not stem.util.connection.is_valid_port(r_comp[7], allow_zero = True): + raise ValueError("%s DirPort is invalid: %s" % (desc._name(), r_comp[7])) + elif not (r_comp[6].isdigit() and r_comp[7].isdigit()): + return + + desc.nickname = r_comp[0] + desc.fingerprint = _decode_fingerprint(r_comp[1], validate) + if include_digest: desc.digest = r_comp[2] + desc.address = r_comp[5] + desc.or_port = int(r_comp[6]) + desc.dir_port = None if r_comp[7] == '0' else int(r_comp[7]) + + try: + published = "%s %s" % (r_comp[3], r_comp[4]) + desc.published = datetime.datetime.strptime(published, "%Y-%m-%d %H:%M:%S") + except ValueError: + if validate: + raise ValueError("Publication time time wasn't parseable: r %s" % value) + +def _parse_s_line(desc, value, validate): + # "s" Flags + # example: s Named Running Stable Valid + + flags = [] if value == "" else value.split(" ") + desc.flags = flags + + if validate: + for flag in flags: + if flags.count(flag) > 1: + raise ValueError("%s had duplicate flags: s %s" % (desc._name(), value)) + elif flag == "": + raise ValueError("%s had extra whitespace on its 's' line: s %s" % (desc._name(), value)) + +def _parse_v_line(desc, value, validate): + # "v" version + # example: v Tor 0.2.2.35 + # + # The spec says that if this starts with "Tor " then what follows is a + # tor version. If not then it has "upgraded to a more sophisticated + # protocol versioning system". + + desc.version_line = value + + if value.startswith("Tor "): + try: + desc.version = stem.version.Version(value[4:]) + except ValueError, exc: + if validate: + raise ValueError("%s has a malformed tor version (%s): v %s" % (desc._name(), exc, value)) + +def _parse_w_line(desc, value, validate): + # "w" "Bandwidth=" INT ["Measured=" INT] + # example: w Bandwidth=7980 + + w_comp = value.split(" ") + + if len(w_comp) < 1: + if not validate: return + raise ValueError("%s 'w' line is blank: w %s" % (desc._name(), value)) + elif not w_comp[0].startswith("Bandwidth="): + if not validate: return + raise ValueError("%s 'w' line needs to start with a 'Bandwidth=' entry: w %s" % (desc._name(), value)) + + for w_entry in w_comp: + if '=' in w_entry: + w_key, w_value = w_entry.split('=', 1) + else: + w_key, w_value = w_entry, None + + if w_key == "Bandwidth": + if not (w_value and w_value.isdigit()): + if not validate: return + raise ValueError("%s 'Bandwidth=' entry needs to have a numeric value: w %s" % (desc._name(), value)) + + desc.bandwidth = int(w_value) + elif w_key == "Measured": + if not (w_value and w_value.isdigit()): + if not validate: return + raise ValueError("%s 'Measured=' entry needs to have a numeric value: w %s" % (desc._name(), value)) + + desc.measured = int(w_value) + else: + desc.unrecognized_bandwidth_entries.append(w_entry) + +def _parse_p_line(desc, value, validate): + # "p" ("accept" / "reject") PortList + # p reject 1-65535 + # example: p accept 80,110,143,443,993,995,6660-6669,6697,7000-7001 + + try: + desc.exit_policy = stem.exit_policy.MicrodescriptorExitPolicy(value) + except ValueError, exc: + if not validate: return + raise ValueError("%s exit policy is malformed (%s): p %s" % (desc._name(), exc, value)) + +def _parse_m_line(desc, value, validate): + # "m" methods 1*(algorithm "=" digest) + # example: m 8,9,10,11,12 sha256=g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs + + m_comp = value.split(" ") + + if not (desc.document and desc.document.is_vote): + if not validate: return + + vote_status = "vote" if desc.document else "<undefined document>" + raise ValueError("%s 'm' line should only appear in votes (appeared in a %s): m %s" % (desc._name(), vote_status, value)) + elif len(m_comp) < 1: + if not validate: return + raise ValueError("%s 'm' line needs to start with a series of methods: m %s" % (desc._name(), value)) + + try: + methods = [int(entry) for entry in m_comp[0].split(",")] + except ValueError: + if not validate: return + raise ValueError("%s microdescriptor methods should be a series of comma separated integers: m %s" % (desc._name(), value)) + + hashes = {} + + for entry in m_comp[1:]: + if not '=' in entry: + if not validate: continue + raise ValueError("%s can only have a series of 'algorithm=digest' mappings after the methods: m %s" % (desc._name(), value)) + + hash_name, digest = entry.split('=', 1) + hashes[hash_name] = digest + + if desc.microdescriptor_hashes is None: + desc.microdescriptor_hashes = [] + + desc.microdescriptor_hashes.append((methods, hashes)) + +def _decode_fingerprint(identity, validate): + """ + Decodes the 'identity' value found in consensuses into the more common hex + encoding of the relay's fingerprint. For example... + + :: + + >>> _decode_fingerprint('p1aag7VwarGxqctS7/fS0y5FU+s') + 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB' + + :param str identity: encoded fingerprint from the consensus + :param bool validate: checks validity if True + + :returns: str with the uppercase hex encoding of the relay's fingerprint + + :raises: ValueError if the result isn't a valid fingerprint + """ + + # trailing equal signs were stripped from the identity + missing_padding = 28 - len(identity) + identity += "=" * missing_padding + + fingerprint = "" + + try: + identity_decoded = base64.b64decode(identity) + except TypeError: + if not validate: return None + raise ValueError("Unable to decode identity string '%s'" % identity) + + for char in identity_decoded: + # Individual characters are either standard ascii or hex encoded, and each + # represent two hex digits. For instnace... + # + # >>> ord('\n') + # 10 + # >>> hex(10) + # '0xa' + # >>> '0xa'[2:].zfill(2).upper() + # '0A' + + fingerprint += hex(ord(char))[2:].zfill(2).upper() + + if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): + if not validate: return None + raise ValueError("Decoded '%s' to be '%s', which isn't a valid fingerprint" % (identity, fingerprint)) + + return fingerprint + diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 6eb3883..0ded3b1 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -507,24 +507,20 @@ class ServerDescriptor(stem.descriptor.Descriptor): :raises: ValueError if an issue arises in validation """ - required_fields = self._required_fields() - if required_fields: - for keyword in required_fields: - if not keyword in entries: - raise ValueError("Descriptor must have a '%s' entry" % keyword) - - single_fields = self._single_fields() - if single_fields: - for keyword in self._single_fields(): - if keyword in entries and len(entries[keyword]) > 1: - raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) + for keyword in self._required_fields(): + if not keyword in entries: + raise ValueError("Descriptor must have a '%s' entry" % keyword) + + for keyword in self._single_fields(): + if keyword in entries and len(entries[keyword]) > 1: + raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) expected_first_keyword = self._first_keyword() - if expected_first_keyword and not first_keyword == expected_first_keyword: + if expected_first_keyword and first_keyword != expected_first_keyword: raise ValueError("Descriptor must start with a '%s' entry" % expected_first_keyword) expected_last_keyword = self._last_keyword() - if expected_last_keyword and not last_keyword == expected_last_keyword: + if expected_last_keyword and last_keyword != expected_last_keyword: raise ValueError("Descriptor must end with a '%s' entry" % expected_last_keyword) if not self.exit_policy: diff --git a/test/mocking.py b/test/mocking.py index 2d656dd..808cb8d 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -27,7 +27,7 @@ calling :func:`test.mocking.revert_mocking`. get_bridge_server_descriptor - stem.descriptor.server_descriptor.BridgeDescriptor get_relay_extrainfo_descriptor - stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor get_bridge_extrainfo_descriptor - stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor - get_router_status_entry - stem.descriptor.networkstatus.RouterStatusEntry + get_router_status_entry_v3 - stem.descriptor.router_status_entry.RouterStatusEntryV3 get_directory_authority - stem.descriptor.networkstatus.DirectoryAuthority get_key_certificate - stem.descriptor.networkstatus.KeyCertificate get_network_status_document - stem.descriptor.networkstatus.NetworkStatusDocument @@ -43,6 +43,7 @@ import stem.socket import stem.descriptor.server_descriptor import stem.descriptor.extrainfo_descriptor import stem.descriptor.networkstatus +import stem.descriptor.router_status_entry # Once we've mocked a function we can't rely on its __module__ or __name__ # attributes, so instead we associate a unique 'mock_id' attribute that maps @@ -106,7 +107,7 @@ BRIDGE_EXTRAINFO_FOOTER = ( ("router-digest", "006FD96BA35E7785A6A3B8B75FE2E2435A13BDB4"), ) -ROUTER_STATUS_ENTRY_HEADER = ( +ROUTER_STATUS_ENTRY_V3_HEADER = ( ("r", "caerSidi p1aag7VwarGxqctS7/fS0y5FU+s oQZFLYe9e4A7bOkWKR7TaNxb0JE 2012-08-06 11:19:31 71.35.150.29 9001 0"), ("s", "Fast Named Running Stable Valid"), ) @@ -522,24 +523,24 @@ def get_bridge_extrainfo_descriptor(attr = None, exclude = (), content = False): else: return stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(desc_content, validate = True) -def get_router_status_entry(attr = None, exclude = (), content = False): +def get_router_status_entry_v3(attr = None, exclude = (), content = False): """ Provides the descriptor content for... - stem.descriptor.networkstatus.RouterStatusEntry + stem.descriptor.router_status_entry.RouterStatusEntryV3 :param dict attr: keyword/value mappings to be included in the descriptor :param list exclude: mandatory keywords to exclude from the descriptor :param bool content: provides the str content of the descriptor rather than the class if True - :returns: RouterStatusEntry for the requested descriptor content + :returns: RouterStatusEntryV3 for the requested descriptor content """ - desc_content = _get_descriptor_content(attr, exclude, ROUTER_STATUS_ENTRY_HEADER) + desc_content = _get_descriptor_content(attr, exclude, ROUTER_STATUS_ENTRY_V3_HEADER) if content: return desc_content else: - return stem.descriptor.networkstatus.RouterStatusEntry(desc_content, validate = True) + return stem.descriptor.router_status_entry.RouterStatusEntryV3(desc_content, validate = True) def get_directory_authority(attr = None, exclude = (), is_vote = False, content = False): """ diff --git a/test/unit/descriptor/__init__.py b/test/unit/descriptor/__init__.py index 4793560..163f33b 100644 --- a/test/unit/descriptor/__init__.py +++ b/test/unit/descriptor/__init__.py @@ -2,5 +2,12 @@ Unit tests for stem.descriptor. """ -__all__ = ["export", "reader", "extrainfo_descriptor", "server_descriptor", "networkstatus"] +__all__ = [ + "export", + "extrainfo_descriptor", + "networkstatus", + "reader", + "router_status_entry", + "server_descriptor", +] diff --git a/test/unit/descriptor/networkstatus/__init__.py b/test/unit/descriptor/networkstatus/__init__.py index b2314cc..a41defe 100644 --- a/test/unit/descriptor/networkstatus/__init__.py +++ b/test/unit/descriptor/networkstatus/__init__.py @@ -2,5 +2,5 @@ Unit tests for stem.descriptor.networkstatus. """ -__all__ = ["entry", "directory_authority", "key_certificate", "document"] +__all__ = ["directory_authority", "key_certificate", "document"] diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py index d0bb06a..e92aee2 100644 --- a/test/unit/descriptor/networkstatus/document.py +++ b/test/unit/descriptor/networkstatus/document.py @@ -8,8 +8,9 @@ import StringIO import stem.version from stem.descriptor import Flag -from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, RouterStatusEntry, DirectoryAuthority, NetworkStatusDocument, parse_file -from test.mocking import get_router_status_entry, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG +from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, DirectoryAuthority, NetworkStatusDocument, parse_file +from stem.descriptor.router_status_entry import RouterStatusEntryV3 +from test.mocking import get_router_status_entry_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG class TestNetworkStatusDocument(unittest.TestCase): def test_minimal_consensus(self): @@ -81,8 +82,8 @@ class TestNetworkStatusDocument(unittest.TestCase): Try parsing a document via the parse_file() function. """ - entry1 = get_router_status_entry({'s': "Fast"}) - entry2 = get_router_status_entry({'s': "Valid"}) + entry1 = get_router_status_entry_v3({'s': "Fast"}) + entry2 = get_router_status_entry_v3({'s': "Valid"}) content = get_network_status_document(routers = (entry1, entry2), content = True) # the document that the entries refer to should actually be the minimal @@ -620,15 +621,15 @@ class TestNetworkStatusDocument(unittest.TestCase): document. """ - entry1 = get_router_status_entry({'s': "Fast"}) - entry2 = get_router_status_entry({'s': "Valid"}) + entry1 = get_router_status_entry_v3({'s': "Fast"}) + entry2 = get_router_status_entry_v3({'s': "Valid"}) document = get_network_status_document(routers = (entry1, entry2)) self.assertEquals((entry1, entry2), document.routers) # try with an invalid RouterStatusEntry - entry3 = RouterStatusEntry(get_router_status_entry({'r': "ugabuga"}, content = True), False) + entry3 = RouterStatusEntryV3(get_router_status_entry_v3({'r': "ugabuga"}, content = True), False) content = get_network_status_document(routers = (entry3,), content = True) self.assertRaises(ValueError, NetworkStatusDocument, content) diff --git a/test/unit/descriptor/networkstatus/entry.py b/test/unit/descriptor/networkstatus/entry.py deleted file mode 100644 index 6400f2b..0000000 --- a/test/unit/descriptor/networkstatus/entry.py +++ /dev/null @@ -1,425 +0,0 @@ -""" -Unit tests for the RouterStatusEntry of stem.descriptor.networkstatus. -""" - -import datetime -import unittest - -from stem.descriptor import Flag -from stem.descriptor.networkstatus import RouterStatusEntry, _decode_fingerprint -from stem.version import Version -from stem.exit_policy import MicrodescriptorExitPolicy -from test.mocking import get_router_status_entry, ROUTER_STATUS_ENTRY_HEADER - -class TestRouterStatusEntry(unittest.TestCase): - def test_fingerprint_decoding(self): - """ - Tests for the _decode_fingerprint() helper. - """ - - # consensus identity field and fingerprint for caerSidi and Amunet1-5 - test_values = { - 'p1aag7VwarGxqctS7/fS0y5FU+s': 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB', - 'IbhGa8T+8tyy/MhxCk/qI+EI2LU': '21B8466BC4FEF2DCB2FCC8710A4FEA23E108D8B5', - '20wYcbFGwFfMktmuffYj6Z1RM9k': 'DB4C1871B146C057CC92D9AE7DF623E99D5133D9', - 'nTv9AG1cZeFW2hXiSIEAF6JLRJ4': '9D3BFD006D5C65E156DA15E248810017A24B449E', - '/UKsQiOSGPi/6es0/ha1prNTeDI': 'FD42AC42239218F8BFE9EB34FE16B5A6B3537832', - '/nHdqoKZ6bKZixxAPzYt9Qen+Is': 'FE71DDAA8299E9B2998B1C403F362DF507A7F88B', - } - - for arg, expected in test_values.items(): - self.assertEqual(expected, _decode_fingerprint(arg, True)) - - # checks with some malformed inputs - for arg in ('', '20wYcb', '20wYcb' * 30): - self.assertRaises(ValueError, _decode_fingerprint, arg, True) - self.assertEqual(None, _decode_fingerprint(arg, False)) - - def test_minimal(self): - """ - Parses a minimal router status entry. - """ - - entry = get_router_status_entry() - - expected_flags = set([Flag.FAST, Flag.NAMED, Flag.RUNNING, Flag.STABLE, Flag.VALID]) - self.assertEqual(None, entry.document) - self.assertEqual("caerSidi", entry.nickname) - self.assertEqual("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", entry.fingerprint) - self.assertEqual("oQZFLYe9e4A7bOkWKR7TaNxb0JE", entry.digest) - self.assertEqual(datetime.datetime(2012, 8, 6, 11, 19, 31), entry.published) - self.assertEqual("71.35.150.29", entry.address) - self.assertEqual(9001, entry.or_port) - self.assertEqual(None, entry.dir_port) - self.assertEqual(expected_flags, set(entry.flags)) - self.assertEqual(None, entry.version_line) - self.assertEqual(None, entry.version) - self.assertEqual(None, entry.bandwidth) - self.assertEqual(None, entry.measured) - self.assertEqual([], entry.unrecognized_bandwidth_entries) - self.assertEqual(None, entry.exit_policy) - self.assertEqual(None, entry.microdescriptor_hashes) - self.assertEqual([], entry.get_unrecognized_lines()) - - def test_missing_fields(self): - """ - Parses a router status entry that's missing fields. - """ - - content = get_router_status_entry(exclude = ('r', 's'), content = True) - self._expect_invalid_attr(content, "address") - - content = get_router_status_entry(exclude = ('r',), content = True) - self._expect_invalid_attr(content, "address") - - content = get_router_status_entry(exclude = ('s',), content = True) - self._expect_invalid_attr(content, "flags") - - def test_unrecognized_lines(self): - """ - Parses a router status entry with new keywords. - """ - - entry = get_router_status_entry({'z': 'New tor feature: sparkly unicorns!'}) - self.assertEquals(['z New tor feature: sparkly unicorns!'], entry.get_unrecognized_lines()) - - def test_proceeding_line(self): - """ - Includes content prior to the 'r' line. - """ - - content = 'z some stuff\n' + get_router_status_entry(content = True) - self._expect_invalid_attr(content, "_unrecognized_lines", ['z some stuff']) - - def test_blank_lines(self): - """ - Includes blank lines, which should be ignored. - """ - - content = get_router_status_entry(content = True) + "\n\nv Tor 0.2.2.35\n\n" - entry = RouterStatusEntry(content) - self.assertEqual("Tor 0.2.2.35", entry.version_line) - - def test_duplicate_lines(self): - """ - Duplicates linesin the entry. - """ - - lines = get_router_status_entry(content = True).split("\n") - - for i in xrange(len(lines)): - content = "\n".join(lines[:i] + [lines[i]] + lines[i:]) - self.assertRaises(ValueError, RouterStatusEntry, content) - - entry = RouterStatusEntry(content, False) - self.assertEqual("caerSidi", entry.nickname) - - def test_missing_r_field(self): - """ - Excludes fields from the 'r' line. - """ - - components = ( - ('nickname', 'caerSidi'), - ('fingerprint', 'p1aag7VwarGxqctS7/fS0y5FU+s'), - ('digest', 'oQZFLYe9e4A7bOkWKR7TaNxb0JE'), - ('published', '2012-08-06 11:19:31'), - ('address', '71.35.150.29'), - ('or_port', '9001'), - ('dir_port', '0'), - ) - - for attr, value in components: - # construct the 'r' line without this field - test_components = [comp[1] for comp in components] - test_components.remove(value) - r_line = ' '.join(test_components) - - content = get_router_status_entry({'r': r_line}, content = True) - self._expect_invalid_attr(content, attr) - - def test_malformed_nickname(self): - """ - Parses an 'r' line with a malformed nickname. - """ - - test_values = ( - "", - "saberrider2008ReallyLongNickname", # too long - "$aberrider2008", # invalid characters - ) - - for value in test_values: - r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("caerSidi", value) - content = get_router_status_entry({'r': r_line}, content = True) - - # TODO: Initial whitespace is consumed as part of the keyword/value - # divider. This is a bug in the case of V3 router status entries, but - # proper behavior for V2 router status entries and server/extrainfo - # descriptors. - # - # I'm inclined to leave this as-is for the moment since fixing it - # requires special KEYWORD_LINE handling, and the only result of this bug - # is that our validation doesn't catch the new SP restriction on V3 - # entries. - - if value == "": value = None - - self._expect_invalid_attr(content, "nickname", value) - - def test_malformed_fingerprint(self): - """ - Parses an 'r' line with a malformed fingerprint. - """ - - test_values = ( - "", - "zzzzz", - "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", - ) - - for value in test_values: - r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("p1aag7VwarGxqctS7/fS0y5FU+s", value) - content = get_router_status_entry({'r': r_line}, content = True) - self._expect_invalid_attr(content, "fingerprint") - - def test_malformed_published_date(self): - """ - Parses an 'r' line with a malformed published date. - """ - - test_values = ( - "", - "2012-08-06 11:19:", - "2012-08-06 11:19:71", - "2012-08-06 11::31", - "2012-08-06 11:79:31", - "2012-08-06 :19:31", - "2012-08-06 41:19:31", - "2012-08- 11:19:31", - "2012-08-86 11:19:31", - "2012--06 11:19:31", - "2012-38-06 11:19:31", - "-08-06 11:19:31", - "2012-08-06 11:19:31", - ) - - for value in test_values: - r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("2012-08-06 11:19:31", value) - content = get_router_status_entry({'r': r_line}, content = True) - self._expect_invalid_attr(content, "published") - - def test_malformed_address(self): - """ - Parses an 'r' line with a malformed address. - """ - - test_values = ( - "", - "71.35.150.", - "71.35..29", - "71.35.150", - "71.35.150.256", - ) - - for value in test_values: - r_line = ROUTER_STATUS_ENTRY_HEADER[0][1].replace("71.35.150.29", value) - content = get_router_status_entry({'r': r_line}, content = True) - self._expect_invalid_attr(content, "address", value) - - def test_malformed_port(self): - """ - Parses an 'r' line with a malformed ORPort or DirPort. - """ - - test_values = ( - "", - "-1", - "399482", - "blarg", - ) - - for value in test_values: - for include_or_port in (False, True): - for include_dir_port in (False, True): - if not include_or_port and not include_dir_port: - continue - - r_line = ROUTER_STATUS_ENTRY_HEADER[0][1] - - if include_or_port: - r_line = r_line.replace(" 9001 ", " %s " % value) - - if include_dir_port: - r_line = r_line[:-1] + value - - attr = "or_port" if include_or_port else "dir_port" - expected = int(value) if value.isdigit() else None - - content = get_router_status_entry({'r': r_line}, content = True) - self._expect_invalid_attr(content, attr, expected) - - def test_flags(self): - """ - Handles a variety of flag inputs. - """ - - test_values = { - "": [], - "Fast": [Flag.FAST], - "Fast Valid": [Flag.FAST, Flag.VALID], - "Ugabuga": ["Ugabuga"], - } - - for s_line, expected in test_values.items(): - entry = get_router_status_entry({'s': s_line}) - self.assertEquals(expected, entry.flags) - - # tries some invalid inputs - test_values = { - "Fast ": [Flag.FAST, "", "", ""], - "Fast Valid": [Flag.FAST, "", Flag.VALID], - "Fast Fast": [Flag.FAST, Flag.FAST], - } - - for s_line, expected in test_values.items(): - content = get_router_status_entry({'s': s_line}, content = True) - self._expect_invalid_attr(content, "flags", expected) - - def test_versions(self): - """ - Handles a variety of version inputs. - """ - - test_values = { - "Tor 0.2.2.35": Version("0.2.2.35"), - "Tor 0.1.2": Version("0.1.2"), - "Torr new_stuff": None, - "new_stuff and stuff": None, - } - - for v_line, expected in test_values.items(): - entry = get_router_status_entry({'v': v_line}) - self.assertEquals(expected, entry.version) - self.assertEquals(v_line, entry.version_line) - - # tries an invalid input - content = get_router_status_entry({'v': "Tor ugabuga"}, content = True) - self._expect_invalid_attr(content, "version") - - def test_bandwidth(self): - """ - Handles a variety of 'w' lines. - """ - - test_values = { - "Bandwidth=0": (0, None, []), - "Bandwidth=63138": (63138, None, []), - "Bandwidth=11111 Measured=482": (11111, 482, []), - "Bandwidth=11111 Measured=482 Blarg!": (11111, 482, ["Blarg!"]), - } - - for w_line, expected in test_values.items(): - entry = get_router_status_entry({'w': w_line}) - self.assertEquals(expected[0], entry.bandwidth) - self.assertEquals(expected[1], entry.measured) - self.assertEquals(expected[2], entry.unrecognized_bandwidth_entries) - - # tries some invalid inputs - test_values = ( - "", - "blarg", - "Bandwidth", - "Bandwidth=", - "Bandwidth:0", - "Bandwidth 0", - "Bandwidth=-10", - "Bandwidth=10 Measured", - "Bandwidth=10 Measured=", - "Bandwidth=10 Measured=-50", - ) - - for w_line in test_values: - content = get_router_status_entry({'w': w_line}, content = True) - self._expect_invalid_attr(content) - - def test_exit_policy(self): - """ - Handles a variety of 'p' lines. - """ - - test_values = { - "reject 1-65535": MicrodescriptorExitPolicy("reject 1-65535"), - "accept 80,110,143,443": MicrodescriptorExitPolicy("accept 80,110,143,443"), - } - - for p_line, expected in test_values.items(): - entry = get_router_status_entry({'p': p_line}) - self.assertEquals(expected, entry.exit_policy) - - # tries some invalid inputs - test_values = ( - "", - "blarg", - "reject -50", - "accept 80,", - ) - - for p_line in test_values: - content = get_router_status_entry({'p': p_line}, content = True) - self._expect_invalid_attr(content, "exit_policy") - - def test_microdescriptor_hashes(self): - """ - Handles a variety of 'm' lines. - """ - - test_values = { - "8,9,10,11,12": - [([8, 9, 10, 11, 12], {})], - "8,9,10,11,12 sha256=g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs": - [([8, 9, 10, 11, 12], {"sha256": "g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs"})], - "8,9,10,11,12 sha256=g1vx9si329muxV md5=3tquWIXXySNOIwRGMeAESKs/v4DWs": - [([8, 9, 10, 11, 12], {"sha256": "g1vx9si329muxV", "md5": "3tquWIXXySNOIwRGMeAESKs/v4DWs"})], - } - - # we need a document that's a vote - mock_document = lambda x: x # just need anything with a __dict__ - mock_document.__dict__["is_vote"] = True - mock_document.__dict__["is_consensus"] = False - - for m_line, expected in test_values.items(): - content = get_router_status_entry({'m': m_line}, content = True) - entry = RouterStatusEntry(content, document = mock_document) - self.assertEquals(expected, entry.microdescriptor_hashes) - - # try without a document - content = get_router_status_entry({'m': "8,9,10,11,12"}, content = True) - self._expect_invalid_attr(content, "microdescriptor_hashes") - - # tries some invalid inputs - test_values = ( - "", - "4,a,2", - "1,2,3 stuff", - ) - - for m_line in test_values: - content = get_router_status_entry({'m': m_line}, content = True) - self.assertRaises(ValueError, RouterStatusEntry, content, True, mock_document) - - def _expect_invalid_attr(self, content, attr = None, expected_value = None): - """ - Asserts that construction will fail due to content having a malformed - attribute. If an attr is provided then we check that it matches an expected - value when we're constructed without validation. - """ - - self.assertRaises(ValueError, RouterStatusEntry, content) - entry = RouterStatusEntry(content, False) - - if attr: - self.assertEquals(expected_value, getattr(entry, attr)) - else: - self.assertEquals("caerSidi", entry.nickname) - diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py new file mode 100644 index 0000000..6816656 --- /dev/null +++ b/test/unit/descriptor/router_status_entry.py @@ -0,0 +1,425 @@ +""" +Unit tests for stem.descriptor.router_status_entry. +""" + +import datetime +import unittest + +from stem.descriptor import Flag +from stem.descriptor.router_status_entry import RouterStatusEntryV3, _decode_fingerprint +from stem.version import Version +from stem.exit_policy import MicrodescriptorExitPolicy +from test.mocking import get_router_status_entry_v3, ROUTER_STATUS_ENTRY_V3_HEADER + +class TestRouterStatusEntry(unittest.TestCase): + def test_fingerprint_decoding(self): + """ + Tests for the _decode_fingerprint() helper. + """ + + # consensus identity field and fingerprint for caerSidi and Amunet1-5 + test_values = { + 'p1aag7VwarGxqctS7/fS0y5FU+s': 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB', + 'IbhGa8T+8tyy/MhxCk/qI+EI2LU': '21B8466BC4FEF2DCB2FCC8710A4FEA23E108D8B5', + '20wYcbFGwFfMktmuffYj6Z1RM9k': 'DB4C1871B146C057CC92D9AE7DF623E99D5133D9', + 'nTv9AG1cZeFW2hXiSIEAF6JLRJ4': '9D3BFD006D5C65E156DA15E248810017A24B449E', + '/UKsQiOSGPi/6es0/ha1prNTeDI': 'FD42AC42239218F8BFE9EB34FE16B5A6B3537832', + '/nHdqoKZ6bKZixxAPzYt9Qen+Is': 'FE71DDAA8299E9B2998B1C403F362DF507A7F88B', + } + + for arg, expected in test_values.items(): + self.assertEqual(expected, _decode_fingerprint(arg, True)) + + # checks with some malformed inputs + for arg in ('', '20wYcb', '20wYcb' * 30): + self.assertRaises(ValueError, _decode_fingerprint, arg, True) + self.assertEqual(None, _decode_fingerprint(arg, False)) + + def test_minimal(self): + """ + Parses a minimal router status entry. + """ + + entry = get_router_status_entry_v3() + + expected_flags = set([Flag.FAST, Flag.NAMED, Flag.RUNNING, Flag.STABLE, Flag.VALID]) + self.assertEqual(None, entry.document) + self.assertEqual("caerSidi", entry.nickname) + self.assertEqual("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", entry.fingerprint) + self.assertEqual("oQZFLYe9e4A7bOkWKR7TaNxb0JE", entry.digest) + self.assertEqual(datetime.datetime(2012, 8, 6, 11, 19, 31), entry.published) + self.assertEqual("71.35.150.29", entry.address) + self.assertEqual(9001, entry.or_port) + self.assertEqual(None, entry.dir_port) + self.assertEqual(expected_flags, set(entry.flags)) + self.assertEqual(None, entry.version_line) + self.assertEqual(None, entry.version) + self.assertEqual(None, entry.bandwidth) + self.assertEqual(None, entry.measured) + self.assertEqual([], entry.unrecognized_bandwidth_entries) + self.assertEqual(None, entry.exit_policy) + self.assertEqual(None, entry.microdescriptor_hashes) + self.assertEqual([], entry.get_unrecognized_lines()) + + def test_missing_fields(self): + """ + Parses a router status entry that's missing fields. + """ + + content = get_router_status_entry_v3(exclude = ('r', 's'), content = True) + self._expect_invalid_attr(content, "address") + + content = get_router_status_entry_v3(exclude = ('r',), content = True) + self._expect_invalid_attr(content, "address") + + content = get_router_status_entry_v3(exclude = ('s',), content = True) + self._expect_invalid_attr(content, "flags") + + def test_unrecognized_lines(self): + """ + Parses a router status entry with new keywords. + """ + + entry = get_router_status_entry_v3({'z': 'New tor feature: sparkly unicorns!'}) + self.assertEquals(['z New tor feature: sparkly unicorns!'], entry.get_unrecognized_lines()) + + def test_proceeding_line(self): + """ + Includes content prior to the 'r' line. + """ + + content = 'z some stuff\n' + get_router_status_entry_v3(content = True) + self._expect_invalid_attr(content, "_unrecognized_lines", ['z some stuff']) + + def test_blank_lines(self): + """ + Includes blank lines, which should be ignored. + """ + + content = get_router_status_entry_v3(content = True) + "\n\nv Tor 0.2.2.35\n\n" + entry = RouterStatusEntryV3(content) + self.assertEqual("Tor 0.2.2.35", entry.version_line) + + def test_duplicate_lines(self): + """ + Duplicates linesin the entry. + """ + + lines = get_router_status_entry_v3(content = True).split("\n") + + for i in xrange(len(lines)): + content = "\n".join(lines[:i] + [lines[i]] + lines[i:]) + self.assertRaises(ValueError, RouterStatusEntryV3, content) + + entry = RouterStatusEntryV3(content, False) + self.assertEqual("caerSidi", entry.nickname) + + def test_missing_r_field(self): + """ + Excludes fields from the 'r' line. + """ + + components = ( + ('nickname', 'caerSidi'), + ('fingerprint', 'p1aag7VwarGxqctS7/fS0y5FU+s'), + ('digest', 'oQZFLYe9e4A7bOkWKR7TaNxb0JE'), + ('published', '2012-08-06 11:19:31'), + ('address', '71.35.150.29'), + ('or_port', '9001'), + ('dir_port', '0'), + ) + + for attr, value in components: + # construct the 'r' line without this field + test_components = [comp[1] for comp in components] + test_components.remove(value) + r_line = ' '.join(test_components) + + content = get_router_status_entry_v3({'r': r_line}, content = True) + self._expect_invalid_attr(content, attr) + + def test_malformed_nickname(self): + """ + Parses an 'r' line with a malformed nickname. + """ + + test_values = ( + "", + "saberrider2008ReallyLongNickname", # too long + "$aberrider2008", # invalid characters + ) + + for value in test_values: + r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1].replace("caerSidi", value) + content = get_router_status_entry_v3({'r': r_line}, content = True) + + # TODO: Initial whitespace is consumed as part of the keyword/value + # divider. This is a bug in the case of V3 router status entries, but + # proper behavior for V2 router status entries and server/extrainfo + # descriptors. + # + # I'm inclined to leave this as-is for the moment since fixing it + # requires special KEYWORD_LINE handling, and the only result of this bug + # is that our validation doesn't catch the new SP restriction on V3 + # entries. + + if value == "": value = None + + self._expect_invalid_attr(content, "nickname", value) + + def test_malformed_fingerprint(self): + """ + Parses an 'r' line with a malformed fingerprint. + """ + + test_values = ( + "", + "zzzzz", + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + ) + + for value in test_values: + r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1].replace("p1aag7VwarGxqctS7/fS0y5FU+s", value) + content = get_router_status_entry_v3({'r': r_line}, content = True) + self._expect_invalid_attr(content, "fingerprint") + + def test_malformed_published_date(self): + """ + Parses an 'r' line with a malformed published date. + """ + + test_values = ( + "", + "2012-08-06 11:19:", + "2012-08-06 11:19:71", + "2012-08-06 11::31", + "2012-08-06 11:79:31", + "2012-08-06 :19:31", + "2012-08-06 41:19:31", + "2012-08- 11:19:31", + "2012-08-86 11:19:31", + "2012--06 11:19:31", + "2012-38-06 11:19:31", + "-08-06 11:19:31", + "2012-08-06 11:19:31", + ) + + for value in test_values: + r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1].replace("2012-08-06 11:19:31", value) + content = get_router_status_entry_v3({'r': r_line}, content = True) + self._expect_invalid_attr(content, "published") + + def test_malformed_address(self): + """ + Parses an 'r' line with a malformed address. + """ + + test_values = ( + "", + "71.35.150.", + "71.35..29", + "71.35.150", + "71.35.150.256", + ) + + for value in test_values: + r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1].replace("71.35.150.29", value) + content = get_router_status_entry_v3({'r': r_line}, content = True) + self._expect_invalid_attr(content, "address", value) + + def test_malformed_port(self): + """ + Parses an 'r' line with a malformed ORPort or DirPort. + """ + + test_values = ( + "", + "-1", + "399482", + "blarg", + ) + + for value in test_values: + for include_or_port in (False, True): + for include_dir_port in (False, True): + if not include_or_port and not include_dir_port: + continue + + r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1] + + if include_or_port: + r_line = r_line.replace(" 9001 ", " %s " % value) + + if include_dir_port: + r_line = r_line[:-1] + value + + attr = "or_port" if include_or_port else "dir_port" + expected = int(value) if value.isdigit() else None + + content = get_router_status_entry_v3({'r': r_line}, content = True) + self._expect_invalid_attr(content, attr, expected) + + def test_flags(self): + """ + Handles a variety of flag inputs. + """ + + test_values = { + "": [], + "Fast": [Flag.FAST], + "Fast Valid": [Flag.FAST, Flag.VALID], + "Ugabuga": ["Ugabuga"], + } + + for s_line, expected in test_values.items(): + entry = get_router_status_entry_v3({'s': s_line}) + self.assertEquals(expected, entry.flags) + + # tries some invalid inputs + test_values = { + "Fast ": [Flag.FAST, "", "", ""], + "Fast Valid": [Flag.FAST, "", Flag.VALID], + "Fast Fast": [Flag.FAST, Flag.FAST], + } + + for s_line, expected in test_values.items(): + content = get_router_status_entry_v3({'s': s_line}, content = True) + self._expect_invalid_attr(content, "flags", expected) + + def test_versions(self): + """ + Handles a variety of version inputs. + """ + + test_values = { + "Tor 0.2.2.35": Version("0.2.2.35"), + "Tor 0.1.2": Version("0.1.2"), + "Torr new_stuff": None, + "new_stuff and stuff": None, + } + + for v_line, expected in test_values.items(): + entry = get_router_status_entry_v3({'v': v_line}) + self.assertEquals(expected, entry.version) + self.assertEquals(v_line, entry.version_line) + + # tries an invalid input + content = get_router_status_entry_v3({'v': "Tor ugabuga"}, content = True) + self._expect_invalid_attr(content, "version") + + def test_bandwidth(self): + """ + Handles a variety of 'w' lines. + """ + + test_values = { + "Bandwidth=0": (0, None, []), + "Bandwidth=63138": (63138, None, []), + "Bandwidth=11111 Measured=482": (11111, 482, []), + "Bandwidth=11111 Measured=482 Blarg!": (11111, 482, ["Blarg!"]), + } + + for w_line, expected in test_values.items(): + entry = get_router_status_entry_v3({'w': w_line}) + self.assertEquals(expected[0], entry.bandwidth) + self.assertEquals(expected[1], entry.measured) + self.assertEquals(expected[2], entry.unrecognized_bandwidth_entries) + + # tries some invalid inputs + test_values = ( + "", + "blarg", + "Bandwidth", + "Bandwidth=", + "Bandwidth:0", + "Bandwidth 0", + "Bandwidth=-10", + "Bandwidth=10 Measured", + "Bandwidth=10 Measured=", + "Bandwidth=10 Measured=-50", + ) + + for w_line in test_values: + content = get_router_status_entry_v3({'w': w_line}, content = True) + self._expect_invalid_attr(content) + + def test_exit_policy(self): + """ + Handles a variety of 'p' lines. + """ + + test_values = { + "reject 1-65535": MicrodescriptorExitPolicy("reject 1-65535"), + "accept 80,110,143,443": MicrodescriptorExitPolicy("accept 80,110,143,443"), + } + + for p_line, expected in test_values.items(): + entry = get_router_status_entry_v3({'p': p_line}) + self.assertEquals(expected, entry.exit_policy) + + # tries some invalid inputs + test_values = ( + "", + "blarg", + "reject -50", + "accept 80,", + ) + + for p_line in test_values: + content = get_router_status_entry_v3({'p': p_line}, content = True) + self._expect_invalid_attr(content, "exit_policy") + + def test_microdescriptor_hashes(self): + """ + Handles a variety of 'm' lines. + """ + + test_values = { + "8,9,10,11,12": + [([8, 9, 10, 11, 12], {})], + "8,9,10,11,12 sha256=g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs": + [([8, 9, 10, 11, 12], {"sha256": "g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs"})], + "8,9,10,11,12 sha256=g1vx9si329muxV md5=3tquWIXXySNOIwRGMeAESKs/v4DWs": + [([8, 9, 10, 11, 12], {"sha256": "g1vx9si329muxV", "md5": "3tquWIXXySNOIwRGMeAESKs/v4DWs"})], + } + + # we need a document that's a vote + mock_document = lambda x: x # just need anything with a __dict__ + mock_document.__dict__["is_vote"] = True + mock_document.__dict__["is_consensus"] = False + + for m_line, expected in test_values.items(): + content = get_router_status_entry_v3({'m': m_line}, content = True) + entry = RouterStatusEntryV3(content, document = mock_document) + self.assertEquals(expected, entry.microdescriptor_hashes) + + # try without a document + content = get_router_status_entry_v3({'m': "8,9,10,11,12"}, content = True) + self._expect_invalid_attr(content, "microdescriptor_hashes") + + # tries some invalid inputs + test_values = ( + "", + "4,a,2", + "1,2,3 stuff", + ) + + for m_line in test_values: + content = get_router_status_entry_v3({'m': m_line}, content = True) + self.assertRaises(ValueError, RouterStatusEntryV3, content, True, mock_document) + + def _expect_invalid_attr(self, content, attr = None, expected_value = None): + """ + Asserts that construction will fail due to content having a malformed + attribute. If an attr is provided then we check that it matches an expected + value when we're constructed without validation. + """ + + self.assertRaises(ValueError, RouterStatusEntryV3, content) + entry = RouterStatusEntryV3(content, False) + + if attr: + self.assertEquals(expected_value, getattr(entry, attr)) + else: + self.assertEquals("caerSidi", entry.nickname) +