January 2015 - tor-commits - lists.torproject.org

[stem/master] Lazy loading server descriptor's contact and platform line
by atagar＠torproject.org 25 Jan '15

25 Jan '15

commit a30d624aea54ff60cc6dda850dda4356d68617d4 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 25 10:57:39 2015 -0800 Lazy loading server descriptor's contact and platform line These lines are special in that they're raw bytes rather than unicode (that is to say, they're not necessarily recognizable text). As such I left their parsing alone which was eager loading, but on reflection we get a nice performance boost by making these lazy too. --- stem/descriptor/__init__.py | 41 ++++++++++++---------------------- stem/descriptor/server_descriptor.py | 16 ++++++------- 2 files changed, 21 insertions(+), 36 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 1a7a097..a5eb87d 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -326,6 +326,20 @@ def _parse_simple_line(keyword, attribute): return _parse +def _parse_bytes_line(keyword, attribute): + def _parse(descriptor, entries): + line_match = re.search(stem.util.str_tools._to_bytes('^(opt )?%s(?:[%s]+(.*))?$' % (keyword, WHITESPACE)), descriptor.get_bytes(), re.MULTILINE) + result = None + + if line_match: + value = line_match.groups()[1] + result = b'' if value is None else value + + setattr(descriptor, attribute, result) + + return _parse + + def _parse_timestamp_line(keyword, attribute): # "<keyword>" YYYY-MM-DD HH:MM:SS @@ -503,33 +517,6 @@ class Descriptor(object): return self._raw_contents -def _get_bytes_field(keyword, content): - """ - Provides the value corresponding to the given keyword. This is handy to fetch - values specifically allowed to be arbitrary bytes prior to converting to - unicode. - - :param str keyword: line to look up - :param bytes content: content to look through - - :returns: **bytes** value on the given line, **None** if the line doesn't - exist - - :raises: **ValueError** if the content isn't bytes - """ - - if not isinstance(content, bytes): - raise ValueError('Content must be bytes, got a %s' % type(content)) - - line_match = re.search(stem.util.str_tools._to_bytes('^(opt )?%s(?:[%s]+(.*))?$' % (keyword, WHITESPACE)), content, re.MULTILINE) - - if line_match: - value = line_match.groups()[1] - return b'' if value is None else value - else: - return None - - def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False, skip = False, end_position = None, include_ending_keyword = False): """ Reads from the descriptor file until we get to one of the given keywords or reach the diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index c4f4bca..5786ab9 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -51,12 +51,12 @@ from stem.util import log from stem.descriptor import ( PGP_BLOCK_END, Descriptor, - _get_bytes_field, _get_descriptor_components, _read_until_keywords, _value, _values, _parse_simple_line, + _parse_bytes_line, _parse_timestamp_line, _parse_forty_character_hex, _parse_key_block, @@ -222,6 +222,8 @@ def _parse_bandwidth_line(descriptor, entries): def _parse_platform_line(descriptor, entries): # "platform" string + _parse_bytes_line('platform', 'platform')(descriptor, entries) + # The platform attribute was set earlier. This line can contain any # arbitrary data, but tor seems to report its version followed by the # os like the following... @@ -367,6 +369,7 @@ def _parse_exit_policy(descriptor, entries): del descriptor._unparsed_exit_policy +_parse_contact_line = _parse_bytes_line('contact', 'contact') _parse_published_line = _parse_timestamp_line('published', 'published') _parse_extrainfo_digest_line = _parse_forty_character_hex('extra-info-digest', 'extra_info_digest') _parse_read_history_line = functools.partial(_parse_history_line, 'read-history', 'read_history_end', 'read_history_interval', 'read_history_values') @@ -437,6 +440,7 @@ class ServerDescriptor(Descriptor): ATTRIBUTES = { 'nickname': (None, _parse_router_line), 'fingerprint': (None, _parse_fingerprint_line), + 'contact': (None, _parse_contact_line), 'published': (None, _parse_published_line), 'exit_policy': (None, _parse_exit_policy), @@ -445,6 +449,7 @@ class ServerDescriptor(Descriptor): 'socks_port': (None, _parse_router_line), 'dir_port': (None, _parse_router_line), + 'platform': (None, _parse_platform_line), 'tor_version': (None, _parse_platform_line), 'operating_system': (None, _parse_platform_line), 'uptime': (None, _parse_uptime_line), @@ -480,6 +485,7 @@ class ServerDescriptor(Descriptor): 'platform': _parse_platform_line, 'published': _parse_published_line, 'fingerprint': _parse_fingerprint_line, + 'contact': _parse_contact_line, 'hibernating': _parse_hibernating_line, 'extra-info-digest': _parse_extrainfo_digest_line, 'hidden-service-dir': _parse_hidden_service_dir_line, @@ -493,7 +499,6 @@ class ServerDescriptor(Descriptor): 'caches-extra-info': _parse_caches_extra_info_line, 'family': _parse_family_line, 'eventdns': _parse_eventdns_line, - 'contact': lambda descriptor, entries: None, # parsed as a bytes field earlier } def __init__(self, raw_contents, validate = True, annotations = None): @@ -515,13 +520,6 @@ class ServerDescriptor(Descriptor): """ super(ServerDescriptor, self).__init__(raw_contents, lazy_load = not validate) - - # Only a few things can be arbitrary bytes according to the dir-spec, so - # parsing them separately. - - self.platform = _get_bytes_field('platform', raw_contents) - self.contact = _get_bytes_field('contact', raw_contents) - self._annotation_lines = annotations if annotations else [] # A descriptor contains a series of 'keyword lines' which are simply a

1 0

[stem/master] Lazy load microdescriptor digest
by atagar＠torproject.org 25 Jan '15

25 Jan '15

commit b93ea3928b9e4f032f5f8d5c7400bc3932f39b8d Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 25 11:17:47 2015 -0800 Lazy load microdescriptor digest Not truly a line attribute, but we can treat it in a similar fashion. --- stem/descriptor/microdescriptor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index 5b8d0a3..c9389e4 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -169,6 +169,7 @@ def _parse_id_line(descriptor, entries): raise ValueError("'id' lines should contain both the key type and digest: id %s" % value) +_parse_digest = lambda descriptor, entries: setattr(descriptor, 'digest', hashlib.sha256(descriptor.get_bytes()).hexdigest().upper()) _parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY') _parse_ntor_onion_key_line = _parse_simple_line('ntor-onion-key', 'ntor_onion_key') _parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', _value('family', entries).split(' ')) @@ -206,6 +207,7 @@ class Microdescriptor(Descriptor): 'exit_policy_v6': (None, _parse_p6_line), 'identifier_type': (None, _parse_id_line), 'identifier': (None, _parse_id_line), + 'digest': (None, _parse_digest), } PARSER_FOR_LINE = { @@ -220,13 +222,11 @@ class Microdescriptor(Descriptor): def __init__(self, raw_contents, validate = True, annotations = None): super(Microdescriptor, self).__init__(raw_contents, lazy_load = not validate) - - self.digest = hashlib.sha256(self.get_bytes()).hexdigest().upper() self._annotation_lines = annotations if annotations else [] - entries = _get_descriptor_components(raw_contents, validate) if validate: + self.digest = hashlib.sha256(self.get_bytes()).hexdigest().upper() self._parse(entries, validate) self._check_constraints(entries) else:

1 0

[stem/master] Router status entry lazy loading
by atagar＠torproject.org 25 Jan '15

25 Jan '15

commit 1fcb9751dcbd7393fba61a6f78902160ad77304d Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 25 10:25:29 2015 -0800 Router status entry lazy loading --- stem/descriptor/microdescriptor.py | 13 +- stem/descriptor/router_status_entry.py | 688 ++++++++++++--------------- stem/descriptor/server_descriptor.py | 31 +- test/unit/descriptor/router_status_entry.py | 13 +- 4 files changed, 320 insertions(+), 425 deletions(-) diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index fd9ef9b..3d9447f 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -66,7 +66,6 @@ Doing the same is trivial with server descriptors... import hashlib -import stem.descriptor.router_status_entry import stem.exit_policy from stem.descriptor import ( @@ -74,11 +73,15 @@ from stem.descriptor import ( _get_descriptor_components, _read_until_keywords, _value, - _values, _parse_simple_line, _parse_key_block, ) +from stem.descriptor.router_status_entry import ( + _parse_a_line, + _parse_p_line, +) + try: # added in python 3.2 from functools import lru_cache @@ -155,11 +158,6 @@ def _parse_file(descriptor_file, validate = True, **kwargs): break # done parsing descriptors -def _parse_a_line(descriptor, entries): - for value in _values('a', entries): - stem.descriptor.router_status_entry._parse_a_line(descriptor, value, True) - - def _parse_id_line(descriptor, entries): value = _value('id', entries) value_comp = value.split() @@ -174,7 +172,6 @@ def _parse_id_line(descriptor, entries): _parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY') _parse_ntor_onion_key_line = _parse_simple_line('ntor-onion-key', 'ntor_onion_key') _parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', _value('family', entries).split(' ')) -_parse_p_line = lambda descriptor, entries: stem.descriptor.router_status_entry._parse_p_line(descriptor, _value('p', entries), True) _parse_p6_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('p6', entries))) diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index ab93ada..f80c56b 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -29,6 +29,8 @@ import stem.util.str_tools from stem.descriptor import ( KEYWORD_LINE, Descriptor, + _value, + _values, _get_descriptor_components, _read_until_keywords, ) @@ -101,6 +103,248 @@ def _parse_file(document_file, validate, entry_class, entry_keyword = 'r', start break +def _parse_r_line(descriptor, entries): + # Parses a RouterStatusEntry's 'r' line. They're very nearly identical for + # all current entry types (v2, v3, and microdescriptor v3) with one little + # wrinkle: only the microdescriptor flavor excludes a 'digest' field. + # + # For v2 and v3 router status entries: + # "r" nickname identity digest publication IP ORPort DirPort + # example: r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0 + # + # For v3 microdescriptor router status entries: + # "r" nickname identity publication IP ORPort DirPort + # example: r Konata ARIJF2zbqirB9IwsW0mQznccWww 2012-09-24 13:40:40 69.64.48.168 9001 9030 + + value = _value('r', entries) + include_digest = not isinstance(descriptor, RouterStatusEntryMicroV3) + + r_comp = value.split(' ') + + # inject a None for the digest to normalize the field positioning + if not include_digest: + r_comp.insert(2, None) + + if len(r_comp) < 8: + expected_field_count = 'eight' if include_digest else 'seven' + raise ValueError("%s 'r' line must have %s values: r %s" % (descriptor._name(), expected_field_count, value)) + + if not stem.util.tor_tools.is_valid_nickname(r_comp[0]): + raise ValueError("%s nickname isn't valid: %s" % (descriptor._name(), r_comp[0])) + elif not stem.util.connection.is_valid_ipv4_address(r_comp[5]): + raise ValueError("%s address isn't a valid IPv4 address: %s" % (descriptor._name(), r_comp[5])) + elif not stem.util.connection.is_valid_port(r_comp[6]): + raise ValueError('%s ORPort is invalid: %s' % (descriptor._name(), r_comp[6])) + elif not stem.util.connection.is_valid_port(r_comp[7], allow_zero = True): + raise ValueError('%s DirPort is invalid: %s' % (descriptor._name(), r_comp[7])) + + descriptor.nickname = r_comp[0] + descriptor.fingerprint = _base64_to_hex(r_comp[1]) + + if include_digest: + descriptor.digest = _base64_to_hex(r_comp[2]) + + descriptor.address = r_comp[5] + descriptor.or_port = int(r_comp[6]) + descriptor.dir_port = None if r_comp[7] == '0' else int(r_comp[7]) + + try: + published = '%s %s' % (r_comp[3], r_comp[4]) + descriptor.published = stem.util.str_tools._parse_timestamp(published) + except ValueError: + raise ValueError("Publication time time wasn't parsable: r %s" % value) + + +def _parse_a_line(descriptor, entries): + # "a" SP address ":" portlist + # example: a [2001:888:2133:0:82:94:251:204]:9001 + + or_addresses = [] + + for value in _values('a', entries): + if ':' not in value: + raise ValueError("%s 'a' line must be of the form '[address]:[ports]': a %s" % (descriptor._name(), value)) + + address, port = value.rsplit(':', 1) + is_ipv6 = address.startswith('[') and address.endswith(']') + + if is_ipv6: + address = address[1:-1] # remove brackets + + if not ((not is_ipv6 and stem.util.connection.is_valid_ipv4_address(address)) or + (is_ipv6 and stem.util.connection.is_valid_ipv6_address(address))): + raise ValueError("%s 'a' line must start with an IPv6 address: a %s" % (descriptor._name(), value)) + + if stem.util.connection.is_valid_port(port): + or_addresses.append((address, int(port), is_ipv6)) + else: + raise ValueError("%s 'a' line had an invalid port (%s): a %s" % (descriptor._name(), port, value)) + + descriptor.or_addresses = or_addresses + + +def _parse_s_line(descriptor, entries): + # "s" Flags + # example: s Named Running Stable Valid + + value = _value('s', entries) + flags = [] if value == '' else value.split(' ') + descriptor.flags = flags + + for flag in flags: + if flags.count(flag) > 1: + raise ValueError('%s had duplicate flags: s %s' % (descriptor._name(), value)) + elif flag == "": + raise ValueError("%s had extra whitespace on its 's' line: s %s" % (descriptor._name(), value)) + + +def _parse_v_line(descriptor, entries): + # "v" version + # example: v Tor 0.2.2.35 + # + # The spec says that if this starts with "Tor " then what follows is a + # tor version. If not then it has "upgraded to a more sophisticated + # protocol versioning system". + + value = _value('v', entries) + descriptor.version_line = value + + if value.startswith('Tor '): + try: + descriptor.version = stem.version._get_version(value[4:]) + except ValueError as exc: + raise ValueError('%s has a malformed tor version (%s): v %s' % (descriptor._name(), exc, value)) + + +def _parse_w_line(descriptor, entries): + # "w" "Bandwidth=" INT ["Measured=" INT] ["Unmeasured=1"] + # example: w Bandwidth=7980 + + value = _value('w', entries) + w_comp = value.split(' ') + + if len(w_comp) < 1: + raise ValueError("%s 'w' line is blank: w %s" % (descriptor._name(), value)) + elif not w_comp[0].startswith('Bandwidth='): + raise ValueError("%s 'w' line needs to start with a 'Bandwidth=' entry: w %s" % (descriptor._name(), value)) + + for w_entry in w_comp: + if '=' in w_entry: + w_key, w_value = w_entry.split('=', 1) + else: + w_key, w_value = w_entry, None + + if w_key == 'Bandwidth': + if not (w_value and w_value.isdigit()): + raise ValueError("%s 'Bandwidth=' entry needs to have a numeric value: w %s" % (descriptor._name(), value)) + + descriptor.bandwidth = int(w_value) + elif w_key == 'Measured': + if not (w_value and w_value.isdigit()): + raise ValueError("%s 'Measured=' entry needs to have a numeric value: w %s" % (descriptor._name(), value)) + + descriptor.measured = int(w_value) + elif w_key == 'Unmeasured': + if w_value != '1': + raise ValueError("%s 'Unmeasured=' should only have the value of '1': w %s" % (descriptor._name(), value)) + + descriptor.is_unmeasured = True + else: + descriptor.unrecognized_bandwidth_entries.append(w_entry) + + +def _parse_p_line(descriptor, entries): + # "p" ("accept" / "reject") PortList + # p reject 1-65535 + # example: p accept 80,110,143,443,993,995,6660-6669,6697,7000-7001 + + value = _value('p', entries) + + try: + descriptor.exit_policy = stem.exit_policy.MicroExitPolicy(value) + except ValueError as exc: + raise ValueError('%s exit policy is malformed (%s): p %s' % (descriptor._name(), exc, value)) + + +def _parse_m_line(descriptor, entries): + # "m" methods 1*(algorithm "=" digest) + # example: m 8,9,10,11,12 sha256=g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs + + all_hashes = [] + + for value in _values('m', entries): + m_comp = value.split(' ') + + if not (descriptor.document and descriptor.document.is_vote): + vote_status = 'vote' if descriptor.document else '<undefined document>' + raise ValueError("%s 'm' line should only appear in votes (appeared in a %s): m %s" % (descriptor._name(), vote_status, value)) + elif len(m_comp) < 1: + raise ValueError("%s 'm' line needs to start with a series of methods: m %s" % (descriptor._name(), value)) + + try: + methods = [int(entry) for entry in m_comp[0].split(',')] + except ValueError: + raise ValueError('%s microdescriptor methods should be a series of comma separated integers: m %s' % (descriptor._name(), value)) + + hashes = {} + + for entry in m_comp[1:]: + if '=' not in entry: + raise ValueError("%s can only have a series of 'algorithm=digest' mappings after the methods: m %s" % (descriptor._name(), value)) + + hash_name, digest = entry.split('=', 1) + hashes[hash_name] = digest + + all_hashes.append((methods, hashes)) + + descriptor.microdescriptor_hashes = all_hashes + + +def _parse_microdescriptor_m_line(descriptor, entries): + # "m" digest + # example: m aiUklwBrua82obG5AsTX+iEpkjQA2+AQHxZ7GwMfY70 + + descriptor.digest = _base64_to_hex(_value('m', entries), check_if_fingerprint = False) + + +def _base64_to_hex(identity, check_if_fingerprint = True): + """ + Decodes a base64 value to hex. For example... + + :: + + >>> _base64_to_hex('p1aag7VwarGxqctS7/fS0y5FU+s') + 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB' + + :param str identity: encoded fingerprint from the consensus + :param bool check_if_fingerprint: asserts that the result is a fingerprint if **True** + + :returns: **str** with the uppercase hex encoding of the relay's fingerprint + + :raises: **ValueError** if the result isn't a valid fingerprint + """ + + # trailing equal signs were stripped from the identity + missing_padding = len(identity) % 4 + identity += '=' * missing_padding + + try: + identity_decoded = base64.b64decode(stem.util.str_tools._to_bytes(identity)) + except (TypeError, binascii.Error): + raise ValueError("Unable to decode identity string '%s'" % identity) + + fingerprint = binascii.b2a_hex(identity_decoded).upper() + + if stem.prereq.is_python_3(): + fingerprint = stem.util.str_tools._to_unicode(fingerprint) + + if check_if_fingerprint: + if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): + raise ValueError("Decoded '%s' to be '%s', which isn't a valid fingerprint" % (identity, fingerprint)) + + return fingerprint + + class RouterStatusEntry(Descriptor): """ Information about an individual router stored within a network status @@ -122,7 +366,27 @@ class RouterStatusEntry(Descriptor): :var str version_line: versioning information reported by the relay """ - def __init__(self, content, validate, document): + ATTRIBUTES = { + 'nickname': (None, _parse_r_line), + 'fingerprint': (None, _parse_r_line), + 'published': (None, _parse_r_line), + 'address': (None, _parse_r_line), + 'or_port': (None, _parse_r_line), + 'dir_port': (None, _parse_r_line), + + 'flags': (None, _parse_s_line), + + 'version_line': (None, _parse_v_line), + 'version': (None, _parse_v_line), + } + + PARSER_FOR_LINE = { + 'r': _parse_r_line, + 's': _parse_s_line, + 'v': _parse_v_line, + } + + def __init__(self, content, validate = True, document = None): """ Parse a router descriptor in a network status document. @@ -134,51 +398,17 @@ class RouterStatusEntry(Descriptor): :raises: **ValueError** if the descriptor data is invalid """ - super(RouterStatusEntry, self).__init__(content) + super(RouterStatusEntry, self).__init__(content, lazy_load = not validate) content = stem.util.str_tools._to_unicode(content) self.document = document - - self.nickname = None - self.fingerprint = None - self.published = None - self.address = None - self.or_port = None - self.dir_port = None - - self.flags = None - - self.version_line = None - self.version = None - - self._unrecognized_lines = [] - entries = _get_descriptor_components(content, validate) if validate: self._check_constraints(entries) - - self._parse(entries, validate) - - def _parse(self, entries, validate): - """ - Parses the given content and applies the attributes. - - :param dict entries: keyword => (value, pgp key) entries - :param bool validate: checks validity if **True** - - :raises: **ValueError** if a validity check fails - """ - - for keyword, values in list(entries.items()): - value, _, _ = values[0] - - if keyword == 's': - _parse_s_line(self, value, validate) - elif keyword == 'v': - _parse_v_line(self, value, validate) - else: - self._unrecognized_lines.append('%s %s' % (keyword, value)) + self._parse(entries, validate) + else: + self._entries = entries def _check_constraints(self, entries): """ @@ -225,15 +455,6 @@ class RouterStatusEntry(Descriptor): return () - def get_unrecognized_lines(self): - """ - Provides any unrecognized lines. - - :returns: list of unrecognized lines - """ - - return list(self._unrecognized_lines) - def _compare(self, other, method): if not isinstance(other, RouterStatusEntry): return False @@ -261,19 +482,9 @@ class RouterStatusEntryV2(RouterStatusEntry): a default value, others are left as **None** if undefined """ - def __init__(self, content, validate = True, document = None): - self.digest = None - super(RouterStatusEntryV2, self).__init__(content, validate, document) - - def _parse(self, entries, validate): - for keyword, values in list(entries.items()): - value, _, _ = values[0] - - if keyword == 'r': - _parse_r_line(self, value, validate, True) - del entries['r'] - - RouterStatusEntry._parse(self, entries, validate) + ATTRIBUTES = dict(RouterStatusEntry.ATTRIBUTES, **{ + 'digest': (None, _parse_r_line), + }) def _name(self, is_plural = False): if is_plural: @@ -331,45 +542,25 @@ class RouterStatusEntryV3(RouterStatusEntry): a default value, others are left as **None** if undefined """ - def __init__(self, content, validate = True, document = None): - self.or_addresses = [] - self.digest = None - - self.bandwidth = None - self.measured = None - self.is_unmeasured = False - self.unrecognized_bandwidth_entries = [] - - self.exit_policy = None - self.microdescriptor_hashes = [] - - super(RouterStatusEntryV3, self).__init__(content, validate, document) + ATTRIBUTES = dict(RouterStatusEntry.ATTRIBUTES, **{ + 'digest': (None, _parse_r_line), + 'or_addresses': ([], _parse_a_line), - def _parse(self, entries, validate): - for keyword, values in list(entries.items()): - value, _, _ = values[0] + 'bandwidth': (None, _parse_w_line), + 'measured': (None, _parse_w_line), + 'is_unmeasured': (False, _parse_w_line), + 'unrecognized_bandwidth_entries': ([], _parse_w_line), - if keyword == 'r': - _parse_r_line(self, value, validate, True) - del entries['r'] - elif keyword == 'a': - for entry, _, _ in values: - _parse_a_line(self, entry, validate) + 'exit_policy': (None, _parse_p_line), + 'microdescriptor_hashes': ([], _parse_m_line), + }) - del entries['a'] - elif keyword == 'w': - _parse_w_line(self, value, validate) - del entries['w'] - elif keyword == 'p': - _parse_p_line(self, value, validate) - del entries['p'] - elif keyword == 'm': - for entry, _, _ in values: - _parse_m_line(self, entry, validate) - - del entries['m'] - - RouterStatusEntry._parse(self, entries, validate) + PARSER_FOR_LINE = dict(RouterStatusEntry.PARSER_FOR_LINE, **{ + 'a': _parse_a_line, + 'w': _parse_w_line, + 'p': _parse_p_line, + 'm': _parse_m_line, + }) def _name(self, is_plural = False): if is_plural: @@ -417,34 +608,19 @@ class RouterStatusEntryMicroV3(RouterStatusEntry): a default value, others are left as **None** if undefined """ - def __init__(self, content, validate = True, document = None): - self.bandwidth = None - self.measured = None - self.is_unmeasured = False - self.unrecognized_bandwidth_entries = [] - - self.digest = None - - super(RouterStatusEntryMicroV3, self).__init__(content, validate, document) - - def _parse(self, entries, validate): - for keyword, values in list(entries.items()): - value, _, _ = values[0] + ATTRIBUTES = dict(RouterStatusEntry.ATTRIBUTES, **{ + 'bandwidth': (None, _parse_w_line), + 'measured': (None, _parse_w_line), + 'is_unmeasured': (False, _parse_w_line), + 'unrecognized_bandwidth_entries': ([], _parse_w_line), - if keyword == 'r': - _parse_r_line(self, value, validate, False) - del entries['r'] - elif keyword == 'w': - _parse_w_line(self, value, validate) - del entries['w'] - elif keyword == 'm': - # "m" digest - # example: m aiUklwBrua82obG5AsTX+iEpkjQA2+AQHxZ7GwMfY70 + 'digest': (None, _parse_microdescriptor_m_line), + }) - self.digest = _base64_to_hex(value, validate, False) - del entries['m'] - - RouterStatusEntry._parse(self, entries, validate) + PARSER_FOR_LINE = dict(RouterStatusEntry.PARSER_FOR_LINE, **{ + 'w': _parse_w_line, + 'm': _parse_microdescriptor_m_line, + }) def _name(self, is_plural = False): if is_plural: @@ -472,269 +648,3 @@ class RouterStatusEntryMicroV3(RouterStatusEntry): def __le__(self, other): return self._compare(other, lambda s, o: s <= o) - - -def _parse_r_line(desc, value, validate, include_digest = True): - # Parses a RouterStatusEntry's 'r' line. They're very nearly identical for - # all current entry types (v2, v3, and microdescriptor v3) with one little - # wrinkle: only the microdescriptor flavor excludes a 'digest' field. - # - # For v2 and v3 router status entries: - # "r" nickname identity digest publication IP ORPort DirPort - # example: r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0 - # - # For v3 microdescriptor router status entries: - # "r" nickname identity publication IP ORPort DirPort - # example: r Konata ARIJF2zbqirB9IwsW0mQznccWww 2012-09-24 13:40:40 69.64.48.168 9001 9030 - - r_comp = value.split(' ') - - # inject a None for the digest to normalize the field positioning - if not include_digest: - r_comp.insert(2, None) - - if len(r_comp) < 8: - if not validate: - return - - expected_field_count = 'eight' if include_digest else 'seven' - raise ValueError("%s 'r' line must have %s values: r %s" % (desc._name(), expected_field_count, value)) - - if validate: - if not stem.util.tor_tools.is_valid_nickname(r_comp[0]): - raise ValueError("%s nickname isn't valid: %s" % (desc._name(), r_comp[0])) - elif not stem.util.connection.is_valid_ipv4_address(r_comp[5]): - raise ValueError("%s address isn't a valid IPv4 address: %s" % (desc._name(), r_comp[5])) - elif not stem.util.connection.is_valid_port(r_comp[6]): - raise ValueError('%s ORPort is invalid: %s' % (desc._name(), r_comp[6])) - elif not stem.util.connection.is_valid_port(r_comp[7], allow_zero = True): - raise ValueError('%s DirPort is invalid: %s' % (desc._name(), r_comp[7])) - elif not (r_comp[6].isdigit() and r_comp[7].isdigit()): - return - - desc.nickname = r_comp[0] - desc.fingerprint = _base64_to_hex(r_comp[1], validate) - - if include_digest: - desc.digest = _base64_to_hex(r_comp[2], validate) - - desc.address = r_comp[5] - desc.or_port = int(r_comp[6]) - desc.dir_port = None if r_comp[7] == '0' else int(r_comp[7]) - - try: - published = '%s %s' % (r_comp[3], r_comp[4]) - desc.published = stem.util.str_tools._parse_timestamp(published) - except ValueError: - if validate: - raise ValueError("Publication time time wasn't parsable: r %s" % value) - - -def _parse_a_line(desc, value, validate): - # "a" SP address ":" portlist - # example: a [2001:888:2133:0:82:94:251:204]:9001 - - if ':' not in value: - if not validate: - return - - raise ValueError("%s 'a' line must be of the form '[address]:[ports]': a %s" % (desc._name(), value)) - - address, port = value.rsplit(':', 1) - is_ipv6 = address.startswith('[') and address.endswith(']') - - if is_ipv6: - address = address[1:-1] # remove brackets - - if not ((not is_ipv6 and stem.util.connection.is_valid_ipv4_address(address)) or - (is_ipv6 and stem.util.connection.is_valid_ipv6_address(address))): - if not validate: - return - else: - raise ValueError("%s 'a' line must start with an IPv6 address: a %s" % (desc._name(), value)) - - if stem.util.connection.is_valid_port(port): - desc.or_addresses.append((address, int(port), is_ipv6)) - elif validate: - raise ValueError("%s 'a' line had an invalid port (%s): a %s" % (desc._name(), port, value)) - - -def _parse_s_line(desc, value, validate): - # "s" Flags - # example: s Named Running Stable Valid - - flags = [] if value == '' else value.split(' ') - desc.flags = flags - - if validate: - for flag in flags: - if flags.count(flag) > 1: - raise ValueError('%s had duplicate flags: s %s' % (desc._name(), value)) - elif flag == "": - raise ValueError("%s had extra whitespace on its 's' line: s %s" % (desc._name(), value)) - - -def _parse_v_line(desc, value, validate): - # "v" version - # example: v Tor 0.2.2.35 - # - # The spec says that if this starts with "Tor " then what follows is a - # tor version. If not then it has "upgraded to a more sophisticated - # protocol versioning system". - - desc.version_line = value - - if value.startswith('Tor '): - try: - desc.version = stem.version._get_version(value[4:]) - except ValueError as exc: - if validate: - raise ValueError('%s has a malformed tor version (%s): v %s' % (desc._name(), exc, value)) - - -def _parse_w_line(desc, value, validate): - # "w" "Bandwidth=" INT ["Measured=" INT] ["Unmeasured=1"] - # example: w Bandwidth=7980 - - w_comp = value.split(' ') - - if len(w_comp) < 1: - if not validate: - return - - raise ValueError("%s 'w' line is blank: w %s" % (desc._name(), value)) - elif not w_comp[0].startswith('Bandwidth='): - if not validate: - return - - raise ValueError("%s 'w' line needs to start with a 'Bandwidth=' entry: w %s" % (desc._name(), value)) - - for w_entry in w_comp: - if '=' in w_entry: - w_key, w_value = w_entry.split('=', 1) - else: - w_key, w_value = w_entry, None - - if w_key == 'Bandwidth': - if not (w_value and w_value.isdigit()): - if not validate: - return - - raise ValueError("%s 'Bandwidth=' entry needs to have a numeric value: w %s" % (desc._name(), value)) - - desc.bandwidth = int(w_value) - elif w_key == 'Measured': - if not (w_value and w_value.isdigit()): - if not validate: - return - - raise ValueError("%s 'Measured=' entry needs to have a numeric value: w %s" % (desc._name(), value)) - - desc.measured = int(w_value) - elif w_key == 'Unmeasured': - if validate and w_value != '1': - raise ValueError("%s 'Unmeasured=' should only have the value of '1': w %s" % (desc._name(), value)) - - desc.is_unmeasured = True - else: - desc.unrecognized_bandwidth_entries.append(w_entry) - - -def _parse_p_line(desc, value, validate): - # "p" ("accept" / "reject") PortList - # p reject 1-65535 - # example: p accept 80,110,143,443,993,995,6660-6669,6697,7000-7001 - - try: - desc.exit_policy = stem.exit_policy.MicroExitPolicy(value) - except ValueError as exc: - if not validate: - return - - raise ValueError('%s exit policy is malformed (%s): p %s' % (desc._name(), exc, value)) - - -def _parse_m_line(desc, value, validate): - # "m" methods 1*(algorithm "=" digest) - # example: m 8,9,10,11,12 sha256=g1vx9si329muxV3tquWIXXySNOIwRGMeAESKs/v4DWs - - m_comp = value.split(' ') - - if not (desc.document and desc.document.is_vote): - if not validate: - return - - vote_status = 'vote' if desc.document else '<undefined document>' - raise ValueError("%s 'm' line should only appear in votes (appeared in a %s): m %s" % (desc._name(), vote_status, value)) - elif len(m_comp) < 1: - if not validate: - return - - raise ValueError("%s 'm' line needs to start with a series of methods: m %s" % (desc._name(), value)) - - try: - methods = [int(entry) for entry in m_comp[0].split(',')] - except ValueError: - if not validate: - return - - raise ValueError('%s microdescriptor methods should be a series of comma separated integers: m %s' % (desc._name(), value)) - - hashes = {} - - for entry in m_comp[1:]: - if '=' not in entry: - if not validate: - continue - - raise ValueError("%s can only have a series of 'algorithm=digest' mappings after the methods: m %s" % (desc._name(), value)) - - hash_name, digest = entry.split('=', 1) - hashes[hash_name] = digest - - desc.microdescriptor_hashes.append((methods, hashes)) - - -def _base64_to_hex(identity, validate, check_if_fingerprint = True): - """ - Decodes a base64 value to hex. For example... - - :: - - >>> _base64_to_hex('p1aag7VwarGxqctS7/fS0y5FU+s', True) - 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB' - - :param str identity: encoded fingerprint from the consensus - :param bool validate: checks validity if **True** - :param bool check_if_fingerprint: asserts that the result is a fingerprint if **True** - - :returns: **str** with the uppercase hex encoding of the relay's fingerprint - - :raises: **ValueError** if the result isn't a valid fingerprint - """ - - # trailing equal signs were stripped from the identity - missing_padding = len(identity) % 4 - identity += '=' * missing_padding - - try: - identity_decoded = base64.b64decode(stem.util.str_tools._to_bytes(identity)) - except (TypeError, binascii.Error): - if not validate: - return None - - raise ValueError("Unable to decode identity string '%s'" % identity) - - fingerprint = binascii.b2a_hex(identity_decoded).upper() - - if stem.prereq.is_python_3(): - fingerprint = stem.util.str_tools._to_unicode(fingerprint) - - if check_if_fingerprint: - if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): - if not validate: - return None - - raise ValueError("Decoded '%s' to be '%s', which isn't a valid fingerprint" % (identity, fingerprint)) - - return fingerprint diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 1cdf9be..c4f4bca 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -537,6 +537,16 @@ class ServerDescriptor(Descriptor): if validate: self._parse(entries, validate) + + _parse_exit_policy(self, entries) + + # if we have a negative uptime and a tor version that shouldn't exhibit + # this bug then fail validation + + if validate and self.uptime and self.tor_version: + if self.uptime < 0 and self.tor_version >= stem.version.Version('0.1.2.7'): + raise ValueError("Descriptor for version '%s' had a negative uptime value: %i" % (self.tor_version, self.uptime)) + self._check_constraints(entries) else: self._entries = entries @@ -588,27 +598,6 @@ class ServerDescriptor(Descriptor): return self._annotation_lines - def _parse(self, entries, validate): - """ - Parses a series of 'keyword => (value, pgp block)' mappings and applies - them as attributes. - - :param dict entries: descriptor contents to be applied - :param bool validate: checks the validity of descriptor content if **True** - - :raises: **ValueError** if an error occurs in validation - """ - - super(ServerDescriptor, self)._parse(entries, validate) - _parse_exit_policy(self, entries) - - # if we have a negative uptime and a tor version that shouldn't exhibit - # this bug then fail validation - - if validate and self.uptime and self.tor_version: - if self.uptime < 0 and self.tor_version >= stem.version.Version('0.1.2.7'): - raise ValueError("Descriptor for version '%s' had a negative uptime value: %i" % (self.tor_version, self.uptime)) - def _check_constraints(self, entries): """ Does a basic check that the entries conform to this descriptor type's diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py index 50924de..1243a55 100644 --- a/test/unit/descriptor/router_status_entry.py +++ b/test/unit/descriptor/router_status_entry.py @@ -39,8 +39,7 @@ class TestRouterStatusEntry(unittest.TestCase): # checks with some malformed inputs for arg in ('', '20wYcb', '20wYcb' * 30): - self.assertRaises(ValueError, _base64_to_hex, arg, True) - self.assertEqual(None, _base64_to_hex(arg, False)) + self.assertRaises(ValueError, _base64_to_hex, arg) def test_minimal_v2(self): """ @@ -138,7 +137,8 @@ class TestRouterStatusEntry(unittest.TestCase): """ content = b'z some stuff\n' + get_router_status_entry_v3(content = True) - self._expect_invalid_attr(content, '_unrecognized_lines', ['z some stuff']) + self.assertRaises(ValueError, RouterStatusEntryV3, content) + self.assertEqual(['z some stuff'], RouterStatusEntryV3(content, False).get_unrecognized_lines()) def test_blank_lines(self): """ @@ -215,7 +215,7 @@ class TestRouterStatusEntry(unittest.TestCase): if value == '': value = None - self._expect_invalid_attr(content, 'nickname', value) + self._expect_invalid_attr(content, 'nickname') def test_malformed_fingerprint(self): """ @@ -275,7 +275,7 @@ class TestRouterStatusEntry(unittest.TestCase): for value in test_values: r_line = ROUTER_STATUS_ENTRY_V3_HEADER[0][1].replace('71.35.150.29', value) content = get_router_status_entry_v3({'r': r_line}, content = True) - self._expect_invalid_attr(content, 'address', value) + self._expect_invalid_attr(content, 'address') def test_malformed_port(self): """ @@ -304,10 +304,9 @@ class TestRouterStatusEntry(unittest.TestCase): r_line = r_line[:-1] + value attr = 'or_port' if include_or_port else 'dir_port' - expected = int(value) if value.isdigit() else None content = get_router_status_entry_v3({'r': r_line}, content = True) - self._expect_invalid_attr(content, attr, expected) + self._expect_invalid_attr(content, attr) def test_ipv6_addresses(self): """

1 0

[stem/master] Moving bulk of server descriptor parsing to helpers
by atagar＠torproject.org 25 Jan '15

25 Jan '15

commit 99e3c7b8c3f8417324290d5bf7952393d968396a Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 4 16:41:12 2015 -0800 Moving bulk of server descriptor parsing to helpers Shifting actual parsing to helper functions so we'll be able to use these for lazy loading. This makes us a little less tolerant of garbage data but, honestly, shouldn't be a bad thing. --- stem/descriptor/server_descriptor.py | 455 +++++++++++++++-------------- test/unit/descriptor/server_descriptor.py | 8 +- 2 files changed, 244 insertions(+), 219 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 6726077..671e96d 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -167,6 +167,203 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True, **kwargs): break # done parsing descriptors +def _parse_router_line(descriptor, value): + # "router" nickname address ORPort SocksPort DirPort + + router_comp = value.split() + + if len(router_comp) < 5: + raise ValueError('Router line must have five values: router %s' % value) + elif not stem.util.tor_tools.is_valid_nickname(router_comp[0]): + raise ValueError("Router line entry isn't a valid nickname: %s" % router_comp[0]) + elif not stem.util.connection.is_valid_ipv4_address(router_comp[1]): + raise ValueError("Router line entry isn't a valid IPv4 address: %s" % router_comp[1]) + elif not stem.util.connection.is_valid_port(router_comp[2], allow_zero = True): + raise ValueError("Router line's ORPort is invalid: %s" % router_comp[2]) + elif not stem.util.connection.is_valid_port(router_comp[3], allow_zero = True): + raise ValueError("Router line's SocksPort is invalid: %s" % router_comp[3]) + elif not stem.util.connection.is_valid_port(router_comp[4], allow_zero = True): + raise ValueError("Router line's DirPort is invalid: %s" % router_comp[4]) + + descriptor.nickname = router_comp[0] + descriptor.address = router_comp[1] + descriptor.or_port = int(router_comp[2]) + descriptor.socks_port = None if router_comp[3] == '0' else int(router_comp[3]) + descriptor.dir_port = None if router_comp[4] == '0' else int(router_comp[4]) + + +def _parse_bandwidth_line(descriptor, value): + # "bandwidth" bandwidth-avg bandwidth-burst bandwidth-observed + + bandwidth_comp = value.split() + + if len(bandwidth_comp) < 3: + raise ValueError('Bandwidth line must have three values: bandwidth %s' % value) + elif not bandwidth_comp[0].isdigit(): + raise ValueError("Bandwidth line's average rate isn't numeric: %s" % bandwidth_comp[0]) + elif not bandwidth_comp[1].isdigit(): + raise ValueError("Bandwidth line's burst rate isn't numeric: %s" % bandwidth_comp[1]) + elif not bandwidth_comp[2].isdigit(): + raise ValueError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2]) + + descriptor.average_bandwidth = int(bandwidth_comp[0]) + descriptor.burst_bandwidth = int(bandwidth_comp[1]) + descriptor.observed_bandwidth = int(bandwidth_comp[2]) + + +def _parse_platform_line(descriptor, value): + # "platform" string + + # The platform attribute was set earlier. This line can contain any + # arbitrary data, but tor seems to report its version followed by the + # os like the following... + # + # platform Tor 0.2.2.35 (git-73ff13ab3cc9570d) on Linux x86_64 + # + # There's no guarantee that we'll be able to pick these out the + # version, but might as well try to save our caller the effort. + + platform_match = re.match('^(?:node-)?Tor (\S*).* on (.*)$', value) + + if platform_match: + version_str, descriptor.operating_system = platform_match.groups() + + try: + descriptor.tor_version = stem.version._get_version(version_str) + except ValueError: + pass + + +def _parse_published_line(descriptor, value): + # "published" YYYY-MM-DD HH:MM:SS + + try: + descriptor.published = stem.util.str_tools._parse_timestamp(value) + except ValueError: + raise ValueError("Published line's time wasn't parsable: published %s" % value) + + +def _parse_fingerprint_line(descriptor, value): + # This is forty hex digits split into space separated groups of four. + # Checking that we match this pattern. + + fingerprint = value.replace(' ', '') + + for grouping in value.split(' '): + if len(grouping) != 4: + raise ValueError('Fingerprint line should have groupings of four hex digits: %s' % value) + + if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): + raise ValueError('Tor relay fingerprints consist of forty hex digits: %s' % value) + + descriptor.fingerprint = fingerprint + + +def _parse_hibernating_line(descriptor, value): + # "hibernating" 0|1 (in practice only set if one) + + if value not in ('0', '1'): + raise ValueError('Hibernating line had an invalid value, must be zero or one: %s' % value) + + descriptor.hibernating = value == '1' + + +def _parse_extrainfo_digest_line(descriptor, value): + # this is forty hex digits which just so happens to be the same a + # fingerprint + + if not stem.util.tor_tools.is_valid_fingerprint(value): + raise ValueError('Extra-info digests should consist of forty hex digits: %s' % value) + + descriptor.extra_info_digest = value + + +def _parse_hidden_service_dir_line(descriptor, value): + if value: + descriptor.hidden_service_dir = value.split(' ') + else: + descriptor.hidden_service_dir = ['2'] + + +def _parse_uptime_line(descriptor, value): + # We need to be tolerant of negative uptimes to accommodate a past tor + # bug... + # + # Changes in version 0.1.2.7-alpha - 2007-02-06 + # - If our system clock jumps back in time, don't publish a negative + # uptime in the descriptor. Also, don't let the global rate limiting + # buckets go absurdly negative. + # + # After parsing all of the attributes we'll double check that negative + # uptimes only occurred prior to this fix. + + try: + descriptor.uptime = int(value) + except ValueError: + raise ValueError('Uptime line must have an integer value: %s' % value) + + +def _parse_protocols_line(descriptor, value): + protocols_match = re.match('^Link (.*) Circuit (.*)$', value) + + if not protocols_match: + raise ValueError('Protocols line did not match the expected pattern: protocols %s' % value) + + link_versions, circuit_versions = protocols_match.groups() + descriptor.link_protocols = link_versions.split(' ') + descriptor.circuit_protocols = circuit_versions.split(' ') + + +def _parse_or_address_line(descriptor, all_values): + or_addresses = [] + + for entry in all_values: + line = 'or-address %s' % entry + + if ':' not in entry: + raise ValueError('or-address line missing a colon: %s' % line) + + address, port = entry.rsplit(':', 1) + is_ipv6 = address.startswith('[') and address.endswith(']') + + if is_ipv6: + address = address[1:-1] # remove brackets + + if not ((not is_ipv6 and stem.util.connection.is_valid_ipv4_address(address)) or + (is_ipv6 and stem.util.connection.is_valid_ipv6_address(address))): + raise ValueError('or-address line has a malformed address: %s' % line) + + if not stem.util.connection.is_valid_port(port): + raise ValueError('or-address line has a malformed port: %s' % line) + + or_addresses.append((address, int(port), is_ipv6)) + + descriptor.or_addresses = or_addresses + + +def _parse_history_line(descriptor, value, is_read): + keyword = 'read-history' if is_read else 'write-history' + timestamp, interval, remainder = \ + stem.descriptor.extrainfo_descriptor._parse_timestamp_and_interval(keyword, value) + + try: + if remainder: + history_values = [int(entry) for entry in remainder.split(',')] + else: + history_values = [] + except ValueError: + raise ValueError('%s line has non-numeric values: %s %s' % (keyword, keyword, value)) + + if is_read: + descriptor.read_history_end = timestamp + descriptor.read_history_interval = interval + descriptor.read_history_values = history_values + else: + descriptor.write_history_end = timestamp + descriptor.write_history_interval = interval + descriptor.write_history_values = history_values + + class ServerDescriptor(Descriptor): """ Common parent for server descriptors. @@ -378,222 +575,50 @@ class ServerDescriptor(Descriptor): if block_contents: line += '\n%s' % block_contents - if keyword == 'router': - # "router" nickname address ORPort SocksPort DirPort - router_comp = value.split() - - if len(router_comp) < 5: - if not validate: - continue - - raise ValueError('Router line must have five values: %s' % line) - - if validate: - if not stem.util.tor_tools.is_valid_nickname(router_comp[0]): - raise ValueError("Router line entry isn't a valid nickname: %s" % router_comp[0]) - elif not stem.util.connection.is_valid_ipv4_address(router_comp[1]): - raise ValueError("Router line entry isn't a valid IPv4 address: %s" % router_comp[1]) - elif not stem.util.connection.is_valid_port(router_comp[2], allow_zero = True): - raise ValueError("Router line's ORPort is invalid: %s" % router_comp[2]) - elif not stem.util.connection.is_valid_port(router_comp[3], allow_zero = True): - raise ValueError("Router line's SocksPort is invalid: %s" % router_comp[3]) - elif not stem.util.connection.is_valid_port(router_comp[4], allow_zero = True): - raise ValueError("Router line's DirPort is invalid: %s" % router_comp[4]) - elif not (router_comp[2].isdigit() and router_comp[3].isdigit() and router_comp[4].isdigit()): - continue - - self.nickname = router_comp[0] - self.address = router_comp[1] - self.or_port = int(router_comp[2]) - self.socks_port = None if router_comp[3] == '0' else int(router_comp[3]) - self.dir_port = None if router_comp[4] == '0' else int(router_comp[4]) - elif keyword == 'bandwidth': - # "bandwidth" bandwidth-avg bandwidth-burst bandwidth-observed - bandwidth_comp = value.split() - - if len(bandwidth_comp) < 3: - if not validate: - continue - - raise ValueError('Bandwidth line must have three values: %s' % line) - elif not bandwidth_comp[0].isdigit(): - if not validate: - continue - - raise ValueError("Bandwidth line's average rate isn't numeric: %s" % bandwidth_comp[0]) - elif not bandwidth_comp[1].isdigit(): - if not validate: - continue - - raise ValueError("Bandwidth line's burst rate isn't numeric: %s" % bandwidth_comp[1]) - elif not bandwidth_comp[2].isdigit(): - if not validate: - continue - - raise ValueError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2]) - - self.average_bandwidth = int(bandwidth_comp[0]) - self.burst_bandwidth = int(bandwidth_comp[1]) - self.observed_bandwidth = int(bandwidth_comp[2]) - elif keyword == 'platform': - # "platform" string - - # The platform attribute was set earlier. This line can contain any - # arbitrary data, but tor seems to report its version followed by the - # os like the following... - # - # platform Tor 0.2.2.35 (git-73ff13ab3cc9570d) on Linux x86_64 - # - # There's no guarantee that we'll be able to pick these out the - # version, but might as well try to save our caller the effort. - - platform_match = re.match('^(?:node-)?Tor (\S*).* on (.*)$', value) - - if platform_match: - version_str, self.operating_system = platform_match.groups() - - try: - self.tor_version = stem.version._get_version(version_str) - except ValueError: - pass - elif keyword == 'published': - # "published" YYYY-MM-DD HH:MM:SS - - try: - self.published = stem.util.str_tools._parse_timestamp(value) - except ValueError: - if validate: - raise ValueError("Published line's time wasn't parsable: %s" % line) - elif keyword == 'fingerprint': - # This is forty hex digits split into space separated groups of four. - # Checking that we match this pattern. - - fingerprint = value.replace(' ', '') - - if validate: - for grouping in value.split(' '): - if len(grouping) != 4: - raise ValueError('Fingerprint line should have groupings of four hex digits: %s' % value) - - if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): - raise ValueError('Tor relay fingerprints consist of forty hex digits: %s' % value) - - self.fingerprint = fingerprint - elif keyword == 'hibernating': - # "hibernating" 0|1 (in practice only set if one) - - if validate and value not in ('0', '1'): - raise ValueError('Hibernating line had an invalid value, must be zero or one: %s' % value) - - self.hibernating = value == '1' - elif keyword == 'allow-single-hop-exits': - self.allow_single_hop_exits = True - elif keyword == 'caches-extra-info': - self.extra_info_cache = True - elif keyword == 'extra-info-digest': - # this is forty hex digits which just so happens to be the same a - # fingerprint - - if validate and not stem.util.tor_tools.is_valid_fingerprint(value): - raise ValueError('Extra-info digests should consist of forty hex digits: %s' % value) - - self.extra_info_digest = value - elif keyword == 'hidden-service-dir': - if value: - self.hidden_service_dir = value.split(' ') + try: + if keyword == 'router': + _parse_router_line(self, value) + elif keyword == 'bandwidth': + _parse_bandwidth_line(self, value) + elif keyword == 'platform': + _parse_platform_line(self, value) + elif keyword == 'published': + _parse_published_line(self, value) + elif keyword == 'fingerprint': + _parse_fingerprint_line(self, value) + elif keyword == 'hibernating': + _parse_hibernating_line(self, value) + elif keyword == 'allow-single-hop-exits': + self.allow_single_hop_exits = True + elif keyword == 'caches-extra-info': + self.extra_info_cache = True + elif keyword == 'extra-info-digest': + _parse_extrainfo_digest_line(self, value) + elif keyword == 'hidden-service-dir': + _parse_hidden_service_dir_line(self, value) + elif keyword == 'uptime': + _parse_uptime_line(self, value) + elif keyword == 'contact': + pass # parsed as a bytes field earlier + elif keyword == 'protocols': + _parse_protocols_line(self, value) + elif keyword == 'family': + self.family = set(value.split(' ')) + elif keyword == 'eventdns': + self.eventdns = value == '1' + elif keyword == 'ipv6-policy': + self.exit_policy_v6 = stem.exit_policy.MicroExitPolicy(value) + elif keyword == 'or-address': + _parse_or_address_line(self, [entry[0] for entry in values]) + elif keyword == 'read-history': + _parse_history_line(self, value, True) + elif keyword == 'write-history': + _parse_history_line(self, value, False) else: - self.hidden_service_dir = ['2'] - elif keyword == 'uptime': - # We need to be tolerant of negative uptimes to accommodate a past tor - # bug... - # - # Changes in version 0.1.2.7-alpha - 2007-02-06 - # - If our system clock jumps back in time, don't publish a negative - # uptime in the descriptor. Also, don't let the global rate limiting - # buckets go absurdly negative. - # - # After parsing all of the attributes we'll double check that negative - # uptimes only occurred prior to this fix. - - try: - self.uptime = int(value) - except ValueError: - if not validate: - continue - - raise ValueError('Uptime line must have an integer value: %s' % value) - elif keyword == 'contact': - pass # parsed as a bytes field earlier - elif keyword == 'protocols': - protocols_match = re.match('^Link (.*) Circuit (.*)$', value) - - if protocols_match: - link_versions, circuit_versions = protocols_match.groups() - self.link_protocols = link_versions.split(' ') - self.circuit_protocols = circuit_versions.split(' ') - elif validate: - raise ValueError('Protocols line did not match the expected pattern: %s' % line) - elif keyword == 'family': - self.family = set(value.split(' ')) - elif keyword == 'eventdns': - self.eventdns = value == '1' - elif keyword == 'ipv6-policy': - self.exit_policy_v6 = stem.exit_policy.MicroExitPolicy(value) - elif keyword == 'or-address': - or_address_entries = [address_entry for (address_entry, _, _) in values] - - for entry in or_address_entries: - line = '%s %s' % (keyword, entry) - - if ':' not in entry: - if not validate: - continue - else: - raise ValueError('or-address line missing a colon: %s' % line) - - address, port = entry.rsplit(':', 1) - is_ipv6 = address.startswith('[') and address.endswith(']') - - if is_ipv6: - address = address[1:-1] # remove brackets - - if not ((not is_ipv6 and stem.util.connection.is_valid_ipv4_address(address)) or - (is_ipv6 and stem.util.connection.is_valid_ipv6_address(address))): - if not validate: - continue - else: - raise ValueError('or-address line has a malformed address: %s' % line) - - if stem.util.connection.is_valid_port(port): - self.or_addresses.append((address, int(port), is_ipv6)) - elif validate: - raise ValueError('or-address line has a malformed port: %s' % line) - elif keyword in ('read-history', 'write-history'): - try: - timestamp, interval, remainder = \ - stem.descriptor.extrainfo_descriptor._parse_timestamp_and_interval(keyword, value) - - try: - if remainder: - history_values = [int(entry) for entry in remainder.split(',')] - else: - history_values = [] - except ValueError: - raise ValueError('%s line has non-numeric values: %s' % (keyword, line)) - - if keyword == 'read-history': - self.read_history_end = timestamp - self.read_history_interval = interval - self.read_history_values = history_values - else: - self.write_history_end = timestamp - self.write_history_interval = interval - self.write_history_values = history_values - except ValueError as exc: - if validate: - raise exc - else: - self._unrecognized_lines.append(line) + self._unrecognized_lines.append(line) + except ValueError as exc: + if validate: + raise exc # if we have a negative uptime and a tor version that shouldn't exhibit # this bug then fail validation diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index c57c476..61654a7 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -378,7 +378,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= """ desc_text = get_relay_server_descriptor({'router': 'saberrider2008ReallyLongNickname 71.35.133.197 9001 0 0'}, content = True) - self._expect_invalid_attr(desc_text, 'nickname', 'saberrider2008ReallyLongNickname') + self._expect_invalid_attr(desc_text, 'nickname') def test_nickname_invalid_char(self): """ @@ -386,7 +386,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= """ desc_text = get_relay_server_descriptor({'router': '$aberrider2008 71.35.133.197 9001 0 0'}, content = True) - self._expect_invalid_attr(desc_text, 'nickname', '$aberrider2008') + self._expect_invalid_attr(desc_text, 'nickname') def test_address_malformed(self): """ @@ -394,7 +394,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= """ desc_text = get_relay_server_descriptor({'router': 'caerSidi 371.35.133.197 9001 0 0'}, content = True) - self._expect_invalid_attr(desc_text, 'address', '371.35.133.197') + self._expect_invalid_attr(desc_text, 'address') def test_port_too_high(self): """ @@ -402,7 +402,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= """ desc_text = get_relay_server_descriptor({'router': 'caerSidi 71.35.133.197 900001 0 0'}, content = True) - self._expect_invalid_attr(desc_text, 'or_port', 900001) + self._expect_invalid_attr(desc_text, 'or_port') def test_port_malformed(self): """

1 0

[stem/master] Server descriptor lazy loading
by atagar＠torproject.org 25 Jan '15

25 Jan '15

commit 7868aeeb2d351b0ae49d21ba60a09a8b92b5aff1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 4 21:08:48 2015 -0800 Server descriptor lazy loading Proof of concept for lazy loading our server descriptors. Tests have a small number of failures at present because we're not lazy loading its subclasses. --- stem/descriptor/server_descriptor.py | 144 ++++++++++++++++++++++++---------- 1 file changed, 103 insertions(+), 41 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 671e96d..f936d13 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -443,46 +443,8 @@ class ServerDescriptor(Descriptor): raw_contents = stem.util.str_tools._to_unicode(raw_contents) - self.nickname = None - self.fingerprint = None - self.published = None - - self.address = None - self.or_port = None - self.socks_port = None - self.dir_port = None - - self.tor_version = None - self.operating_system = None - self.uptime = None - self.exit_policy = None - self.exit_policy_v6 = DEFAULT_IPV6_EXIT_POLICY - self.family = set() - - self.average_bandwidth = None - self.burst_bandwidth = None - self.observed_bandwidth = None - - self.link_protocols = None - self.circuit_protocols = None - self.hibernating = False - self.allow_single_hop_exits = False - self.extra_info_cache = False - self.extra_info_digest = None - self.hidden_service_dir = None - self.eventdns = None - self.or_addresses = [] - - self.read_history_end = None - self.read_history_interval = None - self.read_history_values = None - - self.write_history_end = None - self.write_history_interval = None - self.write_history_values = None - + self._lazy_loading = not validate self._unrecognized_lines = [] - self._annotation_lines = annotations if annotations else [] # A descriptor contains a series of 'keyword lines' which are simply a @@ -500,10 +462,11 @@ class ServerDescriptor(Descriptor): else: self.exit_policy = stem.exit_policy.ExitPolicy(*policy) - self._parse(entries, validate) - if validate: + self._parse(entries, validate) self._check_constraints(entries) + else: + self._entries = entries def digest(self): """ @@ -516,6 +479,11 @@ class ServerDescriptor(Descriptor): raise NotImplementedError('Unsupported Operation: this should be implemented by the ServerDescriptor subclass') def get_unrecognized_lines(self): + if self._lazy_loading: + # we need to go ahead and parse the whole document to figure this out + self._parse(self._entries, False) + self._lazy_loading = False + return list(self._unrecognized_lines) @lru_cache() @@ -566,6 +534,43 @@ class ServerDescriptor(Descriptor): :raises: **ValueError** if an error occurs in validation """ + self.nickname = None + self.fingerprint = None + self.published = None + + self.address = None + self.or_port = None + self.socks_port = None + self.dir_port = None + + self.tor_version = None + self.operating_system = None + self.uptime = None + self.exit_policy_v6 = DEFAULT_IPV6_EXIT_POLICY + self.family = set() + + self.average_bandwidth = None + self.burst_bandwidth = None + self.observed_bandwidth = None + + self.link_protocols = None + self.circuit_protocols = None + self.hibernating = False + self.allow_single_hop_exits = False + self.extra_info_cache = False + self.extra_info_digest = None + self.hidden_service_dir = None + self.eventdns = None + self.or_addresses = [] + + self.read_history_end = None + self.read_history_interval = None + self.read_history_values = None + + self.write_history_end = None + self.write_history_interval = None + self.write_history_values = None + for keyword, values in list(entries.items()): # most just work with the first (and only) value value, block_type, block_contents = values[0] @@ -671,6 +676,63 @@ class ServerDescriptor(Descriptor): def _last_keyword(self): return 'router-signature' + def __getattr__(self, name): + # If attribute isn't already present we might be lazy loading it... + + if self._lazy_loading: + try: + if name in ('nickname', 'address', 'or_port', 'socks_port', 'dir_port'): + _parse_router_line(self, self._entries['router'][0][0]) + elif name in ('average_bandwidth', 'burst_bandwidth', 'observed_bandwidth'): + _parse_bandwidth_line(self, self._entries['bandwidth'][0][0]) + elif name in ('operating_system', 'tor_version'): + _parse_platform_line(self, self._entries['platform'][0][0]) + elif name == 'published': + _parse_published_line(self, self._entries['published'][0][0]) + elif name == 'fingerprint': + _parse_fingerprint_line(self, self._entries['fingerprint'][0][0]) + elif name == 'hibernating': + _parse_hibernating_line(self, self._entries['hibernating'][0][0]) + elif name == 'allow_single_hop_exits': + self.allow_single_hop_exits = 'allow-single-hop-exits' in self._entries + elif name == 'extra_info_cache': + self.extra_info_cache = 'caches-extra-info' in self._entries + elif name == 'extra_info_digest': + _parse_extrainfo_digest_line(self, self._entries['extra-info-digest'][0][0]) + elif name == 'hidden_service_dir': + _parse_hidden_service_dir_line(self, self._entries['hidden-service-dir'][0][0]) + elif name == 'uptime': + _parse_uptime_line(self, self._entries['uptime'][0][0]) + elif name in ('link_protocols', 'circuit_protocols'): + _parse_protocols_line(self, self._entries['protocols'][0][0]) + elif name == 'family': + self.family = set(self._entries['family'][0][0].split(' ')) + elif name == 'eventdns': + self.eventdns = self._entries['eventdns'][0][0] == '1' + elif name == 'exit_policy_v6': + self.exit_policy_v6 = stem.exit_policy.MicroExitPolicy(self._entries['ipv6-policy'][0][0]) + elif name == 'or_addresses': + _parse_or_address_line(self, [entry[0] for entry in self._entries['or-address']]) + elif name in ('read_history_end', 'read_history_interval', 'read_history_values'): + _parse_history_line(self, self._entries['read-history'][0][0], True) + elif name in ('write_history_end', 'write_history_interval', 'write_history_values'): + _parse_history_line(self, self._entries['write-history'][0][0], False) + except (ValueError, KeyError): + if name == 'exit_policy_v6': + default = DEFAULT_IPV6_EXIT_POLICY + elif name == 'family': + default = set() + elif name in ('hibernating', 'allow_single_hop_exits', 'extra_info_cache'): + default = False + elif name == 'or_addresses': + default = [] + else: + default = None + + setattr(self, name, default) + + return super(ServerDescriptor, self).__getattribute__(name) + class RelayDescriptor(ServerDescriptor): """

1 0

[stem/master] Lazyloading server descriptor exit policy
by atagar＠torproject.org 25 Jan '15

25 Jan '15

commit eff8ecec5381f852317638b1ebc8f6893a955761 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 5 07:40:19 2015 -0800 Lazyloading server descriptor exit policy Oops, missed this attribute since it's not part of our parse method. Lazyloading this one's actually pretty impaceful, shaving off another 0.5s from my runtimes (a 13% improvement). --- stem/descriptor/server_descriptor.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index f936d13..e8d6e74 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -457,16 +457,17 @@ class ServerDescriptor(Descriptor): entries, policy = _get_descriptor_components(raw_contents, validate, ('accept', 'reject')) - if policy == [str_type('reject *:*')]: - self.exit_policy = REJECT_ALL_POLICY - else: - self.exit_policy = stem.exit_policy.ExitPolicy(*policy) - if validate: + if policy == [str_type('reject *:*')]: + self.exit_policy = REJECT_ALL_POLICY + else: + self.exit_policy = stem.exit_policy.ExitPolicy(*policy) + self._parse(entries, validate) self._check_constraints(entries) else: self._entries = entries + self._exit_policy_list = policy def digest(self): """ @@ -717,6 +718,13 @@ class ServerDescriptor(Descriptor): _parse_history_line(self, self._entries['read-history'][0][0], True) elif name in ('write_history_end', 'write_history_interval', 'write_history_values'): _parse_history_line(self, self._entries['write-history'][0][0], False) + elif name == 'exit_policy': + if self._exit_policy_list == [str_type('reject *:*')]: + self.exit_policy = REJECT_ALL_POLICY + else: + self.exit_policy = stem.exit_policy.ExitPolicy(*self._exit_policy_list) + + del self._exit_policy_list except (ValueError, KeyError): if name == 'exit_policy_v6': default = DEFAULT_IPV6_EXIT_POLICY

1 0

[stem/master] Flatten parse function mappings
by atagar＠torproject.org 25 Jan '15

25 Jan '15

commit b8af52b411713fd49e50cdcb9ef34851a350bf86 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 6 09:08:32 2015 -0800 Flatten parse function mappings No reason to have these in a for loop. Actually, if we make all functions uniform we'll be able to get rid of all the conditionals... --- stem/descriptor/server_descriptor.py | 35 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index e8d6e74..1ed8d4e 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -572,6 +572,19 @@ class ServerDescriptor(Descriptor): self.write_history_interval = None self.write_history_values = None + parse_functions = { + 'router': _parse_router_line, + 'bandwidth': _parse_bandwidth_line, + 'platform': _parse_platform_line, + 'published': _parse_published_line, + 'fingerprint': _parse_fingerprint_line, + 'hibernating': _parse_hibernating_line, + 'extra-info-digest': _parse_extrainfo_digest_line, + 'hidden-service-dir': _parse_hidden_service_dir_line, + 'uptime': _parse_uptime_line, + 'protocols': _parse_protocols_line, + } + for keyword, values in list(entries.items()): # most just work with the first (and only) value value, block_type, block_contents = values[0] @@ -582,32 +595,14 @@ class ServerDescriptor(Descriptor): line += '\n%s' % block_contents try: - if keyword == 'router': - _parse_router_line(self, value) - elif keyword == 'bandwidth': - _parse_bandwidth_line(self, value) - elif keyword == 'platform': - _parse_platform_line(self, value) - elif keyword == 'published': - _parse_published_line(self, value) - elif keyword == 'fingerprint': - _parse_fingerprint_line(self, value) - elif keyword == 'hibernating': - _parse_hibernating_line(self, value) + if keyword in parse_functions: + parse_functions[keyword](self, value) elif keyword == 'allow-single-hop-exits': self.allow_single_hop_exits = True elif keyword == 'caches-extra-info': self.extra_info_cache = True - elif keyword == 'extra-info-digest': - _parse_extrainfo_digest_line(self, value) - elif keyword == 'hidden-service-dir': - _parse_hidden_service_dir_line(self, value) - elif keyword == 'uptime': - _parse_uptime_line(self, value) elif keyword == 'contact': pass # parsed as a bytes field earlier - elif keyword == 'protocols': - _parse_protocols_line(self, value) elif keyword == 'family': self.family = set(value.split(' ')) elif keyword == 'eventdns':

1 0

[stem/master] Standardizing input argument for parsing helpers
by atagar＠torproject.org 25 Jan '15

25 Jan '15

commit b06ccd70e6fae2362fe36c2e61417b1e45556370 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 11 13:27:52 2015 -0800 Standardizing input argument for parsing helpers Providing all entries to the parse helpers. This way all the parsing helpers can take identical arguments, which will let us make this a little nicer. --- stem/descriptor/server_descriptor.py | 115 +++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 43 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 1ed8d4e..e79445e 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -167,9 +167,18 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True, **kwargs): break # done parsing descriptors -def _parse_router_line(descriptor, value): +def _value(line, entries): + return entries[line][0][0] + + +def _values(line, entries): + return [entry[0] for entry in entries[line]] + + +def _parse_router_line(descriptor, entries): # "router" nickname address ORPort SocksPort DirPort + value = _value('router', entries) router_comp = value.split() if len(router_comp) < 5: @@ -192,9 +201,10 @@ def _parse_router_line(descriptor, value): descriptor.dir_port = None if router_comp[4] == '0' else int(router_comp[4]) -def _parse_bandwidth_line(descriptor, value): +def _parse_bandwidth_line(descriptor, entries): # "bandwidth" bandwidth-avg bandwidth-burst bandwidth-observed + value = _value('bandwidth', entries) bandwidth_comp = value.split() if len(bandwidth_comp) < 3: @@ -211,7 +221,7 @@ def _parse_bandwidth_line(descriptor, value): descriptor.observed_bandwidth = int(bandwidth_comp[2]) -def _parse_platform_line(descriptor, value): +def _parse_platform_line(descriptor, entries): # "platform" string # The platform attribute was set earlier. This line can contain any @@ -223,6 +233,7 @@ def _parse_platform_line(descriptor, value): # There's no guarantee that we'll be able to pick these out the # version, but might as well try to save our caller the effort. + value = _value('platform', entries) platform_match = re.match('^(?:node-)?Tor (\S*).* on (.*)$', value) if platform_match: @@ -234,19 +245,22 @@ def _parse_platform_line(descriptor, value): pass -def _parse_published_line(descriptor, value): +def _parse_published_line(descriptor, entries): # "published" YYYY-MM-DD HH:MM:SS + value = _value('published', entries) + try: descriptor.published = stem.util.str_tools._parse_timestamp(value) except ValueError: raise ValueError("Published line's time wasn't parsable: published %s" % value) -def _parse_fingerprint_line(descriptor, value): +def _parse_fingerprint_line(descriptor, entries): # This is forty hex digits split into space separated groups of four. # Checking that we match this pattern. + value = _value('fingerprint', entries) fingerprint = value.replace(' ', '') for grouping in value.split(' '): @@ -259,33 +273,39 @@ def _parse_fingerprint_line(descriptor, value): descriptor.fingerprint = fingerprint -def _parse_hibernating_line(descriptor, value): +def _parse_hibernating_line(descriptor, entries): # "hibernating" 0|1 (in practice only set if one) + value = _value('hibernating', entries) + if value not in ('0', '1'): raise ValueError('Hibernating line had an invalid value, must be zero or one: %s' % value) descriptor.hibernating = value == '1' -def _parse_extrainfo_digest_line(descriptor, value): +def _parse_extrainfo_digest_line(descriptor, entries): # this is forty hex digits which just so happens to be the same a # fingerprint + value = _value('extra-info-digest', entries) + if not stem.util.tor_tools.is_valid_fingerprint(value): raise ValueError('Extra-info digests should consist of forty hex digits: %s' % value) descriptor.extra_info_digest = value -def _parse_hidden_service_dir_line(descriptor, value): +def _parse_hidden_service_dir_line(descriptor, entries): + value = _value('hidden-service-dir', entries) + if value: descriptor.hidden_service_dir = value.split(' ') else: descriptor.hidden_service_dir = ['2'] -def _parse_uptime_line(descriptor, value): +def _parse_uptime_line(descriptor, entries): # We need to be tolerant of negative uptimes to accommodate a past tor # bug... # @@ -297,13 +317,16 @@ def _parse_uptime_line(descriptor, value): # After parsing all of the attributes we'll double check that negative # uptimes only occurred prior to this fix. + value = _value('uptime', entries) + try: descriptor.uptime = int(value) except ValueError: raise ValueError('Uptime line must have an integer value: %s' % value) -def _parse_protocols_line(descriptor, value): +def _parse_protocols_line(descriptor, entries): + value = _value('protocols', entries) protocols_match = re.match('^Link (.*) Circuit (.*)$', value) if not protocols_match: @@ -314,7 +337,8 @@ def _parse_protocols_line(descriptor, value): descriptor.circuit_protocols = circuit_versions.split(' ') -def _parse_or_address_line(descriptor, all_values): +def _parse_or_address_line(descriptor, entries): + all_values = _values('or-address', entries) or_addresses = [] for entry in all_values: @@ -341,10 +365,25 @@ def _parse_or_address_line(descriptor, all_values): descriptor.or_addresses = or_addresses -def _parse_history_line(descriptor, value, is_read): - keyword = 'read-history' if is_read else 'write-history' - timestamp, interval, remainder = \ - stem.descriptor.extrainfo_descriptor._parse_timestamp_and_interval(keyword, value) +def _parse_read_history_line(descriptor, entries): + timestamp, interval, history_values = _parse_history_line(descriptor, entries, 'read-history') + + descriptor.read_history_end = timestamp + descriptor.read_history_interval = interval + descriptor.read_history_values = history_values + + +def _parse_write_history_line(descriptor, entries): + timestamp, interval, history_values = _parse_history_line(descriptor, entries, 'write-history') + + descriptor.write_history_end = timestamp + descriptor.write_history_interval = interval + descriptor.write_history_values = history_values + + +def _parse_history_line(descriptor, entries, keyword): + value = _value(keyword, entries) + timestamp, interval, remainder = stem.descriptor.extrainfo_descriptor._parse_timestamp_and_interval(keyword, value) try: if remainder: @@ -354,14 +393,7 @@ def _parse_history_line(descriptor, value, is_read): except ValueError: raise ValueError('%s line has non-numeric values: %s %s' % (keyword, keyword, value)) - if is_read: - descriptor.read_history_end = timestamp - descriptor.read_history_interval = interval - descriptor.read_history_values = history_values - else: - descriptor.write_history_end = timestamp - descriptor.write_history_interval = interval - descriptor.write_history_values = history_values + return timestamp, interval, history_values class ServerDescriptor(Descriptor): @@ -583,6 +615,9 @@ class ServerDescriptor(Descriptor): 'hidden-service-dir': _parse_hidden_service_dir_line, 'uptime': _parse_uptime_line, 'protocols': _parse_protocols_line, + 'or-address': _parse_or_address_line, + 'read-history': _parse_read_history_line, + 'write-history': _parse_write_history_line, } for keyword, values in list(entries.items()): @@ -596,7 +631,7 @@ class ServerDescriptor(Descriptor): try: if keyword in parse_functions: - parse_functions[keyword](self, value) + parse_functions[keyword](self, entries) elif keyword == 'allow-single-hop-exits': self.allow_single_hop_exits = True elif keyword == 'caches-extra-info': @@ -609,12 +644,6 @@ class ServerDescriptor(Descriptor): self.eventdns = value == '1' elif keyword == 'ipv6-policy': self.exit_policy_v6 = stem.exit_policy.MicroExitPolicy(value) - elif keyword == 'or-address': - _parse_or_address_line(self, [entry[0] for entry in values]) - elif keyword == 'read-history': - _parse_history_line(self, value, True) - elif keyword == 'write-history': - _parse_history_line(self, value, False) else: self._unrecognized_lines.append(line) except ValueError as exc: @@ -678,29 +707,29 @@ class ServerDescriptor(Descriptor): if self._lazy_loading: try: if name in ('nickname', 'address', 'or_port', 'socks_port', 'dir_port'): - _parse_router_line(self, self._entries['router'][0][0]) + _parse_router_line(self, self._entries) elif name in ('average_bandwidth', 'burst_bandwidth', 'observed_bandwidth'): - _parse_bandwidth_line(self, self._entries['bandwidth'][0][0]) + _parse_bandwidth_line(self, self._entries) elif name in ('operating_system', 'tor_version'): - _parse_platform_line(self, self._entries['platform'][0][0]) + _parse_platform_line(self, self._entries) elif name == 'published': - _parse_published_line(self, self._entries['published'][0][0]) + _parse_published_line(self, self._entries) elif name == 'fingerprint': - _parse_fingerprint_line(self, self._entries['fingerprint'][0][0]) + _parse_fingerprint_line(self, self._entries) elif name == 'hibernating': - _parse_hibernating_line(self, self._entries['hibernating'][0][0]) + _parse_hibernating_line(self, self._entries) elif name == 'allow_single_hop_exits': self.allow_single_hop_exits = 'allow-single-hop-exits' in self._entries elif name == 'extra_info_cache': self.extra_info_cache = 'caches-extra-info' in self._entries elif name == 'extra_info_digest': - _parse_extrainfo_digest_line(self, self._entries['extra-info-digest'][0][0]) + _parse_extrainfo_digest_line(self, self._entries) elif name == 'hidden_service_dir': - _parse_hidden_service_dir_line(self, self._entries['hidden-service-dir'][0][0]) + _parse_hidden_service_dir_line(self, self._entries) elif name == 'uptime': - _parse_uptime_line(self, self._entries['uptime'][0][0]) + _parse_uptime_line(self, self._entries) elif name in ('link_protocols', 'circuit_protocols'): - _parse_protocols_line(self, self._entries['protocols'][0][0]) + _parse_protocols_line(self, self._entries) elif name == 'family': self.family = set(self._entries['family'][0][0].split(' ')) elif name == 'eventdns': @@ -708,11 +737,11 @@ class ServerDescriptor(Descriptor): elif name == 'exit_policy_v6': self.exit_policy_v6 = stem.exit_policy.MicroExitPolicy(self._entries['ipv6-policy'][0][0]) elif name == 'or_addresses': - _parse_or_address_line(self, [entry[0] for entry in self._entries['or-address']]) + _parse_or_address_line(self, self._entries) elif name in ('read_history_end', 'read_history_interval', 'read_history_values'): - _parse_history_line(self, self._entries['read-history'][0][0], True) + _parse_read_history_line(self, self._entries) elif name in ('write_history_end', 'write_history_interval', 'write_history_values'): - _parse_history_line(self, self._entries['write-history'][0][0], False) + _parse_write_history_line(self, self._entries) elif name == 'exit_policy': if self._exit_policy_list == [str_type('reject *:*')]: self.exit_policy = REJECT_ALL_POLICY

1 0

[stem/master] Mapping for attributes to default and parser
by atagar＠torproject.org 25 Jan '15

25 Jan '15

commit ede02fffe4a6c459a8aa2d7472779f1450675cc9 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 11 14:07:52 2015 -0800 Mapping for attributes to default and parser Global mapping that lets us deduplicate defaults and the parser function. --- stem/descriptor/server_descriptor.py | 160 +++++++++++++++------------------- 1 file changed, 68 insertions(+), 92 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index e79445e..c2451bb 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -396,6 +396,63 @@ def _parse_history_line(descriptor, entries, keyword): return timestamp, interval, history_values +SERVER_DESCRIPTOR_ATTRIBUTES = { + 'nickname': (None, _parse_router_line), + 'fingerprint': (None, _parse_fingerprint_line), + 'published': (None, _parse_published_line), + + 'address': (None, _parse_router_line), + 'or_port': (None, _parse_router_line), + 'socks_port': (None, _parse_router_line), + 'dir_port': (None, _parse_router_line), + + 'tor_version': (None, _parse_platform_line), + 'operating_system': (None, _parse_platform_line), + 'uptime': (None, _parse_uptime_line), + 'exit_policy_v6': (DEFAULT_IPV6_EXIT_POLICY, None), + 'family': (set(), None), + + 'average_bandwidth': (None, _parse_bandwidth_line), + 'burst_bandwidth': (None, _parse_bandwidth_line), + 'observed_bandwidth': (None, _parse_bandwidth_line), + + 'link_protocols': (None, _parse_protocols_line), + 'circuit_protocols': (None, _parse_protocols_line), + 'hibernating': (False, _parse_hibernating_line), + 'allow_single_hop_exits': (False, None), + 'extra_info_cache': (False, None), + 'extra_info_digest': (None, _parse_extrainfo_digest_line), + 'hidden_service_dir': (None, _parse_hidden_service_dir_line), + 'eventdns': (None, None), + 'or_addresses': ([], _parse_or_address_line), + + 'read_history_end': (None, _parse_read_history_line), + 'read_history_interval': (None, _parse_read_history_line), + 'read_history_values': (None, _parse_read_history_line), + + 'write_history_end': (None, _parse_write_history_line), + 'write_history_interval': (None, _parse_write_history_line), + 'write_history_values': (None, _parse_write_history_line), +} + + +PARSER_FOR_LINE = { + 'router': _parse_router_line, + 'bandwidth': _parse_bandwidth_line, + 'platform': _parse_platform_line, + 'published': _parse_published_line, + 'fingerprint': _parse_fingerprint_line, + 'hibernating': _parse_hibernating_line, + 'extra-info-digest': _parse_extrainfo_digest_line, + 'hidden-service-dir': _parse_hidden_service_dir_line, + 'uptime': _parse_uptime_line, + 'protocols': _parse_protocols_line, + 'or-address': _parse_or_address_line, + 'read-history': _parse_read_history_line, + 'write-history': _parse_write_history_line, +} + + class ServerDescriptor(Descriptor): """ Common parent for server descriptors. @@ -567,58 +624,10 @@ class ServerDescriptor(Descriptor): :raises: **ValueError** if an error occurs in validation """ - self.nickname = None - self.fingerprint = None - self.published = None - - self.address = None - self.or_port = None - self.socks_port = None - self.dir_port = None - - self.tor_version = None - self.operating_system = None - self.uptime = None - self.exit_policy_v6 = DEFAULT_IPV6_EXIT_POLICY - self.family = set() - - self.average_bandwidth = None - self.burst_bandwidth = None - self.observed_bandwidth = None - - self.link_protocols = None - self.circuit_protocols = None - self.hibernating = False - self.allow_single_hop_exits = False - self.extra_info_cache = False - self.extra_info_digest = None - self.hidden_service_dir = None - self.eventdns = None - self.or_addresses = [] - - self.read_history_end = None - self.read_history_interval = None - self.read_history_values = None - - self.write_history_end = None - self.write_history_interval = None - self.write_history_values = None - - parse_functions = { - 'router': _parse_router_line, - 'bandwidth': _parse_bandwidth_line, - 'platform': _parse_platform_line, - 'published': _parse_published_line, - 'fingerprint': _parse_fingerprint_line, - 'hibernating': _parse_hibernating_line, - 'extra-info-digest': _parse_extrainfo_digest_line, - 'hidden-service-dir': _parse_hidden_service_dir_line, - 'uptime': _parse_uptime_line, - 'protocols': _parse_protocols_line, - 'or-address': _parse_or_address_line, - 'read-history': _parse_read_history_line, - 'write-history': _parse_write_history_line, - } + # set defaults + + for attr in SERVER_DESCRIPTOR_ATTRIBUTES: + setattr(self, attr, SERVER_DESCRIPTOR_ATTRIBUTES[attr][0]) for keyword, values in list(entries.items()): # most just work with the first (and only) value @@ -630,8 +639,8 @@ class ServerDescriptor(Descriptor): line += '\n%s' % block_contents try: - if keyword in parse_functions: - parse_functions[keyword](self, entries) + if keyword in PARSER_FOR_LINE: + PARSER_FOR_LINE[keyword](self, entries) elif keyword == 'allow-single-hop-exits': self.allow_single_hop_exits = True elif keyword == 'caches-extra-info': @@ -704,44 +713,22 @@ class ServerDescriptor(Descriptor): def __getattr__(self, name): # If attribute isn't already present we might be lazy loading it... - if self._lazy_loading: + if self._lazy_loading and name in SERVER_DESCRIPTOR_ATTRIBUTES: + default, parsing_function = SERVER_DESCRIPTOR_ATTRIBUTES[name] + try: - if name in ('nickname', 'address', 'or_port', 'socks_port', 'dir_port'): - _parse_router_line(self, self._entries) - elif name in ('average_bandwidth', 'burst_bandwidth', 'observed_bandwidth'): - _parse_bandwidth_line(self, self._entries) - elif name in ('operating_system', 'tor_version'): - _parse_platform_line(self, self._entries) - elif name == 'published': - _parse_published_line(self, self._entries) - elif name == 'fingerprint': - _parse_fingerprint_line(self, self._entries) - elif name == 'hibernating': - _parse_hibernating_line(self, self._entries) + if parsing_function: + parsing_function(self, self._entries) elif name == 'allow_single_hop_exits': self.allow_single_hop_exits = 'allow-single-hop-exits' in self._entries elif name == 'extra_info_cache': self.extra_info_cache = 'caches-extra-info' in self._entries - elif name == 'extra_info_digest': - _parse_extrainfo_digest_line(self, self._entries) - elif name == 'hidden_service_dir': - _parse_hidden_service_dir_line(self, self._entries) - elif name == 'uptime': - _parse_uptime_line(self, self._entries) - elif name in ('link_protocols', 'circuit_protocols'): - _parse_protocols_line(self, self._entries) elif name == 'family': self.family = set(self._entries['family'][0][0].split(' ')) elif name == 'eventdns': self.eventdns = self._entries['eventdns'][0][0] == '1' elif name == 'exit_policy_v6': self.exit_policy_v6 = stem.exit_policy.MicroExitPolicy(self._entries['ipv6-policy'][0][0]) - elif name == 'or_addresses': - _parse_or_address_line(self, self._entries) - elif name in ('read_history_end', 'read_history_interval', 'read_history_values'): - _parse_read_history_line(self, self._entries) - elif name in ('write_history_end', 'write_history_interval', 'write_history_values'): - _parse_write_history_line(self, self._entries) elif name == 'exit_policy': if self._exit_policy_list == [str_type('reject *:*')]: self.exit_policy = REJECT_ALL_POLICY @@ -750,17 +737,6 @@ class ServerDescriptor(Descriptor): del self._exit_policy_list except (ValueError, KeyError): - if name == 'exit_policy_v6': - default = DEFAULT_IPV6_EXIT_POLICY - elif name == 'family': - default = set() - elif name in ('hibernating', 'allow_single_hop_exits', 'extra_info_cache'): - default = False - elif name == 'or_addresses': - default = [] - else: - default = None - setattr(self, name, default) return super(ServerDescriptor, self).__getattribute__(name)

1 0

[translation/whisperback_completed] Update translations for whisperback_completed
by translation＠torproject.org 25 Jan '15

25 Jan '15

commit 3ca97446c64f356cd5f529df59bd60268b66e66f Author: Translation commit bot <translation(a)torproject.org> Date: Sun Jan 25 15:45:14 2015 +0000 Update translations for whisperback_completed --- de/de.po | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/de/de.po b/de/de.po index 23d59fb..affcdb7 100644 --- a/de/de.po +++ b/de/de.po @@ -15,7 +15,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2014-03-17 17:40+0100\n" -"PO-Revision-Date: 2015-01-25 14:41+0000\n" +"PO-Revision-Date: 2015-01-25 15:21+0000\n" "Last-Translator: Frank Anonima <franka1234(a)yandex.com>\n" "Language-Team: German (http://www.transifex.com/projects/p/torproject/language/de/)\n" "MIME-Version: 1.0\n" @@ -86,7 +86,7 @@ msgid "" "The bug report could not be sent, likely due to network problems. Please try to reconnect to the network and click send again.\n" "\n" "If it does not work, you will be offered to save the bug report." -msgstr "\n\nDie Fehlermeldung konnte nicht gesendet werden. Bitte erneut mit dem Netzwerk verbinden und auf Senden klicken.\n\nSollte das nicht funktionieren, wird Ihnen angeboten, die Fehlerbericht zu speichern." +msgstr "\n\nDer Fehlerbericht konnte nicht gesendet werden. Bitte erneut mit dem Netzwerk verbinden und auf Senden klicken.\n\nSollte das nicht funktionieren, wird Ihnen angeboten, den Fehlerbericht zu speichern." #: ../whisperBack/gui.py:274 msgid "Your message has been sent." @@ -110,7 +110,7 @@ msgid "" "As a work-around you can save the bug report as a file on a USB drive and try to send it to us at %s from your email account using another system. Note that your bug report will not be anonymous when doing so unless you take further steps yourself (e.g. using Tor with a throw-away email account).\n" "\n" "Do you want to save the bug report to a file?" -msgstr "Die Fehlermeldung konnte aufgrund von Netzwerkproblemen nicht gesendet werden.\n\nAls Umgehungslösung können Sie die Fehlermeldung auf einem USB-Stick speichern und versuchen, Sie uns per E-Mail, an %s, zu senden. Bedenken Sie bitte, dass dieser Fehlerbericht so nicht anonym sein wird, es sei denn, Sie unternehmen dazu weitere Schritte (z.b. die Verwendung einer Wegwerf-E-Mail-Adresse in Verbindung mit Tor).\n\nMöchten Sie diesen Fehlerbericht in eine Datei speichern?" +msgstr "Der Fehlerbericht konnte – vermutlich aufgrund von Netzwerkproblemen – nicht gesendet werden.\n\nAls Umgehungslösung können Sie den Fehlerbericht auf einem USB-Stick speichern und versuchen, ihn uns von einem anderen Rechner aus per E-Mail an %s zu senden. Bedenken Sie bitte, dass dieser Fehlerbericht dann nicht anonym sein wird, es sei denn, Sie unternehmen dazu weitere Schritte (z. B. die Verwendung einer Wegwerf-E-Mail-Adresse in Verbindung mit Tor).\n\nMöchten Sie diesen Fehlerbericht in einer Datei speichern?" #: ../whisperBack/gui.py:389 ../data/whisperback.ui.h:21 msgid "WhisperBack" @@ -122,7 +122,7 @@ msgstr "Ihre Rückmeldung in einer verschlüsselten Nachricht versenden." #: ../whisperBack/gui.py:393 msgid "Copyright © 2009-2012 Tails developpers (tails(a)boum.org)" -msgstr "Urheberrecht © 2009-2012 Tails-Entwickler (tails(a)boum.org)" +msgstr "Copyright © 2009–2012 Tails-Entwickler (tails(a)boum.org)" #: ../whisperBack/gui.py:394 msgid "Tails developers <tails(a)boum.org>" @@ -130,19 +130,19 @@ msgstr "Tails-Entwickler <tails(a)boum.org>" #: ../whisperBack/gui.py:395 msgid "translator-credits" -msgstr "Übersetzerdanksagung" +msgstr "Wir danken allen freiwilligen Übersetzern dieser Software!" #: ../whisperBack/gui.py:422 msgid "This doesn't seem to be a valid URL or OpenPGP key." -msgstr "Dieses scheint keine gültige Adresse oder OpenPGP-Schlüssel sein." +msgstr "Dies scheint keine gültige Adresse oder OpenPGP-Schlüssel sein." #: ../data/whisperback.ui.h:1 msgid "Copyright © 2009-2012 tails(a)boum.org" -msgstr "Urheberrecht © 2009-2012 tails(a)boum.org" +msgstr "Copyright © 2009–2012 tails(a)boum.org" #: ../data/whisperback.ui.h:3 msgid "https://tails.boum.org/" -msgstr "https://tails.boum.org/" +msgstr "https://tails.boum.org/index.de.html" #: ../data/whisperback.ui.h:4 msgid "" @@ -161,7 +161,7 @@ msgid "" "\n" "You should have received a copy of the GNU General Public License\n" "along with this program. If not, see <http://www.gnu.org/licenses/>.\n" -msgstr "WhisperBack - Rückmeldungen in einer verschlüsselten Nachricht versenden\nUrheberrecht (C) 2009-2012 Tails-Entwickler <tails(a)boum.org>\n\nDieses Programm ist freie Software; Sie können es weitergeben und/oder es \nverändern unter den Bedingungen der GNU General Public License wie \nveröffentlicht von der Free Software Foundation; entweder gemäß der Version 3 der Lizenz oder (nach Ihrer Wahl) einer späteren Version.\n\nDieses Programm wird in der Hoffnung vetrieben, dass es nützlich sein wird, \naber OHNE GARANTIE, sogar ohne die implizite Garantie der MARKTFÄHIGKEIT\noder DER EIGNUNG FÜR EINEN BESTIMMTEN ZWECK. Lesen Sie die GNU\nGeneral Public License für weitere Details.\n\nSie sollten eine Kopie der GNU General Public License mit diesem Programm erhalten haben. Wenn nicht, siehe: <http://www.gnu.org/licenses/>.\n" +msgstr "WhisperBack – Rückmeldungen in einer verschlüsselten Nachricht versenden\nCopyright © 2009–2012 Tails-Entwickler <tails(a)boum.org>\n\nDieses Programm ist freie Software: Sie können es weitergeben und/oder\nverändern, solange Sie sich an die Regeln der GNU General Public License\nhalten, so wie sie von der Free Software Foundation festgelegt wurden,\nentweder in Version 3 der Lizenz oder (nach Ihrem Ermessen) in jeder\nneueren Version.\n\nDieses Programm wurde mit dem Ziel veröffentlicht, dass Sie es nützlich\nfinden, jedoch OHNE JEGLICHE GARANTIE, sogar ohne eine implizite Garantie\nder VERKAUFBARKEIT oder der VERWENDBARKEIT FÜR EINEN SPEZIELLEN ZWECK.\nSchauen Sie für weitere Informationen bitte in der GNU General Public\nLicense nach.\n\nZusammen mit diesem Programm sollten Sie außerdem eine Kopie der GNU\nGeneral Public License erhalten haben. Wenn dem nicht so ist, siehe\n<http://www.gnu.org/licenses/>.\n" #: ../data/whisperback.ui.h:20 msgid ""

1 0