commit f307434824f74550b836927f3c02a4bac53c6c7c Author: Damian Johnson atagar@torproject.org Date: Tue Aug 20 15:23:58 2019 -0700
Rename stem.descriptor.hidden_service_descriptor module
Dropping the redundant '_descriptor' suffix from this module name. The old name still works as an alias. --- docs/api.rst | 2 +- docs/api/descriptor/hidden_service.rst | 5 + docs/api/descriptor/hidden_service_descriptor.rst | 5 - docs/change_log.rst | 7 +- docs/contents.rst | 2 +- docs/tutorials/mirror_mirror_on_the_wall.rst | 2 +- docs/tutorials/over_the_river.rst | 2 +- stem/control.py | 2 +- stem/descriptor/__init__.py | 12 +- stem/descriptor/hidden_service.py | 441 ++++++++++++++++++++ stem/descriptor/hidden_service_descriptor.py | 446 +-------------------- stem/response/events.py | 4 +- test/settings.cfg | 10 +- ...den_service_descriptor.py => hidden_service.py} | 4 +- 14 files changed, 474 insertions(+), 470 deletions(-)
diff --git a/docs/api.rst b/docs/api.rst index a8ba7e24..cbbf0dd0 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -35,7 +35,7 @@ remotely like Tor does. * `stem.descriptor.microdescriptor <api/descriptor/microdescriptor.html>`_ - Minimalistic counterpart for server descriptors. * `stem.descriptor.networkstatus <api/descriptor/networkstatus.html>`_ - Network status documents which make up the Tor consensus. * `stem.descriptor.router_status_entry <api/descriptor/router_status_entry.html>`_ - Relay entries within a network status document. - * `stem.descriptor.hidden_service_descriptor <api/descriptor/hidden_service_descriptor.html>`_ - Descriptors generated for hidden services. + * `stem.descriptor.hidden_service <api/descriptor/hidden_service.html>`_ - Descriptors generated for hidden services. * `stem.descriptor.bandwidth_file <api/descriptor/bandwidth_file.html>`_ - Bandwidth authority metrics. * `stem.descriptor.tordnsel <api/descriptor/tordnsel.html>`_ - `TorDNSEL https://www.torproject.org/projects/tordnsel.html.en`_ exit lists. * `stem.descriptor.certificate <api/descriptor/certificate.html>`_ - `Ed25519 certificates https://gitweb.torproject.org/torspec.git/tree/cert-spec.txt`_. diff --git a/docs/api/descriptor/hidden_service.rst b/docs/api/descriptor/hidden_service.rst new file mode 100644 index 00000000..21b9bd7b --- /dev/null +++ b/docs/api/descriptor/hidden_service.rst @@ -0,0 +1,5 @@ +Hidden Service Descriptor +========================= + +.. automodule:: stem.descriptor.hidden_service + diff --git a/docs/api/descriptor/hidden_service_descriptor.rst b/docs/api/descriptor/hidden_service_descriptor.rst deleted file mode 100644 index 145203e6..00000000 --- a/docs/api/descriptor/hidden_service_descriptor.rst +++ /dev/null @@ -1,5 +0,0 @@ -Hidden Service Descriptor -========================= - -.. automodule:: stem.descriptor.hidden_service_descriptor - diff --git a/docs/change_log.rst b/docs/change_log.rst index c244d7d2..d9c6bfa3 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -52,16 +52,16 @@ The following are only available within Stem's `git repository * Controller events could fail to be delivered in a timely fashion (:trac:`27173`) * Adjusted :func:`~stem.control.Controller.get_microdescriptors` fallback to also use '.new' cache files (:trac:`28508`) * ExitPolicies could raise TypeError when read concurrently (:trac:`29899`) - * **STALE_DESC** :data:`~stem.Flag` (:spec:`d14164d8`) + * **STALE_DESC** :data:`~stem.Flag` (:spec:`d14164d`) * **DORMANT** and **ACTIVE** :data:`~stem.Signal` (:spec:`4421149`) * **QUERY_RATE_LIMITED** :data:`~stem.HSDescReason` (:spec:`bd80679`) - * **EXTOR** and **HTTPTUNNEL** :data:`~stem.Listener` + * **EXTOR** and **HTTPTUNNEL** :data:`~stem.control.Listener`
* **Descriptors**
* Added the `stem.descriptor.collector <api/descriptor/collector.html>`_ module (:trac:`17979`) * `Bandwidth file support <api/descriptor/bandwidth_file.html>`_ (:trac:`29056`) - * `stem.descriptor.remote <api/descriptor/remote.html>`_ now raise :class:`stem.DownloadFailed` + * `stem.descriptor.remote <api/descriptor/remote.html>`_ methods now raise :class:`stem.DownloadFailed` * Check Ed25519 validity though the cryptography module rather than PyNaCl (:trac:`22022`) * Download compressed descriptors by default (:trac:`29186`) * Added :class:`~stem.descriptor.Compression` class @@ -78,6 +78,7 @@ The following are only available within Stem's `git repository * Replaced the **digest** attribute of :class:`~stem.descriptor.microdescriptor.Microdescriptor` with a method by the same name (:trac:`28398`) * Default the **version_flavor** attribute of :class:`~stem.descriptor.networkstatus.NetworkStatusDocumentV3` to 'ns' (:spec:`d97f8d9`) * DescriptorDownloader crashed if **use_mirrors** is set (:trac:`28393`) + * Renamed stem.descriptor.hidden_service_descriptor to stem.descriptor.hidden_service * Don't download from Serge, a bridge authority that frequently timeout * Updated dizum authority's address (:trac:`31406`)
diff --git a/docs/contents.rst b/docs/contents.rst index 267979e0..98e80a5f 100644 --- a/docs/contents.rst +++ b/docs/contents.rst @@ -50,7 +50,7 @@ Contents api/descriptor/microdescriptor api/descriptor/networkstatus api/descriptor/router_status_entry - api/descriptor/hidden_service_descriptor + api/descriptor/hidden_service api/descriptor/tordnsel
api/descriptor/export diff --git a/docs/tutorials/mirror_mirror_on_the_wall.rst b/docs/tutorials/mirror_mirror_on_the_wall.rst index 04cc86de..699625e4 100644 --- a/docs/tutorials/mirror_mirror_on_the_wall.rst +++ b/docs/tutorials/mirror_mirror_on_the_wall.rst @@ -34,7 +34,7 @@ Descriptor Type `Microdescriptor <../api/descriptor/microdescriptor.html>`_ Minimalistic document that just includes the information necessary for Tor clients to work. `Network Status Document <../api/descriptor/networkstatus.html>`_ Though Tor relays are decentralized, the directories that track the overall network are not. These central points are called **directory authorities**, and every hour they publish a document called a **consensus** (aka, network status document). The consensus in turn is made up of **router status entries**. `Router Status Entry <../api/descriptor/router_status_entry.html>`_ Relay information provided by the directory authorities including flags, heuristics used for relay selection, etc. -`Hidden Service Descriptor <../api/descriptor/hidden_service_descriptor.html>`_ Information pertaining to a `Hidden Service https://www.torproject.org/docs/hidden-services.html.en`_. These can only be `queried through the tor process <over_the_river.html#hidden-service-descriptors>`_. +`Hidden Service Descriptor <../api/descriptor/hidden_service.html>`_ Information pertaining to a `Hidden Service https://www.torproject.org/docs/hidden-services.html.en`_. These can only be `queried through the tor process <over_the_river.html#hidden-service-descriptors>`_. ================================================================================ ===========
.. _where-do-descriptors-come-from: diff --git a/docs/tutorials/over_the_river.rst b/docs/tutorials/over_the_river.rst index dac78827..ff8c7feb 100644 --- a/docs/tutorials/over_the_river.rst +++ b/docs/tutorials/over_the_river.rst @@ -171,7 +171,7 @@ its :func:`~stem.control.Controller.get_hidden_service_descriptor` method... A hidden service's introduction points are a base64 encoded field that's possibly encrypted. These can be decoded (and decrypted if necessary) with the descriptor's -:func:`~stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor.introduction_points` +:func:`~stem.descriptor.hidden_service.HiddenServiceDescriptor.introduction_points` method.
.. literalinclude:: /_static/example/introduction_points.py diff --git a/stem/control.py b/stem/control.py index d8423ffa..81cb1682 100644 --- a/stem/control.py +++ b/stem/control.py @@ -2136,7 +2136,7 @@ class Controller(BaseController): :param list servers: requrest the descriptor from these specific servers :param float timeout: seconds to wait when **await_result** is **True**
- :returns: :class:`~stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor` + :returns: :class:`~stem.descriptor.hidden_service.HiddenServiceDescriptor` for the given service if **await_result** is **True**, or **None** otherwise
:raises: diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index c099ca86..fd96c042 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -118,7 +118,7 @@ __all__ = [ 'collector', 'export', 'extrainfo_descriptor', - 'hidden_service_descriptor', + 'hidden_service', 'microdescriptor', 'networkstatus', 'reader', @@ -329,7 +329,7 @@ def parse_file(descriptor_file, descriptor_type = None, validate = False, docume torperf 1.0 **unsupported** bridge-pool-assignment 1.0 **unsupported** tordnsel 1.0 :class:`~stem.descriptor.tordnsel.TorDNSEL` - hidden-service-descriptor 1.0 :class:`~stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor` + hidden-service-descriptor 1.0 :class:`~stem.descriptor.hidden_service.HiddenServiceDescriptor` ========================================= =====
If you're using **python 3** then beware that the open() function defaults to @@ -536,10 +536,10 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto
for desc in stem.descriptor.tordnsel._parse_file(descriptor_file, validate = validate, **kwargs): yield desc - elif descriptor_type == stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor.TYPE_ANNOTATION_NAME and major_version == 1: - document_type = stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor + elif descriptor_type == stem.descriptor.hidden_service.HiddenServiceDescriptor.TYPE_ANNOTATION_NAME and major_version == 1: + document_type = stem.descriptor.hidden_service.HiddenServiceDescriptor
- for desc in stem.descriptor.hidden_service_descriptor._parse_file(descriptor_file, validate = validate, **kwargs): + for desc in stem.descriptor.hidden_service._parse_file(descriptor_file, validate = validate, **kwargs): yield desc elif descriptor_type == stem.descriptor.bandwidth_file.BandwidthFile.TYPE_ANNOTATION_NAME and major_version == 1: document_type = stem.descriptor.bandwidth_file.BandwidthFile @@ -1521,7 +1521,7 @@ def _descriptor_components(raw_contents, validate, extra_keywords = (), non_asci
import stem.descriptor.bandwidth_file import stem.descriptor.extrainfo_descriptor -import stem.descriptor.hidden_service_descriptor +import stem.descriptor.hidden_service import stem.descriptor.microdescriptor import stem.descriptor.networkstatus import stem.descriptor.server_descriptor diff --git a/stem/descriptor/hidden_service.py b/stem/descriptor/hidden_service.py new file mode 100644 index 00000000..665d8664 --- /dev/null +++ b/stem/descriptor/hidden_service.py @@ -0,0 +1,441 @@ +# Copyright 2015-2019, Damian Johnson and The Tor Project +# See LICENSE for licensing information + +""" +Parsing for Tor hidden service descriptors as described in Tor's `rend-spec +https://gitweb.torproject.org/torspec.git/tree/rend-spec.txt`_. + +Unlike other descriptor types these describe a hidden service rather than a +relay. They're created by the service, and can only be fetched via relays with +the HSDir flag. + +These are only available through the Controller's +:func:`~stem.control.get_hidden_service_descriptor` method. + +**Module Overview:** + +:: + + HiddenServiceDescriptor - Tor hidden service descriptor. + +.. versionadded:: 1.4.0 +""" + +import base64 +import binascii +import collections +import hashlib +import io + +import stem.prereq +import stem.util.connection +import stem.util.str_tools + +from stem.descriptor import ( + PGP_BLOCK_END, + Descriptor, + _descriptor_content, + _descriptor_components, + _read_until_keywords, + _bytes_for_block, + _value, + _parse_simple_line, + _parse_timestamp_line, + _parse_key_block, + _random_date, + _random_crypto_blob, +) + +if stem.prereq._is_lru_cache_available(): + from functools import lru_cache +else: + from stem.util.lru_cache import lru_cache + +REQUIRED_FIELDS = ( + 'rendezvous-service-descriptor', + 'version', + 'permanent-key', + 'secret-id-part', + 'publication-time', + 'protocol-versions', + 'signature', +) + +INTRODUCTION_POINTS_ATTR = { + 'identifier': None, + 'address': None, + 'port': None, + 'onion_key': None, + 'service_key': None, + 'intro_authentication': [], +} + +# introduction-point fields that can only appear once + +SINGLE_INTRODUCTION_POINT_FIELDS = [ + 'introduction-point', + 'ip-address', + 'onion-port', + 'onion-key', + 'service-key', +] + +BASIC_AUTH = 1 +STEALTH_AUTH = 2 + + +class IntroductionPoints(collections.namedtuple('IntroductionPoints', INTRODUCTION_POINTS_ATTR.keys())): + """ + :var str identifier: hash of this introduction point's identity key + :var str address: address of this introduction point + :var int port: port where this introduction point is listening + :var str onion_key: public key for communicating with this introduction point + :var str service_key: public key for communicating with this hidden service + :var list intro_authentication: tuples of the form (auth_type, auth_data) for + establishing a connection + """ + + +class DecryptionFailure(Exception): + """ + Failure to decrypt the hidden service descriptor's introduction-points. + """ + + +def _parse_file(descriptor_file, validate = False, **kwargs): + """ + Iterates over the hidden service descriptors in a file. + + :param file descriptor_file: file with descriptor content + :param bool validate: checks the validity of the descriptor's content if + **True**, skips these checks otherwise + :param dict kwargs: additional arguments for the descriptor constructor + + :returns: iterator for :class:`~stem.descriptor.hidden_service.HiddenServiceDescriptor` + instances in the file + + :raises: + * **ValueError** if the contents is malformed and validate is **True** + * **IOError** if the file can't be read + """ + + while True: + descriptor_content = _read_until_keywords('signature', descriptor_file) + + # we've reached the 'signature', now include the pgp style block + block_end_prefix = PGP_BLOCK_END.split(' ', 1)[0] + descriptor_content += _read_until_keywords(block_end_prefix, descriptor_file, True) + + if descriptor_content: + if descriptor_content[0].startswith(b'@type'): + descriptor_content = descriptor_content[1:] + + yield HiddenServiceDescriptor(bytes.join(b'', descriptor_content), validate, **kwargs) + else: + break # done parsing file + + +def _parse_version_line(descriptor, entries): + value = _value('version', entries) + + if value.isdigit(): + descriptor.version = int(value) + else: + raise ValueError('version line must have a positive integer value: %s' % value) + + +def _parse_protocol_versions_line(descriptor, entries): + value = _value('protocol-versions', entries) + + try: + versions = [int(entry) for entry in value.split(',')] + except ValueError: + raise ValueError('protocol-versions line has non-numeric versoins: protocol-versions %s' % value) + + for v in versions: + if v <= 0: + raise ValueError('protocol-versions must be positive integers: %s' % value) + + descriptor.protocol_versions = versions + + +def _parse_introduction_points_line(descriptor, entries): + _, block_type, block_contents = entries['introduction-points'][0] + + if not block_contents or block_type != 'MESSAGE': + raise ValueError("'introduction-points' should be followed by a MESSAGE block, but was a %s" % block_type) + + descriptor.introduction_points_encoded = block_contents + descriptor.introduction_points_auth = [] # field was never implemented in tor (#15190) + + try: + descriptor.introduction_points_content = _bytes_for_block(block_contents) + except TypeError: + raise ValueError("'introduction-points' isn't base64 encoded content:\n%s" % block_contents) + + +_parse_rendezvous_service_descriptor_line = _parse_simple_line('rendezvous-service-descriptor', 'descriptor_id') +_parse_permanent_key_line = _parse_key_block('permanent-key', 'permanent_key', 'RSA PUBLIC KEY') +_parse_secret_id_part_line = _parse_simple_line('secret-id-part', 'secret_id_part') +_parse_publication_time_line = _parse_timestamp_line('publication-time', 'published') +_parse_signature_line = _parse_key_block('signature', 'signature', 'SIGNATURE') + + +class HiddenServiceDescriptor(Descriptor): + """ + Hidden service descriptor. + + :var str descriptor_id: **\*** identifier for this descriptor, this is a base32 hash of several fields + :var int version: **\*** hidden service descriptor version + :var str permanent_key: **\*** long term key of the hidden service + :var str secret_id_part: **\*** hash of the time period, cookie, and replica + values so our descriptor_id can be validated + :var datetime published: **\*** time in UTC when this descriptor was made + :var list protocol_versions: **\*** list of **int** versions that are supported when establishing a connection + :var str introduction_points_encoded: raw introduction points blob + :var list introduction_points_auth: **\*** tuples of the form + (auth_method, auth_data) for our introduction_points_content + (**deprecated**, always **[]**) + :var bytes introduction_points_content: decoded introduction-points content + without authentication data, if using cookie authentication this is + encrypted + :var str signature: signature of the descriptor content + + **\*** attribute is either required when we're parsed with validation or has + a default value, others are left as **None** if undefined + + .. versionchanged:: 1.6.0 + Moved from the deprecated `pycrypto + https://www.dlitz.net/software/pycrypto/`_ module to `cryptography + https://pypi.org/project/cryptography/`_ for validating signatures. + + .. versionchanged:: 1.6.0 + Added the **skip_crypto_validation** constructor argument. + """ + + TYPE_ANNOTATION_NAME = 'hidden-service-descriptor' + + ATTRIBUTES = { + 'descriptor_id': (None, _parse_rendezvous_service_descriptor_line), + 'version': (None, _parse_version_line), + 'permanent_key': (None, _parse_permanent_key_line), + 'secret_id_part': (None, _parse_secret_id_part_line), + 'published': (None, _parse_publication_time_line), + 'protocol_versions': ([], _parse_protocol_versions_line), + 'introduction_points_encoded': (None, _parse_introduction_points_line), + 'introduction_points_auth': ([], _parse_introduction_points_line), + 'introduction_points_content': (None, _parse_introduction_points_line), + 'signature': (None, _parse_signature_line), + } + + PARSER_FOR_LINE = { + 'rendezvous-service-descriptor': _parse_rendezvous_service_descriptor_line, + 'version': _parse_version_line, + 'permanent-key': _parse_permanent_key_line, + 'secret-id-part': _parse_secret_id_part_line, + 'publication-time': _parse_publication_time_line, + 'protocol-versions': _parse_protocol_versions_line, + 'introduction-points': _parse_introduction_points_line, + 'signature': _parse_signature_line, + } + + @classmethod + def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + + return _descriptor_content(attr, exclude, ( + ('rendezvous-service-descriptor', 'y3olqqblqw2gbh6phimfuiroechjjafa'), + ('version', '2'), + ('permanent-key', _random_crypto_blob('RSA PUBLIC KEY')), + ('secret-id-part', 'e24kgecavwsznj7gpbktqsiwgvngsf4e'), + ('publication-time', _random_date()), + ('protocol-versions', '2,3'), + ('introduction-points', '\n-----BEGIN MESSAGE-----\n-----END MESSAGE-----'), + ), ( + ('signature', _random_crypto_blob('SIGNATURE')), + )) + + @classmethod + def create(cls, attr = None, exclude = (), validate = True, sign = False): + return cls(cls.content(attr, exclude, sign), validate = validate, skip_crypto_validation = not sign) + + def __init__(self, raw_contents, validate = False, skip_crypto_validation = False): + super(HiddenServiceDescriptor, self).__init__(raw_contents, lazy_load = not validate) + entries = _descriptor_components(raw_contents, validate, non_ascii_fields = ('introduction-points')) + + if validate: + for keyword in REQUIRED_FIELDS: + if keyword not in entries: + raise ValueError("Hidden service descriptor must have a '%s' entry" % keyword) + elif keyword in entries and len(entries[keyword]) > 1: + raise ValueError("The '%s' entry can only appear once in a hidden service descriptor" % keyword) + + if 'rendezvous-service-descriptor' != list(entries.keys())[0]: + raise ValueError("Hidden service descriptor must start with a 'rendezvous-service-descriptor' entry") + elif 'signature' != list(entries.keys())[-1]: + raise ValueError("Hidden service descriptor must end with a 'signature' entry") + + self._parse(entries, validate) + + if not skip_crypto_validation and stem.prereq.is_crypto_available(): + signed_digest = self._digest_for_signature(self.permanent_key, self.signature) + digest_content = self._content_range('rendezvous-service-descriptor ', '\nsignature\n') + content_digest = hashlib.sha1(digest_content).hexdigest().upper() + + if signed_digest != content_digest: + raise ValueError('Decrypted digest does not match local digest (calculated: %s, local: %s)' % (signed_digest, content_digest)) + else: + self._entries = entries + + @lru_cache() + def introduction_points(self, authentication_cookie = None): + """ + Provided this service's introduction points. + + :returns: **list** of :class:`~stem.descriptor.hidden_service.IntroductionPoints` + + :raises: + * **ValueError** if the our introduction-points is malformed + * **DecryptionFailure** if unable to decrypt this field + """ + + content = self.introduction_points_content + + if not content: + return [] + elif authentication_cookie: + if not stem.prereq.is_crypto_available(): + raise DecryptionFailure('Decrypting introduction-points requires the cryptography module') + + try: + missing_padding = len(authentication_cookie) % 4 + authentication_cookie = base64.b64decode(stem.util.str_tools._to_bytes(authentication_cookie) + b'=' * missing_padding) + except TypeError as exc: + raise DecryptionFailure('authentication_cookie must be a base64 encoded string (%s)' % exc) + + authentication_type = int(binascii.hexlify(content[0:1]), 16) + + if authentication_type == BASIC_AUTH: + content = HiddenServiceDescriptor._decrypt_basic_auth(content, authentication_cookie) + elif authentication_type == STEALTH_AUTH: + content = HiddenServiceDescriptor._decrypt_stealth_auth(content, authentication_cookie) + else: + raise DecryptionFailure("Unrecognized authentication type '%s', currently we only support basic auth (%s) and stealth auth (%s)" % (authentication_type, BASIC_AUTH, STEALTH_AUTH)) + + if not content.startswith(b'introduction-point '): + raise DecryptionFailure('Unable to decrypt the introduction-points, maybe this is the wrong key?') + elif not content.startswith(b'introduction-point '): + raise DecryptionFailure('introduction-points content is encrypted, you need to provide its authentication_cookie') + + return HiddenServiceDescriptor._parse_introduction_points(content) + + @staticmethod + def _decrypt_basic_auth(content, authentication_cookie): + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend + + try: + client_blocks = int(binascii.hexlify(content[1:2]), 16) + except ValueError: + raise DecryptionFailure("When using basic auth the content should start with a number of blocks but wasn't a hex digit: %s" % binascii.hexlify(content[1:2])) + + # parse the client id and encrypted session keys + + client_entries_length = client_blocks * 16 * 20 + client_entries = content[2:2 + client_entries_length] + client_keys = [(client_entries[i:i + 4], client_entries[i + 4:i + 20]) for i in range(0, client_entries_length, 4 + 16)] + + iv = content[2 + client_entries_length:2 + client_entries_length + 16] + encrypted = content[2 + client_entries_length + 16:] + + client_id = hashlib.sha1(authentication_cookie + iv).digest()[:4] + + for entry_id, encrypted_session_key in client_keys: + if entry_id != client_id: + continue # not the session key for this client + + # try decrypting the session key + + cipher = Cipher(algorithms.AES(authentication_cookie), modes.CTR(b'\x00' * len(iv)), default_backend()) + decryptor = cipher.decryptor() + session_key = decryptor.update(encrypted_session_key) + decryptor.finalize() + + # attempt to decrypt the intro points with the session key + + cipher = Cipher(algorithms.AES(session_key), modes.CTR(iv), default_backend()) + decryptor = cipher.decryptor() + decrypted = decryptor.update(encrypted) + decryptor.finalize() + + # check if the decryption looks correct + + if decrypted.startswith(b'introduction-point '): + return decrypted + + return content # nope, unable to decrypt the content + + @staticmethod + def _decrypt_stealth_auth(content, authentication_cookie): + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend + + # byte 1 = authentication type, 2-17 = input vector, 18 on = encrypted content + iv, encrypted = content[1:17], content[17:] + cipher = Cipher(algorithms.AES(authentication_cookie), modes.CTR(iv), default_backend()) + decryptor = cipher.decryptor() + + return decryptor.update(encrypted) + decryptor.finalize() + + @staticmethod + def _parse_introduction_points(content): + """ + Provides the parsed list of IntroductionPoints for the unencrypted content. + """ + + introduction_points = [] + content_io = io.BytesIO(content) + + while True: + content = b''.join(_read_until_keywords('introduction-point', content_io, ignore_first = True)) + + if not content: + break # reached the end + + attr = dict(INTRODUCTION_POINTS_ATTR) + entries = _descriptor_components(content, False) + + for keyword, values in list(entries.items()): + value, block_type, block_contents = values[0] + + if keyword in SINGLE_INTRODUCTION_POINT_FIELDS and len(values) > 1: + raise ValueError("'%s' can only appear once in an introduction-point block, but appeared %i times" % (keyword, len(values))) + + if keyword == 'introduction-point': + attr['identifier'] = value + elif keyword == 'ip-address': + if not stem.util.connection.is_valid_ipv4_address(value): + raise ValueError("'%s' is an invalid IPv4 address" % value) + + attr['address'] = value + elif keyword == 'onion-port': + if not stem.util.connection.is_valid_port(value): + raise ValueError("'%s' is an invalid port" % value) + + attr['port'] = int(value) + elif keyword == 'onion-key': + attr['onion_key'] = block_contents + elif keyword == 'service-key': + attr['service_key'] = block_contents + elif keyword == 'intro-authentication': + auth_entries = [] + + for auth_value, _, _ in values: + if ' ' not in auth_value: + raise ValueError("We expected 'intro-authentication [auth_type] [auth_data]', but had '%s'" % auth_value) + + auth_type, auth_data = auth_value.split(' ')[:2] + auth_entries.append((auth_type, auth_data)) + + introduction_points.append(IntroductionPoints(**attr)) + + return introduction_points diff --git a/stem/descriptor/hidden_service_descriptor.py b/stem/descriptor/hidden_service_descriptor.py index 99d6414e..d77d88aa 100644 --- a/stem/descriptor/hidden_service_descriptor.py +++ b/stem/descriptor/hidden_service_descriptor.py @@ -1,444 +1,4 @@ -# Copyright 2015-2019, Damian Johnson and The Tor Project -# See LICENSE for licensing information +# TODO: This module (hidden_service_descriptor) is a temporary alias for +# hidden_service. This alias will be removed in Stem 2.x.
-""" -Parsing for Tor hidden service descriptors as described in Tor's `rend-spec -https://gitweb.torproject.org/torspec.git/tree/rend-spec.txt`_. - -Unlike other descriptor types these describe a hidden service rather than a -relay. They're created by the service, and can only be fetched via relays with -the HSDir flag. - -These are only available through the Controller's -:func:`~stem.control.get_hidden_service_descriptor` method. - -**Module Overview:** - -:: - - HiddenServiceDescriptor - Tor hidden service descriptor. - -.. versionadded:: 1.4.0 -""" - -# TODO: In stem 2.x rename this module to 'hidden_service' (ie, drop the -# redundant '_descriptor' suffix). - -import base64 -import binascii -import collections -import hashlib -import io - -import stem.prereq -import stem.util.connection -import stem.util.str_tools - -from stem.descriptor import ( - PGP_BLOCK_END, - Descriptor, - _descriptor_content, - _descriptor_components, - _read_until_keywords, - _bytes_for_block, - _value, - _parse_simple_line, - _parse_timestamp_line, - _parse_key_block, - _random_date, - _random_crypto_blob, -) - -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - -REQUIRED_FIELDS = ( - 'rendezvous-service-descriptor', - 'version', - 'permanent-key', - 'secret-id-part', - 'publication-time', - 'protocol-versions', - 'signature', -) - -INTRODUCTION_POINTS_ATTR = { - 'identifier': None, - 'address': None, - 'port': None, - 'onion_key': None, - 'service_key': None, - 'intro_authentication': [], -} - -# introduction-point fields that can only appear once - -SINGLE_INTRODUCTION_POINT_FIELDS = [ - 'introduction-point', - 'ip-address', - 'onion-port', - 'onion-key', - 'service-key', -] - -BASIC_AUTH = 1 -STEALTH_AUTH = 2 - - -class IntroductionPoints(collections.namedtuple('IntroductionPoints', INTRODUCTION_POINTS_ATTR.keys())): - """ - :var str identifier: hash of this introduction point's identity key - :var str address: address of this introduction point - :var int port: port where this introduction point is listening - :var str onion_key: public key for communicating with this introduction point - :var str service_key: public key for communicating with this hidden service - :var list intro_authentication: tuples of the form (auth_type, auth_data) for - establishing a connection - """ - - -class DecryptionFailure(Exception): - """ - Failure to decrypt the hidden service descriptor's introduction-points. - """ - - -def _parse_file(descriptor_file, validate = False, **kwargs): - """ - Iterates over the hidden service descriptors in a file. - - :param file descriptor_file: file with descriptor content - :param bool validate: checks the validity of the descriptor's content if - **True**, skips these checks otherwise - :param dict kwargs: additional arguments for the descriptor constructor - - :returns: iterator for :class:`~stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor` - instances in the file - - :raises: - * **ValueError** if the contents is malformed and validate is **True** - * **IOError** if the file can't be read - """ - - while True: - descriptor_content = _read_until_keywords('signature', descriptor_file) - - # we've reached the 'signature', now include the pgp style block - block_end_prefix = PGP_BLOCK_END.split(' ', 1)[0] - descriptor_content += _read_until_keywords(block_end_prefix, descriptor_file, True) - - if descriptor_content: - if descriptor_content[0].startswith(b'@type'): - descriptor_content = descriptor_content[1:] - - yield HiddenServiceDescriptor(bytes.join(b'', descriptor_content), validate, **kwargs) - else: - break # done parsing file - - -def _parse_version_line(descriptor, entries): - value = _value('version', entries) - - if value.isdigit(): - descriptor.version = int(value) - else: - raise ValueError('version line must have a positive integer value: %s' % value) - - -def _parse_protocol_versions_line(descriptor, entries): - value = _value('protocol-versions', entries) - - try: - versions = [int(entry) for entry in value.split(',')] - except ValueError: - raise ValueError('protocol-versions line has non-numeric versoins: protocol-versions %s' % value) - - for v in versions: - if v <= 0: - raise ValueError('protocol-versions must be positive integers: %s' % value) - - descriptor.protocol_versions = versions - - -def _parse_introduction_points_line(descriptor, entries): - _, block_type, block_contents = entries['introduction-points'][0] - - if not block_contents or block_type != 'MESSAGE': - raise ValueError("'introduction-points' should be followed by a MESSAGE block, but was a %s" % block_type) - - descriptor.introduction_points_encoded = block_contents - descriptor.introduction_points_auth = [] # field was never implemented in tor (#15190) - - try: - descriptor.introduction_points_content = _bytes_for_block(block_contents) - except TypeError: - raise ValueError("'introduction-points' isn't base64 encoded content:\n%s" % block_contents) - - -_parse_rendezvous_service_descriptor_line = _parse_simple_line('rendezvous-service-descriptor', 'descriptor_id') -_parse_permanent_key_line = _parse_key_block('permanent-key', 'permanent_key', 'RSA PUBLIC KEY') -_parse_secret_id_part_line = _parse_simple_line('secret-id-part', 'secret_id_part') -_parse_publication_time_line = _parse_timestamp_line('publication-time', 'published') -_parse_signature_line = _parse_key_block('signature', 'signature', 'SIGNATURE') - - -class HiddenServiceDescriptor(Descriptor): - """ - Hidden service descriptor. - - :var str descriptor_id: **\*** identifier for this descriptor, this is a base32 hash of several fields - :var int version: **\*** hidden service descriptor version - :var str permanent_key: **\*** long term key of the hidden service - :var str secret_id_part: **\*** hash of the time period, cookie, and replica - values so our descriptor_id can be validated - :var datetime published: **\*** time in UTC when this descriptor was made - :var list protocol_versions: **\*** list of **int** versions that are supported when establishing a connection - :var str introduction_points_encoded: raw introduction points blob - :var list introduction_points_auth: **\*** tuples of the form - (auth_method, auth_data) for our introduction_points_content - (**deprecated**, always **[]**) - :var bytes introduction_points_content: decoded introduction-points content - without authentication data, if using cookie authentication this is - encrypted - :var str signature: signature of the descriptor content - - **\*** attribute is either required when we're parsed with validation or has - a default value, others are left as **None** if undefined - - .. versionchanged:: 1.6.0 - Moved from the deprecated `pycrypto - https://www.dlitz.net/software/pycrypto/`_ module to `cryptography - https://pypi.org/project/cryptography/`_ for validating signatures. - - .. versionchanged:: 1.6.0 - Added the **skip_crypto_validation** constructor argument. - """ - - TYPE_ANNOTATION_NAME = 'hidden-service-descriptor' - - ATTRIBUTES = { - 'descriptor_id': (None, _parse_rendezvous_service_descriptor_line), - 'version': (None, _parse_version_line), - 'permanent_key': (None, _parse_permanent_key_line), - 'secret_id_part': (None, _parse_secret_id_part_line), - 'published': (None, _parse_publication_time_line), - 'protocol_versions': ([], _parse_protocol_versions_line), - 'introduction_points_encoded': (None, _parse_introduction_points_line), - 'introduction_points_auth': ([], _parse_introduction_points_line), - 'introduction_points_content': (None, _parse_introduction_points_line), - 'signature': (None, _parse_signature_line), - } - - PARSER_FOR_LINE = { - 'rendezvous-service-descriptor': _parse_rendezvous_service_descriptor_line, - 'version': _parse_version_line, - 'permanent-key': _parse_permanent_key_line, - 'secret-id-part': _parse_secret_id_part_line, - 'publication-time': _parse_publication_time_line, - 'protocol-versions': _parse_protocol_versions_line, - 'introduction-points': _parse_introduction_points_line, - 'signature': _parse_signature_line, - } - - @classmethod - def content(cls, attr = None, exclude = (), sign = False): - if sign: - raise NotImplementedError('Signing of %s not implemented' % cls.__name__) - - return _descriptor_content(attr, exclude, ( - ('rendezvous-service-descriptor', 'y3olqqblqw2gbh6phimfuiroechjjafa'), - ('version', '2'), - ('permanent-key', _random_crypto_blob('RSA PUBLIC KEY')), - ('secret-id-part', 'e24kgecavwsznj7gpbktqsiwgvngsf4e'), - ('publication-time', _random_date()), - ('protocol-versions', '2,3'), - ('introduction-points', '\n-----BEGIN MESSAGE-----\n-----END MESSAGE-----'), - ), ( - ('signature', _random_crypto_blob('SIGNATURE')), - )) - - @classmethod - def create(cls, attr = None, exclude = (), validate = True, sign = False): - return cls(cls.content(attr, exclude, sign), validate = validate, skip_crypto_validation = not sign) - - def __init__(self, raw_contents, validate = False, skip_crypto_validation = False): - super(HiddenServiceDescriptor, self).__init__(raw_contents, lazy_load = not validate) - entries = _descriptor_components(raw_contents, validate, non_ascii_fields = ('introduction-points')) - - if validate: - for keyword in REQUIRED_FIELDS: - if keyword not in entries: - raise ValueError("Hidden service descriptor must have a '%s' entry" % keyword) - elif keyword in entries and len(entries[keyword]) > 1: - raise ValueError("The '%s' entry can only appear once in a hidden service descriptor" % keyword) - - if 'rendezvous-service-descriptor' != list(entries.keys())[0]: - raise ValueError("Hidden service descriptor must start with a 'rendezvous-service-descriptor' entry") - elif 'signature' != list(entries.keys())[-1]: - raise ValueError("Hidden service descriptor must end with a 'signature' entry") - - self._parse(entries, validate) - - if not skip_crypto_validation and stem.prereq.is_crypto_available(): - signed_digest = self._digest_for_signature(self.permanent_key, self.signature) - digest_content = self._content_range('rendezvous-service-descriptor ', '\nsignature\n') - content_digest = hashlib.sha1(digest_content).hexdigest().upper() - - if signed_digest != content_digest: - raise ValueError('Decrypted digest does not match local digest (calculated: %s, local: %s)' % (signed_digest, content_digest)) - else: - self._entries = entries - - @lru_cache() - def introduction_points(self, authentication_cookie = None): - """ - Provided this service's introduction points. - - :returns: **list** of :class:`~stem.descriptor.hidden_service_descriptor.IntroductionPoints` - - :raises: - * **ValueError** if the our introduction-points is malformed - * **DecryptionFailure** if unable to decrypt this field - """ - - content = self.introduction_points_content - - if not content: - return [] - elif authentication_cookie: - if not stem.prereq.is_crypto_available(): - raise DecryptionFailure('Decrypting introduction-points requires the cryptography module') - - try: - missing_padding = len(authentication_cookie) % 4 - authentication_cookie = base64.b64decode(stem.util.str_tools._to_bytes(authentication_cookie) + b'=' * missing_padding) - except TypeError as exc: - raise DecryptionFailure('authentication_cookie must be a base64 encoded string (%s)' % exc) - - authentication_type = int(binascii.hexlify(content[0:1]), 16) - - if authentication_type == BASIC_AUTH: - content = HiddenServiceDescriptor._decrypt_basic_auth(content, authentication_cookie) - elif authentication_type == STEALTH_AUTH: - content = HiddenServiceDescriptor._decrypt_stealth_auth(content, authentication_cookie) - else: - raise DecryptionFailure("Unrecognized authentication type '%s', currently we only support basic auth (%s) and stealth auth (%s)" % (authentication_type, BASIC_AUTH, STEALTH_AUTH)) - - if not content.startswith(b'introduction-point '): - raise DecryptionFailure('Unable to decrypt the introduction-points, maybe this is the wrong key?') - elif not content.startswith(b'introduction-point '): - raise DecryptionFailure('introduction-points content is encrypted, you need to provide its authentication_cookie') - - return HiddenServiceDescriptor._parse_introduction_points(content) - - @staticmethod - def _decrypt_basic_auth(content, authentication_cookie): - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - from cryptography.hazmat.backends import default_backend - - try: - client_blocks = int(binascii.hexlify(content[1:2]), 16) - except ValueError: - raise DecryptionFailure("When using basic auth the content should start with a number of blocks but wasn't a hex digit: %s" % binascii.hexlify(content[1:2])) - - # parse the client id and encrypted session keys - - client_entries_length = client_blocks * 16 * 20 - client_entries = content[2:2 + client_entries_length] - client_keys = [(client_entries[i:i + 4], client_entries[i + 4:i + 20]) for i in range(0, client_entries_length, 4 + 16)] - - iv = content[2 + client_entries_length:2 + client_entries_length + 16] - encrypted = content[2 + client_entries_length + 16:] - - client_id = hashlib.sha1(authentication_cookie + iv).digest()[:4] - - for entry_id, encrypted_session_key in client_keys: - if entry_id != client_id: - continue # not the session key for this client - - # try decrypting the session key - - cipher = Cipher(algorithms.AES(authentication_cookie), modes.CTR(b'\x00' * len(iv)), default_backend()) - decryptor = cipher.decryptor() - session_key = decryptor.update(encrypted_session_key) + decryptor.finalize() - - # attempt to decrypt the intro points with the session key - - cipher = Cipher(algorithms.AES(session_key), modes.CTR(iv), default_backend()) - decryptor = cipher.decryptor() - decrypted = decryptor.update(encrypted) + decryptor.finalize() - - # check if the decryption looks correct - - if decrypted.startswith(b'introduction-point '): - return decrypted - - return content # nope, unable to decrypt the content - - @staticmethod - def _decrypt_stealth_auth(content, authentication_cookie): - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - from cryptography.hazmat.backends import default_backend - - # byte 1 = authentication type, 2-17 = input vector, 18 on = encrypted content - iv, encrypted = content[1:17], content[17:] - cipher = Cipher(algorithms.AES(authentication_cookie), modes.CTR(iv), default_backend()) - decryptor = cipher.decryptor() - - return decryptor.update(encrypted) + decryptor.finalize() - - @staticmethod - def _parse_introduction_points(content): - """ - Provides the parsed list of IntroductionPoints for the unencrypted content. - """ - - introduction_points = [] - content_io = io.BytesIO(content) - - while True: - content = b''.join(_read_until_keywords('introduction-point', content_io, ignore_first = True)) - - if not content: - break # reached the end - - attr = dict(INTRODUCTION_POINTS_ATTR) - entries = _descriptor_components(content, False) - - for keyword, values in list(entries.items()): - value, block_type, block_contents = values[0] - - if keyword in SINGLE_INTRODUCTION_POINT_FIELDS and len(values) > 1: - raise ValueError("'%s' can only appear once in an introduction-point block, but appeared %i times" % (keyword, len(values))) - - if keyword == 'introduction-point': - attr['identifier'] = value - elif keyword == 'ip-address': - if not stem.util.connection.is_valid_ipv4_address(value): - raise ValueError("'%s' is an invalid IPv4 address" % value) - - attr['address'] = value - elif keyword == 'onion-port': - if not stem.util.connection.is_valid_port(value): - raise ValueError("'%s' is an invalid port" % value) - - attr['port'] = int(value) - elif keyword == 'onion-key': - attr['onion_key'] = block_contents - elif keyword == 'service-key': - attr['service_key'] = block_contents - elif keyword == 'intro-authentication': - auth_entries = [] - - for auth_value, _, _ in values: - if ' ' not in auth_value: - raise ValueError("We expected 'intro-authentication [auth_type] [auth_data]', but had '%s'" % auth_value) - - auth_type, auth_data = auth_value.split(' ')[:2] - auth_entries.append((auth_type, auth_data)) - - introduction_points.append(IntroductionPoints(**attr)) - - return introduction_points +from stem.descriptor.hidden_service import * diff --git a/stem/response/events.py b/stem/response/events.py index a9f563c6..7a5f749b 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -701,7 +701,7 @@ class HSDescContentEvent(Event): :var str directory: hidden service directory servicing the request :var str directory_fingerprint: hidden service directory's finterprint :var str directory_nickname: hidden service directory's nickname if it was provided - :var stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor descriptor: descriptor that was retrieved + :var stem.descriptor.hidden_service.HiddenServiceDescriptor descriptor: descriptor that was retrieved """
_VERSION_ADDED = stem.version.Requirement.EVENT_HS_DESC_CONTENT @@ -726,7 +726,7 @@ class HSDescContentEvent(Event): self.descriptor = None
if desc_content: - self.descriptor = list(stem.descriptor.hidden_service_descriptor._parse_file(io.BytesIO(desc_content)))[0] + self.descriptor = list(stem.descriptor.hidden_service._parse_file(io.BytesIO(desc_content)))[0]
class LogEvent(Event): diff --git a/test/settings.cfg b/test/settings.cfg index 1bdb1a0a..3308c104 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -172,7 +172,7 @@ pycodestyle.ignore E722 pycodestyle.ignore stem/__init__.py => E402: import stem.util.connection pycodestyle.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.bandwidth_file pycodestyle.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.extrainfo_descriptor -pycodestyle.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.hidden_service_descriptor +pycodestyle.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.hidden_service pycodestyle.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.microdescriptor pycodestyle.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.networkstatus pycodestyle.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.server_descriptor @@ -184,9 +184,7 @@ pycodestyle.ignore test/unit/util/connection.py => W291: _tor tor 158 # issue.
pyflakes.ignore run_tests.py => 'unittest' imported but unused -pyflakes.ignore stem/client/datatype.py => redefinition of unused 'pop' from * pyflakes.ignore stem/control.py => undefined name 'controller' -pyflakes.ignore stem/interpreter/__init__.py => undefined name 'raw_input' pyflakes.ignore stem/manual.py => undefined name 'unichr' pyflakes.ignore stem/prereq.py => 'int_to_bytes' imported but unused pyflakes.ignore stem/prereq.py => 'int_from_bytes' imported but unused @@ -210,6 +208,10 @@ pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.modes' pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.Cipher' imported but unused pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.algorithms' imported but unused pyflakes.ignore stem/prereq.py => 'lzma' imported but unused +pyflakes.ignore stem/client/datatype.py => redefinition of unused 'pop' from * +pyflakes.ignore stem/descriptor/hidden_service_descriptor.py => 'stem.descriptor.hidden_service.*' imported but unused +pyflakes.ignore stem/descriptor/hidden_service_descriptor.py => 'from stem.descriptor.hidden_service import *' used; unable to detect undefined names +pyflakes.ignore stem/interpreter/__init__.py => undefined name 'raw_input' pyflakes.ignore stem/response/events.py => undefined name 'long' pyflakes.ignore stem/util/__init__.py => undefined name 'long' pyflakes.ignore stem/util/__init__.py => undefined name 'unicode' @@ -256,7 +258,7 @@ test.unit_tests |test.unit.descriptor.networkstatus.document_v2.TestNetworkStatusDocument |test.unit.descriptor.networkstatus.document_v3.TestNetworkStatusDocument |test.unit.descriptor.networkstatus.bridge_document.TestBridgeNetworkStatusDocument -|test.unit.descriptor.hidden_service_descriptor.TestHiddenServiceDescriptor +|test.unit.descriptor.hidden_service.TestHiddenServiceDescriptor |test.unit.descriptor.certificate.TestEd25519Certificate |test.unit.descriptor.bandwidth_file.TestBandwidthFile |test.unit.exit_policy.rule.TestExitPolicyRule diff --git a/test/unit/descriptor/hidden_service_descriptor.py b/test/unit/descriptor/hidden_service.py similarity index 99% rename from test/unit/descriptor/hidden_service_descriptor.py rename to test/unit/descriptor/hidden_service.py index e9ba012b..c49741d3 100644 --- a/test/unit/descriptor/hidden_service_descriptor.py +++ b/test/unit/descriptor/hidden_service.py @@ -1,5 +1,5 @@ """ -Unit tests for stem.descriptor.hidden_service_descriptor. +Unit tests for stem.descriptor.hidden_service. """
import datetime @@ -10,7 +10,7 @@ import stem.descriptor import stem.prereq import test.require
-from stem.descriptor.hidden_service_descriptor import ( +from stem.descriptor.hidden_service import ( REQUIRED_FIELDS, DecryptionFailure, HiddenServiceDescriptor,