October 2019 - tor-commits - lists.torproject.org

[stem/master] Note when ed25519 support was added
by atagar＠torproject.org 06 Oct '19

06 Oct '19

commit d7608f7916e3af20805662a062cd76589ee144ec Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Sep 12 15:21:01 2019 -0700 Note when ed25519 support was added The ed25519 module itself does not say when it was added [1], but the changelog [2] indicates that support was added in version 2.6... 2.6 - 2019-02-27 Added support for Ed25519 signing when using OpenSSL 1.1.1b or newer. [1] https://cryptography.io/en/latest/hazmat/primitives/asymmetric/ed25519/ [2] https://cryptography.io/en/latest/changelog/ --- stem/descriptor/hsv3_crypto.py | 4 +--- stem/prereq.py | 7 +++---- test/unit/descriptor/hidden_service_v3.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/stem/descriptor/hsv3_crypto.py b/stem/descriptor/hsv3_crypto.py index 8f304c52..c3f1feb0 100644 --- a/stem/descriptor/hsv3_crypto.py +++ b/stem/descriptor/hsv3_crypto.py @@ -30,10 +30,8 @@ def decode_address(onion_address_str): :raises: ValueError """ - # TODO: note the module version - if not stem.prereq.is_crypto_available(ed25519 = True): - raise ImportError('Onion address decoding requires cryptography version XXX') + raise ImportError('Onion address decoding requires cryptography version 2.6') from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey diff --git a/stem/prereq.py b/stem/prereq.py index 0aadb35a..bf86ddef 100644 --- a/stem/prereq.py +++ b/stem/prereq.py @@ -119,7 +119,9 @@ def is_crypto_available(ed25519 = False): Checks if the cryptography functions we use are available. This is used for verifying relay descriptor signatures. - :param bool ed25519: check for ed25519 support + :param bool ed25519: check for `ed25519 support + <https://cryptography.io/en/latest/hazmat/primitives/asymmetric/ed25519/>`_, + which was added in version 2.6 :returns: **True** if we can use the cryptography module and **False** otherwise @@ -136,9 +138,6 @@ def is_crypto_available(ed25519 = False): if not hasattr(rsa.RSAPrivateKey, 'sign'): raise ImportError() - # TODO: Check when the cryptography module's ed25519 class was added - # (it's not present in 2.0.3). - if ed25519: from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey diff --git a/test/unit/descriptor/hidden_service_v3.py b/test/unit/descriptor/hidden_service_v3.py index f4c42008..4fdc72bd 100644 --- a/test/unit/descriptor/hidden_service_v3.py +++ b/test/unit/descriptor/hidden_service_v3.py @@ -40,7 +40,7 @@ class TestHiddenServiceDescriptorV3(unittest.TestCase): """ if not stem.prereq.is_crypto_available(ed25519 = True): - self.skipTest('(requires cryptography ed25519 support)') # TODO: note version here + self.skipTest('(requires cryptography v2.6)') return with open(get_resource('hidden_service_v3_test'), 'rb') as descriptor_file:

1 0

[stem/master] Passing unit tests
by atagar＠torproject.org 06 Oct '19

06 Oct '19

commit 419a2938c1d313491401b48de2697a5a8ca5c5a4 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Sep 5 12:42:39 2019 -0700 Passing unit tests Fix remaining test failures so unit tests once again pass. Stylistic checks still report issues so that's next. --- stem/descriptor/hidden_service.py | 12 ++++++------ test/unit/descriptor/__init__.py | 8 +++++++- test/unit/descriptor/hidden_service_v3.py | 7 ++++++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/stem/descriptor/hidden_service.py b/stem/descriptor/hidden_service.py index 1c51b7b2..9e6de0d9 100644 --- a/stem/descriptor/hidden_service.py +++ b/stem/descriptor/hidden_service.py @@ -554,12 +554,6 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): else: self._entries = entries - # ATAGAR XXX need to do this cert extraction in the parsing handler - assert(self.signing_cert) - cert_lines = self.signing_cert.split('\n') - assert(cert_lines[0] == '-----BEGIN ED25519 CERT-----' and cert_lines[-1] == '-----END ED25519 CERT-----') - desc_signing_cert = stem.descriptor.certificate.Ed25519Certificate.parse(''.join(cert_lines[1:-1])) - # crypto validation (check skip_crypto_validation) # ASN XXX need to verify descriptor signing certificate (for now we trust Tor to do it) # ASN XXX need to verify descriptor signature (for now we trust Tor to do it) @@ -568,6 +562,12 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): if self.onion_address is None: raise ValueError("Onion address is required to decrypt v3 hidden service descriptors") + # ATAGAR XXX need to do this cert extraction in the parsing handler + assert(self.signing_cert) + cert_lines = self.signing_cert.split('\n') + assert(cert_lines[0] == '-----BEGIN ED25519 CERT-----' and cert_lines[-1] == '-----END ED25519 CERT-----') + + desc_signing_cert = stem.descriptor.certificate.Ed25519Certificate.parse(''.join(cert_lines[1:-1])) plaintext = self.decrypt_descriptor(desc_signing_cert) def decrypt_descriptor(self, desc_signing_cert): diff --git a/test/unit/descriptor/__init__.py b/test/unit/descriptor/__init__.py index 867ed0a4..268e61e4 100644 --- a/test/unit/descriptor/__init__.py +++ b/test/unit/descriptor/__init__.py @@ -4,6 +4,8 @@ Unit tests for stem.descriptor. import os +import stem.descriptor.hidden_service + __all__ = [ 'bandwidth_file', 'collector', @@ -49,7 +51,11 @@ def base_expect_invalid_attr_for_text(cls, default_attr, default_prefix, test, d """ test.assertRaises(ValueError, cls, desc_text, True) - desc = cls(desc_text, validate = False) + + if cls == stem.descriptor.hidden_service.HiddenServiceDescriptorV3: + desc = cls(desc_text, validate = False, skip_crypto_validation = True) + else: + desc = cls(desc_text, validate = False) if attr: # check that the invalid attribute matches the expected value when diff --git a/test/unit/descriptor/hidden_service_v3.py b/test/unit/descriptor/hidden_service_v3.py index 97763a39..f4c42008 100644 --- a/test/unit/descriptor/hidden_service_v3.py +++ b/test/unit/descriptor/hidden_service_v3.py @@ -6,6 +6,7 @@ import functools import unittest import stem.descriptor +import stem.prereq from stem.descriptor.hidden_service import ( REQUIRED_V3_FIELDS, @@ -38,6 +39,10 @@ class TestHiddenServiceDescriptorV3(unittest.TestCase): sltib6sxkuxh2scmtuvd5w2g7pahnzkovefxpo4e4ptnkzl5kkq5h2ad.onion """ + if not stem.prereq.is_crypto_available(ed25519 = True): + self.skipTest('(requires cryptography ed25519 support)') # TODO: note version here + return + with open(get_resource('hidden_service_v3_test'), 'rb') as descriptor_file: desc = next(stem.descriptor.parse_file(descriptor_file, 'hidden-service-descriptor-3 1.0', validate = True, onion_address="sltib6sxkuxh2scmtuvd5w2g7pahnzkovefxpo4e4ptnkzl5kkq5h2ad.onion")) @@ -54,7 +59,7 @@ class TestHiddenServiceDescriptorV3(unittest.TestCase): """ with open(get_resource('hidden_service_v3'), 'rb') as descriptor_file: - desc = next(stem.descriptor.parse_file(descriptor_file, 'hidden-service-descriptor-3 1.0', validate = True)) + desc = next(stem.descriptor.parse_file(descriptor_file, 'hidden-service-descriptor-3 1.0', validate = True, skip_crypto_validation = True)) self.assertEqual(3, desc.version) self.assertEqual(180, desc.lifetime)

1 0

[stem/master] Only require onion addresses when decrypting
by atagar＠torproject.org 06 Oct '19

06 Oct '19

commit 60864adc966a03ec9d31aa2312c5068286ac73fa Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Sep 4 17:59:03 2019 -0700 Only require onion addresses when decrypting Minor tweek so a few of our tests can pass... ====================================================================== ERROR: test_invalid_lifetime ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/atagar/Desktop/stem/test/unit/descriptor/hidden_service_v3.py", line 110, in test_invalid_lifetime expect_invalid_attr(self, {'descriptor-lifetime': test_value}, 'lifetime') File "/home/atagar/Desktop/stem/test/unit/descriptor/__init__.py", line 41, in base_expect_invalid_attr return base_expect_invalid_attr_for_text(cls, default_attr, default_value, test, cls.content(desc_attrs), attr, expected_value) File "/home/atagar/Desktop/stem/test/unit/descriptor/__init__.py", line 52, in base_expect_invalid_attr_for_text desc = cls(desc_text, validate = False) File "/home/atagar/Desktop/stem/stem/descriptor/hidden_service.py", line 540, in __init__ raise ValueError("The onion address MUST be provided to parse a V3 descriptor") ValueError: The onion address MUST be provided to parse a V3 descriptor --- stem/descriptor/hidden_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/hidden_service.py b/stem/descriptor/hidden_service.py index 150ba368..1c51b7b2 100644 --- a/stem/descriptor/hidden_service.py +++ b/stem/descriptor/hidden_service.py @@ -535,9 +535,6 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): """ super(HiddenServiceDescriptorV3, self).__init__(raw_contents, lazy_load = not validate) entries = _descriptor_components(raw_contents, validate) - - if onion_address == None: - raise ValueError("The onion address MUST be provided to parse a V3 descriptor") self.onion_address = onion_address # XXX Do this parsing in its own function @@ -568,6 +565,9 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): # ASN XXX need to verify descriptor signature (for now we trust Tor to do it) if not skip_crypto_validation and stem.prereq.is_crypto_available(): + if self.onion_address is None: + raise ValueError("Onion address is required to decrypt v3 hidden service descriptors") + plaintext = self.decrypt_descriptor(desc_signing_cert) def decrypt_descriptor(self, desc_signing_cert):

1 0

[stem/master] Adjust hsv3_crypto indentation
by atagar＠torproject.org 06 Oct '19

06 Oct '19

commit c901fa5a9bfd51978e2bb3f72c0d6322804abe11 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Sep 12 16:51:58 2019 -0700 Adjust hsv3_crypto indentation Stem uses two space indentations rather than four. Adjusting this new module to match the rest of the codebase. --- stem/descriptor/hsv3_crypto.py | 279 ++++++++++++++++++++--------------------- 1 file changed, 138 insertions(+), 141 deletions(-) diff --git a/stem/descriptor/hsv3_crypto.py b/stem/descriptor/hsv3_crypto.py index c83165ed..2e5288a1 100644 --- a/stem/descriptor/hsv3_crypto.py +++ b/stem/descriptor/hsv3_crypto.py @@ -6,68 +6,67 @@ import stem.prereq """ Onion addresses - onion_address = base32(PUBKEY | CHECKSUM | VERSION) + '.onion' - CHECKSUM = H('.onion checksum' | PUBKEY | VERSION)[:2] - - - PUBKEY is the 32 bytes ed25519 master pubkey of the hidden service. - - VERSION is an one byte version field (default value '\x03') - - '.onion checksum' is a constant string - - CHECKSUM is truncated to two bytes before inserting it in onion_address + onion_address = base32(PUBKEY | CHECKSUM | VERSION) + '.onion' + CHECKSUM = H('.onion checksum' | PUBKEY | VERSION)[:2] + - PUBKEY is the 32 bytes ed25519 master pubkey of the hidden service. + - VERSION is an one byte version field (default value '\x03') + - '.onion checksum' is a constant string + - CHECKSUM is truncated to two bytes before inserting it in onion_address """ CHECKSUM_CONSTANT = b'.onion checksum' def decode_address(onion_address_str): - """ - Parse onion_address_str and return the pubkey. + """ + Parse onion_address_str and return the pubkey. - onion_address = base32(PUBKEY | CHECKSUM | VERSION) + '.onion' - CHECKSUM = H('.onion checksum' | PUBKEY | VERSION)[:2] + onion_address = base32(PUBKEY | CHECKSUM | VERSION) + '.onion' + CHECKSUM = H('.onion checksum' | PUBKEY | VERSION)[:2] - :return: Ed25519PublicKey + :return: Ed25519PublicKey - :raises: ValueError - """ + :raises: ValueError + """ - if not stem.prereq.is_crypto_available(ed25519 = True): - raise ImportError('Onion address decoding requires cryptography version 2.6') + if not stem.prereq.is_crypto_available(ed25519 = True): + raise ImportError('Onion address decoding requires cryptography version 2.6') - from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey - if (len(onion_address_str) != 56 + len('.onion')): - raise ValueError('Wrong address length') + if (len(onion_address_str) != 56 + len('.onion')): + raise ValueError('Wrong address length') - # drop the '.onion' - onion_address = onion_address_str[:56] + # drop the '.onion' + onion_address = onion_address_str[:56] - # base32 decode the addr (convert to uppercase since that's what python expects) - onion_address = base64.b32decode(onion_address.upper()) - assert(len(onion_address) == 35) + # base32 decode the addr (convert to uppercase since that's what python expects) + onion_address = base64.b32decode(onion_address.upper()) + assert(len(onion_address) == 35) - # extract pieces of information - pubkey = onion_address[:32] - checksum = onion_address[32:34] - version = onion_address[34] + # extract pieces of information + pubkey = onion_address[:32] + checksum = onion_address[32:34] + version = onion_address[34] - # Do checksum validation - my_checksum_body = b'%s%s%s' % (CHECKSUM_CONSTANT, pubkey, bytes([version])) - my_checksum = hashlib.sha3_256(my_checksum_body).digest() + # Do checksum validation + my_checksum_body = b'%s%s%s' % (CHECKSUM_CONSTANT, pubkey, bytes([version])) + my_checksum = hashlib.sha3_256(my_checksum_body).digest() - if (checksum != my_checksum[:2]): - raise ValueError('Bad checksum') + if (checksum != my_checksum[:2]): + raise ValueError('Bad checksum') - return Ed25519PublicKey.from_public_bytes(pubkey) + return Ed25519PublicKey.from_public_bytes(pubkey) """ Blinded key stuff - Now wrt SRVs, if a client is in the time segment between a new time period - and a new SRV (i.e. the segments drawn with '-') it uses the current SRV, - else if the client is in a time segment between a new SRV and a new time - period (i.e. the segments drawn with '='), it uses the previous SRV. + Now wrt SRVs, if a client is in the time segment between a new time period + and a new SRV (i.e. the segments drawn with '-') it uses the current SRV, + else if the client is in a time segment between a new SRV and a new time + period (i.e. the segments drawn with '='), it uses the previous SRV. """ pass @@ -75,55 +74,55 @@ pass """ Subcredential: - subcredential = H('subcredential' | credential | blinded-public-key - credential = H('credential' | public-identity-key) + subcredential = H('subcredential' | credential | blinded-public-key + credential = H('credential' | public-identity-key) Both keys are in bytes """ def get_subcredential(public_identity_key, blinded_key): - cred_bytes_constant = 'credential'.encode() - subcred_bytes_constant = 'subcredential'.encode() + cred_bytes_constant = 'credential'.encode() + subcred_bytes_constant = 'subcredential'.encode() - credential = hashlib.sha3_256(b'%s%s' % (cred_bytes_constant, public_identity_key)).digest() - subcredential = hashlib.sha3_256(b'%s%s%s' % (subcred_bytes_constant, credential, blinded_key)).digest() + credential = hashlib.sha3_256(b'%s%s' % (cred_bytes_constant, public_identity_key)).digest() + subcredential = hashlib.sha3_256(b'%s%s%s' % (subcred_bytes_constant, credential, blinded_key)).digest() - print('public_identity_key: %s' % (public_identity_key.hex())) - print('credential: %s' % (credential.hex())) - print('blinded_key: %s' % (blinded_key.hex())) - print('subcredential: %s' % (subcredential.hex())) + print('public_identity_key: %s' % (public_identity_key.hex())) + print('credential: %s' % (credential.hex())) + print('blinded_key: %s' % (blinded_key.hex())) + print('subcredential: %s' % (subcredential.hex())) - print('===') + print('===') - return subcredential + return subcredential """ Basic descriptor logic: - SALT = 16 bytes from H(random), changes each time we rebuld the - descriptor even if the content of the descriptor hasn't changed. - (So that we don't leak whether the intro point list etc. changed) + SALT = 16 bytes from H(random), changes each time we rebuld the + descriptor even if the content of the descriptor hasn't changed. + (So that we don't leak whether the intro point list etc. changed) - secret_input = SECRET_DATA | subcredential | INT_8(revision_counter) + secret_input = SECRET_DATA | subcredential | INT_8(revision_counter) - keys = KDF(secret_input | salt | STRING_CONSTANT, S_KEY_LEN + S_IV_LEN + MAC_KEY_LEN) + keys = KDF(secret_input | salt | STRING_CONSTANT, S_KEY_LEN + S_IV_LEN + MAC_KEY_LEN) - SECRET_KEY = first S_KEY_LEN bytes of keys - SECRET_IV = next S_IV_LEN bytes of keys - MAC_KEY = last MAC_KEY_LEN bytes of keys + SECRET_KEY = first S_KEY_LEN bytes of keys + SECRET_IV = next S_IV_LEN bytes of keys + MAC_KEY = last MAC_KEY_LEN bytes of keys Layer data: - 2.5.1.1. First layer encryption logic - SECRET_DATA = blinded-public-key - STRING_CONSTANT = 'hsdir-superencrypted-data' + 2.5.1.1. First layer encryption logic + SECRET_DATA = blinded-public-key + STRING_CONSTANT = 'hsdir-superencrypted-data' - 2.5.2.1. Second layer encryption keys - SECRET_DATA = blinded-public-key | descriptor_cookie - STRING_CONSTANT = 'hsdir-encrypted-data' + 2.5.2.1. Second layer encryption keys + SECRET_DATA = blinded-public-key | descriptor_cookie + STRING_CONSTANT = 'hsdir-encrypted-data' """ SALT_LEN = 16 @@ -135,117 +134,115 @@ MAC_KEY_LEN = 32 def _ciphertext_mac_is_valid(key, salt, ciphertext, mac): - """ - Instantiate MAC(key=k, message=m) with H(k_len | k | m), where k_len is - htonll(len(k)). + """ + Instantiate MAC(key=k, message=m) with H(k_len | k | m), where k_len is + htonll(len(k)). - XXX spec: H(mac_key_len | mac_key | salt_len | salt | encrypted) - """ + XXX spec: H(mac_key_len | mac_key | salt_len | salt | encrypted) + """ - # Construct our own MAC first - key_len = len(key).to_bytes(8, 'big') - salt_len = len(salt).to_bytes(8, 'big') + # Construct our own MAC first + key_len = len(key).to_bytes(8, 'big') + salt_len = len(salt).to_bytes(8, 'big') - my_mac_body = b'%s%s%s%s%s' % (key_len, key, salt_len, salt, ciphertext) - my_mac = hashlib.sha3_256(my_mac_body).digest() + my_mac_body = b'%s%s%s%s%s' % (key_len, key, salt_len, salt, ciphertext) + my_mac = hashlib.sha3_256(my_mac_body).digest() - print('===') - print('my mac: %s' % my_mac.hex()) - print('their mac: %s' % mac.hex()) + print('===') + print('my mac: %s' % my_mac.hex()) + print('their mac: %s' % mac.hex()) - # Compare the two MACs - return my_mac == mac + # Compare the two MACs + return my_mac == mac -def _decrypt_descriptor_layer(ciphertext_blob_b64, revision_counter, - public_identity_key, subcredential, - secret_data, string_constant): - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - from cryptography.hazmat.backends import default_backend +def _decrypt_descriptor_layer(ciphertext_blob_b64, revision_counter, public_identity_key, subcredential, secret_data, string_constant): + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend - # decode the thing - ciphertext_blob = base64.b64decode(ciphertext_blob_b64) + # decode the thing + ciphertext_blob = base64.b64decode(ciphertext_blob_b64) - if (len(ciphertext_blob) < SALT_LEN + MAC_LEN): - raise ValueError('bad encrypted blob') + if (len(ciphertext_blob) < SALT_LEN + MAC_LEN): + raise ValueError('bad encrypted blob') - salt = ciphertext_blob[:16] - ciphertext = ciphertext_blob[16:-32] - mac = ciphertext_blob[-32:] + salt = ciphertext_blob[:16] + ciphertext = ciphertext_blob[16:-32] + mac = ciphertext_blob[-32:] - print('encrypted blob lenth :%s' % len(ciphertext_blob)) - print('salt: %s' % salt.hex()) - print('ciphertext length: %s' % len(ciphertext)) - print('mac: %s' % mac.hex()) - print('===') + print('encrypted blob lenth :%s' % len(ciphertext_blob)) + print('salt: %s' % salt.hex()) + print('ciphertext length: %s' % len(ciphertext)) + print('mac: %s' % mac.hex()) + print('===') - # INT_8(revision_counter) - rev_counter_int_8 = revision_counter.to_bytes(8, 'big') - secret_input = b'%s%s%s' % (secret_data, subcredential, rev_counter_int_8) - secret_input = secret_input + # INT_8(revision_counter) + rev_counter_int_8 = revision_counter.to_bytes(8, 'big') + secret_input = b'%s%s%s' % (secret_data, subcredential, rev_counter_int_8) + secret_input = secret_input - print('secret_data (%d): %s' % (len(secret_data), secret_data.hex())) - print('subcredential (%d): %s' % (len(subcredential), subcredential.hex())) - print('rev counter int 8 (%d): %s' % (len(rev_counter_int_8), rev_counter_int_8.hex())) - print('secret_input (%s): %s' % (len(secret_input), secret_input.hex())) - print('===') + print('secret_data (%d): %s' % (len(secret_data), secret_data.hex())) + print('subcredential (%d): %s' % (len(subcredential), subcredential.hex())) + print('rev counter int 8 (%d): %s' % (len(rev_counter_int_8), rev_counter_int_8.hex())) + print('secret_input (%s): %s' % (len(secret_input), secret_input.hex())) + print('===') - kdf = hashlib.shake_256(b'%s%s%s' % (secret_input, salt, string_constant)) - keys = kdf.digest(S_KEY_LEN + S_IV_LEN + MAC_KEY_LEN) + kdf = hashlib.shake_256(b'%s%s%s' % (secret_input, salt, string_constant)) + keys = kdf.digest(S_KEY_LEN + S_IV_LEN + MAC_KEY_LEN) - secret_key = keys[:S_KEY_LEN] - secret_iv = keys[S_KEY_LEN:S_KEY_LEN + S_IV_LEN] - mac_key = keys[S_KEY_LEN + S_IV_LEN:] + secret_key = keys[:S_KEY_LEN] + secret_iv = keys[S_KEY_LEN:S_KEY_LEN + S_IV_LEN] + mac_key = keys[S_KEY_LEN + S_IV_LEN:] - print('secret_key: %s' % secret_key.hex()) - print('secret_iv: %s' % secret_iv.hex()) - print('mac_key: %s' % mac_key.hex()) + print('secret_key: %s' % secret_key.hex()) + print('secret_iv: %s' % secret_iv.hex()) + print('mac_key: %s' % mac_key.hex()) - # Now time to decrypt descriptor - cipher = Cipher(algorithms.AES(secret_key), modes.CTR(secret_iv), default_backend()) - decryptor = cipher.decryptor() - decrypted = decryptor.update(ciphertext) + decryptor.finalize() + # Now time to decrypt descriptor + cipher = Cipher(algorithms.AES(secret_key), modes.CTR(secret_iv), default_backend()) + decryptor = cipher.decryptor() + decrypted = decryptor.update(ciphertext) + decryptor.finalize() - # validate mac (the mac validates the two fields before the mac) - if not _ciphertext_mac_is_valid(mac_key, salt, ciphertext, mac): - raise ValueError('Bad MAC!!!') + # validate mac (the mac validates the two fields before the mac) + if not _ciphertext_mac_is_valid(mac_key, salt, ciphertext, mac): + raise ValueError('Bad MAC!!!') - return decrypted + return decrypted def decrypt_outter_layer(superencrypted_blob_b64, revision_counter, public_identity_key, blinded_key, subcredential): - secret_data = blinded_key - string_constant = b'hsdir-superencrypted-data' + secret_data = blinded_key + string_constant = b'hsdir-superencrypted-data' - # XXX Remove the BEGIN MESSSAGE around the thing - superencrypted_blob_b64_lines = superencrypted_blob_b64.split('\n') - assert(superencrypted_blob_b64_lines[0] == '-----BEGIN MESSAGE-----') - assert(superencrypted_blob_b64_lines[-1] == '-----END MESSAGE-----') - superencrypted_blob_b64 = ''.join(superencrypted_blob_b64_lines[1:-1]) + # XXX Remove the BEGIN MESSSAGE around the thing + superencrypted_blob_b64_lines = superencrypted_blob_b64.split('\n') + assert(superencrypted_blob_b64_lines[0] == '-----BEGIN MESSAGE-----') + assert(superencrypted_blob_b64_lines[-1] == '-----END MESSAGE-----') + superencrypted_blob_b64 = ''.join(superencrypted_blob_b64_lines[1:-1]) - print('====== Decrypting outter layer =======') + print('====== Decrypting outter layer =======') - return _decrypt_descriptor_layer(superencrypted_blob_b64, revision_counter, public_identity_key, subcredential, secret_data, string_constant) + return _decrypt_descriptor_layer(superencrypted_blob_b64, revision_counter, public_identity_key, subcredential, secret_data, string_constant) def decrypt_inner_layer(encrypted_blob_b64, revision_counter, public_identity_key, blinded_key, subcredential): - secret_data = blinded_key - string_constant = b'hsdir-encrypted-data' + secret_data = blinded_key + string_constant = b'hsdir-encrypted-data' - print('====== Decrypting inner layer =======') + print('====== Decrypting inner layer =======') - return _decrypt_descriptor_layer(encrypted_blob_b64, revision_counter, public_identity_key, subcredential, secret_data, string_constant) + return _decrypt_descriptor_layer(encrypted_blob_b64, revision_counter, public_identity_key, subcredential, secret_data, string_constant) def parse_superencrypted_plaintext(outter_layer_plaintext): - """Super hacky function to parse the superencrypted plaintext. This will need to be replaced by proper stem code.""" + """Super hacky function to parse the superencrypted plaintext. This will need to be replaced by proper stem code.""" - START_CONSTANT = b'-----BEGIN MESSAGE-----\n' - END_CONSTANT = b'\n-----END MESSAGE-----' + START_CONSTANT = b'-----BEGIN MESSAGE-----\n' + END_CONSTANT = b'\n-----END MESSAGE-----' - start = outter_layer_plaintext.find(START_CONSTANT) - end = outter_layer_plaintext.find(END_CONSTANT) + start = outter_layer_plaintext.find(START_CONSTANT) + end = outter_layer_plaintext.find(END_CONSTANT) - start = start + len(START_CONSTANT) + start = start + len(START_CONSTANT) - return outter_layer_plaintext[start:end] + return outter_layer_plaintext[start:end]

1 0

[stem/master] Separate hidden service decryption from constructor
by atagar＠torproject.org 06 Oct '19

06 Oct '19

commit 14a44b1c6e1438abdf5687a1c468536d88481f81 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 22 14:02:06 2019 -0700 Separate hidden service decryption from constructor Moving hidden service description from the constructor into a method for a couple reasons... 1. Wherever possible we lazy rather than eagerly evaluate descriptor content. We do not know upfront what fields our user will want, so best to avoid doing unnecessarily decryption (which is particularly cpu intensive) unless our user indicates that they want this content. 2. We cannot modify constructor signatures without plumbing arguments through several other methods. Not necessarily a no-go, but decryption requires an onion_address whereas construction does not so separating this into its own method sidesteps that headache. --- stem/descriptor/hidden_service.py | 32 ++++++++++--------------------- test/unit/descriptor/hidden_service_v3.py | 5 +++-- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/stem/descriptor/hidden_service.py b/stem/descriptor/hidden_service.py index fc898f93..fda5147b 100644 --- a/stem/descriptor/hidden_service.py +++ b/stem/descriptor/hidden_service.py @@ -530,16 +530,10 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): def create(cls, attr = None, exclude = (), validate = True, sign = False): return cls(cls.content(attr, exclude, sign), validate = validate, skip_crypto_validation = not sign) - def __init__(self, raw_contents, validate = False, onion_address = None, skip_crypto_validation = False): - """ - The onion_address is needed so that we can decrypt the descriptor, which is - impossible without the full onion address. - """ + def __init__(self, raw_contents, validate = False, skip_crypto_validation = False): super(HiddenServiceDescriptorV3, self).__init__(raw_contents, lazy_load = not validate) entries = _descriptor_components(raw_contents, validate) - self.onion_address = onion_address - # XXX Do this parsing in its own function if validate: for keyword in REQUIRED_V3_FIELDS: if keyword not in entries: @@ -556,23 +550,17 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): else: self._entries = entries - # crypto validation (check skip_crypto_validation) - # ASN XXX need to verify descriptor signing certificate (for now we trust Tor to do it) - # ASN XXX need to verify descriptor signature (for now we trust Tor to do it) - - if not skip_crypto_validation and stem.prereq.is_crypto_available(): - if self.onion_address is None: - raise ValueError('Onion address is required to decrypt v3 hidden service descriptors') + # TODO: The following is only marked as private because it is a work in + # progress. This will probably become something like "body()" which decrypts + # and parses the internal descriptor content. - # ATAGAR XXX need to do this cert extraction in the parsing handler - assert(self.signing_cert) - cert_lines = self.signing_cert.split('\n') - assert(cert_lines[0] == '-----BEGIN ED25519 CERT-----' and cert_lines[-1] == '-----END ED25519 CERT-----') + def _decrypt(self, onion_address): + assert(self.signing_cert) + cert_lines = self.signing_cert.split('\n') + assert(cert_lines[0] == '-----BEGIN ED25519 CERT-----' and cert_lines[-1] == '-----END ED25519 CERT-----') - desc_signing_cert = stem.descriptor.certificate.Ed25519Certificate.parse(''.join(cert_lines[1:-1])) - self.plaintext = self.decrypt_descriptor(desc_signing_cert) + desc_signing_cert = stem.descriptor.certificate.Ed25519Certificate.parse(''.join(cert_lines[1:-1])) - def decrypt_descriptor(self, desc_signing_cert): # Get crypto material. # ASN XXX Extract to its own function and assign them to class variables from cryptography.hazmat.primitives import serialization @@ -585,7 +573,7 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): if not blinded_key_bytes: raise ValueError('No signing key extension present') - identity_public_key = stem.descriptor.hsv3_crypto.decode_address(self.onion_address) + identity_public_key = stem.descriptor.hsv3_crypto.decode_address(onion_address) identity_public_key_bytes = identity_public_key.public_bytes(encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw) assert(len(identity_public_key_bytes) == 32) diff --git a/test/unit/descriptor/hidden_service_v3.py b/test/unit/descriptor/hidden_service_v3.py index 98de246e..ac2f8ea7 100644 --- a/test/unit/descriptor/hidden_service_v3.py +++ b/test/unit/descriptor/hidden_service_v3.py @@ -44,13 +44,14 @@ class TestHiddenServiceDescriptorV3(unittest.TestCase): return with open(get_resource('hidden_service_v3_test'), 'rb') as descriptor_file: - desc = next(stem.descriptor.parse_file(descriptor_file, 'hidden-service-descriptor-3 1.0', validate = True, - onion_address = 'sltib6sxkuxh2scmtuvd5w2g7pahnzkovefxpo4e4ptnkzl5kkq5h2ad.onion')) + desc = next(stem.descriptor.parse_file(descriptor_file, 'hidden-service-descriptor-3 1.0', validate = True)) self.assertEqual(3, desc.version) self.assertEqual(180, desc.lifetime) self.assertEqual(42, desc.revision_counter) + desc._decrypt('sltib6sxkuxh2scmtuvd5w2g7pahnzkovefxpo4e4ptnkzl5kkq5h2ad.onion') + def test_for_riseup(self): """ Parse riseup's descriptor...

1 0

[stem/master] Drop get_signing_key() helper
by atagar＠torproject.org 06 Oct '19

06 Oct '19

commit 051870fc669e37e1954fc3bbd9a878ad02099ae5 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 22 13:33:40 2019 -0700 Drop get_signing_key() helper Our validate() method already does this, but can't share the helper due to differing exception messages. This helper was tiny and did not need to check signing key size because our constructor already does so. --- stem/descriptor/certificate.py | 31 +++++-------------------------- stem/descriptor/hidden_service.py | 12 ++++++++++-- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/stem/descriptor/certificate.py b/stem/descriptor/certificate.py index 5895790f..f1f7483e 100644 --- a/stem/descriptor/certificate.py +++ b/stem/descriptor/certificate.py @@ -4,7 +4,11 @@ """ Parsing for `Tor Ed25519 certificates <https://gitweb.torproject.org/torspec.git/tree/cert-spec.txt>`_, which are -used to validate the key used to sign server descriptors. +used to for a variety of purposes... + + * validating the key used to sign server descriptors + * validating the key used to sign hidden service v3 descriptors + * signing and encrypting hidden service v3 indroductory points .. versionadded:: 1.6.0 @@ -295,28 +299,3 @@ class Ed25519CertificateV1(Ed25519Certificate): verify_key.verify(signature_bytes, descriptor_sha256_digest) except InvalidSignature: raise ValueError('Descriptor Ed25519 certificate signature invalid (Signature was forged or corrupt)') - - def get_signing_key(self): - """ - Get the signing key for this certificate. This is included in the extensions. - WARNING: This is the key that signed the certificate, not the key that got - certified. - - :returns: Raw bytes of an ed25519 key. - - :raises: **ValueError** if the signing key cannot be found. - """ - signing_key_extension = None - - for extension in self.extensions: - if extension.type == ExtensionType.HAS_SIGNING_KEY: - signing_key_extension = extension - break - - if not signing_key_extension: - raise ValueError('Signing key extension could not be found') - - if (len(signing_key_extension.data) != 32): - raise ValueError('Signing key extension has malformed key') - - return signing_key_extension.data diff --git a/stem/descriptor/hidden_service.py b/stem/descriptor/hidden_service.py index 83aff8b8..fc898f93 100644 --- a/stem/descriptor/hidden_service.py +++ b/stem/descriptor/hidden_service.py @@ -53,6 +53,8 @@ from stem.descriptor import ( _random_crypto_blob, ) +from stem.descriptor.certificate import ExtensionType + if stem.prereq._is_lru_cache_available(): from functools import lru_cache else: @@ -575,12 +577,18 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): # ASN XXX Extract to its own function and assign them to class variables from cryptography.hazmat.primitives import serialization - blinded_key_bytes = desc_signing_cert.get_signing_key() + for extension in desc_signing_cert.extensions: + if extension.type == ExtensionType.HAS_SIGNING_KEY: + blinded_key_bytes = extension.data + break + + if not blinded_key_bytes: + raise ValueError('No signing key extension present') + identity_public_key = stem.descriptor.hsv3_crypto.decode_address(self.onion_address) identity_public_key_bytes = identity_public_key.public_bytes(encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw) assert(len(identity_public_key_bytes) == 32) - assert(len(blinded_key_bytes) == 32) subcredential_bytes = stem.descriptor.hsv3_crypto.get_subcredential(identity_public_key_bytes, blinded_key_bytes)

1 0

[stem/master] Stylistic check adjustments
by atagar＠torproject.org 06 Oct '19

06 Oct '19

commit 461394f2c3527d3a510d3b5cb71f041dcba98fdf Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Sep 12 16:25:46 2019 -0700 Stylistic check adjustments Quick handful of corrections so our stylistic checks once again pass... STATIC CHECKS * /home/atagar/Desktop/stem/test/unit/descriptor/hidden_service_v3.py line 48 - use single rather than double quotes | onion_address="sltib6sxkuxh2scmtuvd5w2g7pahnzkovefxpo4e4ptnkzl5kkq5h2ad.onion")) * /home/atagar/Desktop/stem/stem/descriptor/certificate.py line 77 - use single rather than double quotes | "HS_V3_DESC_SIGNING_KEY", "HS_V3_INTRO_POINT_AUTH_KEY", "HS_V3_INTRO_POINT_ENC_KEY") line 165 - E261 at least two spaces before inline comment | elif cert_type == 8: # see rend-spec-v3.txt appendix E for these defintions line 172 - use single rather than double quotes | raise ValueError("Ed25519 certificate type is an unknown value %i." % cert_type) line 311 - W391 blank line at end of file * /home/atagar/Desktop/stem/stem/prereq.py line 142 - 'cryptography.hazmat.primitives.asymmetric.ed25519.Ed25519PublicKey' imported but unused | from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey * /home/atagar/Desktop/stem/stem/descriptor/hsv3_crypto.py line 19 - use single rather than double quotes | CHECKSUM_CONSTANT = b".onion checksum" line 21 - E302 expected 2 blank lines, found 1 | def decode_address(onion_address_str): line 38 - use single rather than double quotes | if (len(onion_address_str) != 56 + len(".onion")): line 39 - use single rather than double quotes | raise ValueError("Wrong address length") line 54 - use single rather than double quotes | my_checksum_body = b"%s%s%s" % (CHECKSUM_CONSTANT, pubkey, bytes([version])) line 58 - use single rather than double quotes | raise ValueError("Bad checksum") line 62 - E305 expected 2 blank lines after class or function definition, found 1 | """ line 81 - E302 expected 2 blank lines, found 0 | def get_subcredential(public_identity_key, blinded_key): line 82 - use single rather than double quotes | cred_bytes_constant = "credential".encode() line 83 - use single rather than double quotes | subcred_bytes_constant = "subcredential".encode() line 85 - use single rather than double quotes | credential = hashlib.sha3_256(b"%s%s" % (cred_bytes_constant, public_identity_key)).digest() line 86 - use single rather than double quotes | subcredential = hashlib.sha3_256(b"%s%s%s" % (subcred_bytes_constant, credential, blinded_key)).digest() line 88 - use single rather than double quotes | print("public_identity_key: %s" % (public_identity_key.hex())) line 89 - use single rather than double quotes | print("credential: %s" % (credential.hex())) line 90 - use single rather than double quotes | print("blinded_key: %s" % (blinded_key.hex())) line 91 - use single rather than double quotes | print("subcredential: %s" % (subcredential.hex())) line 93 - use single rather than double quotes | print("===") line 97 - E305 expected 2 blank lines after class or function definition, found 1 | """ line 131 - E302 expected 2 blank lines, found 1 | def _ciphertext_mac_is_valid(key, salt, ciphertext, mac): line 142 - use single rather than double quotes | my_mac_body = b"%s%s%s%s%s" % (key_len, key, salt_len, salt, ciphertext) line 145 - use single rather than double quotes | print("===") line 146 - use single rather than double quotes | print("my mac: %s" % my_mac.hex()) line 147 - use single rather than double quotes | print("their mac: %s" % mac.hex()) line 152 - E302 expected 2 blank lines, found 1 | def _decrypt_descriptor_layer(ciphertext_blob_b64, revision_counter, line 162 - use single rather than double quotes | raise ValueError("bad encrypted blob") line 168 - use single rather than double quotes | print("encrypted blob lenth :%s" % len(ciphertext_blob)) line 169 - use single rather than double quotes | print("salt: %s" % salt.hex()) line 170 - use single rather than double quotes | print("ciphertext length: %s" % len(ciphertext)) line 171 - use single rather than double quotes | print("mac: %s" % mac.hex()) line 172 - use single rather than double quotes | print("===") line 176 - use single rather than double quotes | secret_input = b"%s%s%s" % (secret_data, subcredential, rev_counter_int_8) line 179 - use single rather than double quotes | print("secret_data (%d): %s" % (len(secret_data), secret_data.hex())) line 180 - use single rather than double quotes | print("subcredential (%d): %s" % (len(subcredential), subcredential.hex())) line 181 - use single rather than double quotes | print("rev counter int 8 (%d): %s" % (len(rev_counter_int_8), rev_counter_int_8.hex())) line 182 - use single rather than double quotes | print("secret_input (%s): %s" % (len(secret_input), secret_input.hex())) line 183 - use single rather than double quotes | print("===") line 185 - use single rather than double quotes | kdf = hashlib.shake_256(b"%s%s%s" % (secret_input, salt, string_constant)) line 186 - E226 missing whitespace around arithmetic operator | keys = kdf.digest(S_KEY_LEN+S_IV_LEN+MAC_KEY_LEN) line 189 - E226 missing whitespace around arithmetic operator | secret_iv = keys[S_KEY_LEN:S_KEY_LEN+S_IV_LEN] line 190 - E226 missing whitespace around arithmetic operator | mac_key = keys[S_KEY_LEN+S_IV_LEN:] line 192 - use single rather than double quotes | print("secret_key: %s" % secret_key.hex()) line 193 - use single rather than double quotes | print("secret_iv: %s" % secret_iv.hex()) line 194 - use single rather than double quotes | print("mac_key: %s" % mac_key.hex()) line 203 - use single rather than double quotes | raise ValueError("Bad MAC!!!") line 207 - E302 expected 2 blank lines, found 1 | def decrypt_outter_layer(superencrypted_blob_b64, revision_counter, line 208 - E128 continuation line under-indented for visual indent | public_identity_key, blinded_key, subcredential): line 210 - use single rather than double quotes | string_constant = b"hsdir-superencrypted-data" line 218 - use single rather than double quotes | print("====== Decrypting outter layer =======") line 221 - E128 continuation line under-indented for visual indent | public_identity_key, subcredential, line 222 - E128 continuation line under-indented for visual indent | secret_data, string_constant) line 224 - E302 expected 2 blank lines, found 1 | def decrypt_inner_layer(encrypted_blob_b64, revision_counter, line 227 - use single rather than double quotes | string_constant = b"hsdir-encrypted-data" line 229 - use single rather than double quotes | print("====== Decrypting inner layer =======") line 235 - E302 expected 2 blank lines, found 1 | def parse_superencrypted_plaintext(outter_layer_plaintext): line 237 - 're' imported but unused | import re line 248 - W391 blank line at end of file * /home/atagar/Desktop/stem/stem/descriptor/hidden_service.py line 563 - use single rather than double quotes | raise ValueError("Onion address is required to decrypt v3 hidden service descriptors") line 571 - local variable 'plaintext' is assigned to but never used | plaintext = self.decrypt_descriptor(desc_signing_cert) line 587 - E266 too many leading '#' for block comment | ####################################### Do the decryption ################################### line 590 - E128 continuation line under-indented for visual indent | identity_public_key_bytes, blinded_key_bytes, subcredential_bytes) line 595 - E222 multiple spaces after operator | inner_layer_plaintext = stem.descriptor.hsv3_crypto.decrypt_inner_layer(inner_layer_ciphertext, self.revision_counter, line 596 - E128 continuation line under-indented for visual indent | identity_public_key_bytes, blinded_key_bytes, subcredential_bytes) line 602 - E305 expected 2 blank lines after class or function definition, found 1 | HiddenServiceDescriptor = HiddenServiceDescriptorV2 --- stem/descriptor/certificate.py | 18 ++-- stem/descriptor/hidden_service.py | 13 ++- stem/descriptor/hsv3_crypto.py | 133 +++++++++++++++--------------- test/settings.cfg | 1 + test/unit/descriptor/hidden_service_v3.py | 2 +- 5 files changed, 87 insertions(+), 80 deletions(-) diff --git a/stem/descriptor/certificate.py b/stem/descriptor/certificate.py index 470fa623..cdde0c24 100644 --- a/stem/descriptor/certificate.py +++ b/stem/descriptor/certificate.py @@ -73,8 +73,15 @@ ED25519_HEADER_LENGTH = 40 ED25519_SIGNATURE_LENGTH = 64 ED25519_ROUTER_SIGNATURE_PREFIX = b'Tor router descriptor signature v1' -CertType = stem.util.enum.UppercaseEnum('SIGNING', 'LINK_CERT', 'AUTH', - "HS_V3_DESC_SIGNING_KEY", "HS_V3_INTRO_POINT_AUTH_KEY", "HS_V3_INTRO_POINT_ENC_KEY") +CertType = stem.util.enum.UppercaseEnum( + 'SIGNING', + 'LINK_CERT', + 'AUTH', + 'HS_V3_DESC_SIGNING_KEY', + 'HS_V3_INTRO_POINT_AUTH_KEY', + 'HS_V3_INTRO_POINT_ENC_KEY', +) + ExtensionType = stem.util.enum.Enum(('HAS_SIGNING_KEY', 4),) ExtensionFlag = stem.util.enum.UppercaseEnum('AFFECTS_VALIDATION', 'UNKNOWN') @@ -162,14 +169,15 @@ class Ed25519CertificateV1(Ed25519Certificate): self.type = CertType.AUTH elif cert_type == 7: raise ValueError('Ed25519 certificate cannot have a type of 7. This is reserved for RSA identity cross-certification.') - elif cert_type == 8: # see rend-spec-v3.txt appendix E for these defintions + elif cert_type == 8: + # see rend-spec-v3.txt appendix E for these defintions self.type = CertType.HS_V3_DESC_SIGNING_KEY elif cert_type == 9: self.type = CertType.HS_V3_INTRO_POINT_AUTH_KEY elif cert_type == 0x0B: self.type = CertType.HS_V3_INTRO_POINT_ENC_KEY else: - raise ValueError("Ed25519 certificate type is an unknown value %i." % cert_type) + raise ValueError('Ed25519 certificate type %i is unrecognized' % cert_type) # expiration time is in hours since epoch try: @@ -307,5 +315,3 @@ class Ed25519CertificateV1(Ed25519Certificate): raise ValueError('Signing key extension has malformed key') return signing_key_extension.data - - diff --git a/stem/descriptor/hidden_service.py b/stem/descriptor/hidden_service.py index 9e6de0d9..83aff8b8 100644 --- a/stem/descriptor/hidden_service.py +++ b/stem/descriptor/hidden_service.py @@ -560,7 +560,7 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): if not skip_crypto_validation and stem.prereq.is_crypto_available(): if self.onion_address is None: - raise ValueError("Onion address is required to decrypt v3 hidden service descriptors") + raise ValueError('Onion address is required to decrypt v3 hidden service descriptors') # ATAGAR XXX need to do this cert extraction in the parsing handler assert(self.signing_cert) @@ -568,7 +568,7 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): assert(cert_lines[0] == '-----BEGIN ED25519 CERT-----' and cert_lines[-1] == '-----END ED25519 CERT-----') desc_signing_cert = stem.descriptor.certificate.Ed25519Certificate.parse(''.join(cert_lines[1:-1])) - plaintext = self.decrypt_descriptor(desc_signing_cert) + self.plaintext = self.decrypt_descriptor(desc_signing_cert) def decrypt_descriptor(self, desc_signing_cert): # Get crypto material. @@ -584,19 +584,16 @@ class HiddenServiceDescriptorV3(BaseHiddenServiceDescriptor): subcredential_bytes = stem.descriptor.hsv3_crypto.get_subcredential(identity_public_key_bytes, blinded_key_bytes) - ####################################### Do the decryption ################################### - - outter_layer_plaintext = stem.descriptor.hsv3_crypto.decrypt_outter_layer(self.superencrypted, self.revision_counter, - identity_public_key_bytes, blinded_key_bytes, subcredential_bytes) + outter_layer_plaintext = stem.descriptor.hsv3_crypto.decrypt_outter_layer(self.superencrypted, self.revision_counter, identity_public_key_bytes, blinded_key_bytes, subcredential_bytes) # ATAGAR XXX this parsing function is a hack. need to replace it with some stem parsing. inner_layer_ciphertext = stem.descriptor.hsv3_crypto.parse_superencrypted_plaintext(outter_layer_plaintext) - inner_layer_plaintext = stem.descriptor.hsv3_crypto.decrypt_inner_layer(inner_layer_ciphertext, self.revision_counter, - identity_public_key_bytes, blinded_key_bytes, subcredential_bytes) + inner_layer_plaintext = stem.descriptor.hsv3_crypto.decrypt_inner_layer(inner_layer_ciphertext, self.revision_counter, identity_public_key_bytes, blinded_key_bytes, subcredential_bytes) print(inner_layer_plaintext) + # TODO: drop this alias in stem 2.x HiddenServiceDescriptor = HiddenServiceDescriptorV2 diff --git a/stem/descriptor/hsv3_crypto.py b/stem/descriptor/hsv3_crypto.py index c3f1feb0..c83165ed 100644 --- a/stem/descriptor/hsv3_crypto.py +++ b/stem/descriptor/hsv3_crypto.py @@ -6,24 +6,25 @@ import stem.prereq """ Onion addresses - onion_address = base32(PUBKEY | CHECKSUM | VERSION) + ".onion" - CHECKSUM = H(".onion checksum" | PUBKEY | VERSION)[:2] + onion_address = base32(PUBKEY | CHECKSUM | VERSION) + '.onion' + CHECKSUM = H('.onion checksum' | PUBKEY | VERSION)[:2] - PUBKEY is the 32 bytes ed25519 master pubkey of the hidden service. - VERSION is an one byte version field (default value '\x03') - - ".onion checksum" is a constant string + - '.onion checksum' is a constant string - CHECKSUM is truncated to two bytes before inserting it in onion_address """ -CHECKSUM_CONSTANT = b".onion checksum" +CHECKSUM_CONSTANT = b'.onion checksum' + def decode_address(onion_address_str): """ Parse onion_address_str and return the pubkey. - onion_address = base32(PUBKEY | CHECKSUM | VERSION) + ".onion" - CHECKSUM = H(".onion checksum" | PUBKEY | VERSION)[:2] + onion_address = base32(PUBKEY | CHECKSUM | VERSION) + '.onion' + CHECKSUM = H('.onion checksum' | PUBKEY | VERSION)[:2] :return: Ed25519PublicKey @@ -35,8 +36,8 @@ def decode_address(onion_address_str): from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey - if (len(onion_address_str) != 56 + len(".onion")): - raise ValueError("Wrong address length") + if (len(onion_address_str) != 56 + len('.onion')): + raise ValueError('Wrong address length') # drop the '.onion' onion_address = onion_address_str[:56] @@ -51,21 +52,22 @@ def decode_address(onion_address_str): version = onion_address[34] # Do checksum validation - my_checksum_body = b"%s%s%s" % (CHECKSUM_CONSTANT, pubkey, bytes([version])) + my_checksum_body = b'%s%s%s' % (CHECKSUM_CONSTANT, pubkey, bytes([version])) my_checksum = hashlib.sha3_256(my_checksum_body).digest() if (checksum != my_checksum[:2]): - raise ValueError("Bad checksum") + raise ValueError('Bad checksum') return Ed25519PublicKey.from_public_bytes(pubkey) + """ Blinded key stuff Now wrt SRVs, if a client is in the time segment between a new time period - and a new SRV (i.e. the segments drawn with "-") it uses the current SRV, + and a new SRV (i.e. the segments drawn with '-') it uses the current SRV, else if the client is in a time segment between a new SRV and a new time - period (i.e. the segments drawn with "="), it uses the previous SRV. + period (i.e. the segments drawn with '='), it uses the previous SRV. """ pass @@ -73,27 +75,30 @@ pass """ Subcredential: - subcredential = H("subcredential" | credential | blinded-public-key - credential = H("credential" | public-identity-key) + subcredential = H('subcredential' | credential | blinded-public-key + credential = H('credential' | public-identity-key) Both keys are in bytes """ + + def get_subcredential(public_identity_key, blinded_key): - cred_bytes_constant = "credential".encode() - subcred_bytes_constant = "subcredential".encode() + cred_bytes_constant = 'credential'.encode() + subcred_bytes_constant = 'subcredential'.encode() - credential = hashlib.sha3_256(b"%s%s" % (cred_bytes_constant, public_identity_key)).digest() - subcredential = hashlib.sha3_256(b"%s%s%s" % (subcred_bytes_constant, credential, blinded_key)).digest() + credential = hashlib.sha3_256(b'%s%s' % (cred_bytes_constant, public_identity_key)).digest() + subcredential = hashlib.sha3_256(b'%s%s%s' % (subcred_bytes_constant, credential, blinded_key)).digest() - print("public_identity_key: %s" % (public_identity_key.hex())) - print("credential: %s" % (credential.hex())) - print("blinded_key: %s" % (blinded_key.hex())) - print("subcredential: %s" % (subcredential.hex())) + print('public_identity_key: %s' % (public_identity_key.hex())) + print('credential: %s' % (credential.hex())) + print('blinded_key: %s' % (blinded_key.hex())) + print('subcredential: %s' % (subcredential.hex())) - print("===") + print('===') return subcredential + """ Basic descriptor logic: @@ -114,11 +119,11 @@ Layer data: 2.5.1.1. First layer encryption logic SECRET_DATA = blinded-public-key - STRING_CONSTANT = "hsdir-superencrypted-data" + STRING_CONSTANT = 'hsdir-superencrypted-data' 2.5.2.1. Second layer encryption keys SECRET_DATA = blinded-public-key | descriptor_cookie - STRING_CONSTANT = "hsdir-encrypted-data" + STRING_CONSTANT = 'hsdir-encrypted-data' """ SALT_LEN = 16 @@ -128,6 +133,7 @@ S_KEY_LEN = 32 S_IV_LEN = 16 MAC_KEY_LEN = 32 + def _ciphertext_mac_is_valid(key, salt, ciphertext, mac): """ Instantiate MAC(key=k, message=m) with H(k_len | k | m), where k_len is @@ -135,20 +141,22 @@ def _ciphertext_mac_is_valid(key, salt, ciphertext, mac): XXX spec: H(mac_key_len | mac_key | salt_len | salt | encrypted) """ + # Construct our own MAC first key_len = len(key).to_bytes(8, 'big') salt_len = len(salt).to_bytes(8, 'big') - my_mac_body = b"%s%s%s%s%s" % (key_len, key, salt_len, salt, ciphertext) + my_mac_body = b'%s%s%s%s%s' % (key_len, key, salt_len, salt, ciphertext) my_mac = hashlib.sha3_256(my_mac_body).digest() - print("===") - print("my mac: %s" % my_mac.hex()) - print("their mac: %s" % mac.hex()) + print('===') + print('my mac: %s' % my_mac.hex()) + print('their mac: %s' % mac.hex()) # Compare the two MACs return my_mac == mac + def _decrypt_descriptor_layer(ciphertext_blob_b64, revision_counter, public_identity_key, subcredential, secret_data, string_constant): @@ -159,39 +167,39 @@ def _decrypt_descriptor_layer(ciphertext_blob_b64, revision_counter, ciphertext_blob = base64.b64decode(ciphertext_blob_b64) if (len(ciphertext_blob) < SALT_LEN + MAC_LEN): - raise ValueError("bad encrypted blob") + raise ValueError('bad encrypted blob') salt = ciphertext_blob[:16] ciphertext = ciphertext_blob[16:-32] mac = ciphertext_blob[-32:] - print("encrypted blob lenth :%s" % len(ciphertext_blob)) - print("salt: %s" % salt.hex()) - print("ciphertext length: %s" % len(ciphertext)) - print("mac: %s" % mac.hex()) - print("===") + print('encrypted blob lenth :%s' % len(ciphertext_blob)) + print('salt: %s' % salt.hex()) + print('ciphertext length: %s' % len(ciphertext)) + print('mac: %s' % mac.hex()) + print('===') # INT_8(revision_counter) rev_counter_int_8 = revision_counter.to_bytes(8, 'big') - secret_input = b"%s%s%s" % (secret_data, subcredential, rev_counter_int_8) + secret_input = b'%s%s%s' % (secret_data, subcredential, rev_counter_int_8) secret_input = secret_input - print("secret_data (%d): %s" % (len(secret_data), secret_data.hex())) - print("subcredential (%d): %s" % (len(subcredential), subcredential.hex())) - print("rev counter int 8 (%d): %s" % (len(rev_counter_int_8), rev_counter_int_8.hex())) - print("secret_input (%s): %s" % (len(secret_input), secret_input.hex())) - print("===") + print('secret_data (%d): %s' % (len(secret_data), secret_data.hex())) + print('subcredential (%d): %s' % (len(subcredential), subcredential.hex())) + print('rev counter int 8 (%d): %s' % (len(rev_counter_int_8), rev_counter_int_8.hex())) + print('secret_input (%s): %s' % (len(secret_input), secret_input.hex())) + print('===') - kdf = hashlib.shake_256(b"%s%s%s" % (secret_input, salt, string_constant)) - keys = kdf.digest(S_KEY_LEN+S_IV_LEN+MAC_KEY_LEN) + kdf = hashlib.shake_256(b'%s%s%s' % (secret_input, salt, string_constant)) + keys = kdf.digest(S_KEY_LEN + S_IV_LEN + MAC_KEY_LEN) secret_key = keys[:S_KEY_LEN] - secret_iv = keys[S_KEY_LEN:S_KEY_LEN+S_IV_LEN] - mac_key = keys[S_KEY_LEN+S_IV_LEN:] + secret_iv = keys[S_KEY_LEN:S_KEY_LEN + S_IV_LEN] + mac_key = keys[S_KEY_LEN + S_IV_LEN:] - print("secret_key: %s" % secret_key.hex()) - print("secret_iv: %s" % secret_iv.hex()) - print("mac_key: %s" % mac_key.hex()) + print('secret_key: %s' % secret_key.hex()) + print('secret_iv: %s' % secret_iv.hex()) + print('mac_key: %s' % mac_key.hex()) # Now time to decrypt descriptor cipher = Cipher(algorithms.AES(secret_key), modes.CTR(secret_iv), default_backend()) @@ -200,14 +208,14 @@ def _decrypt_descriptor_layer(ciphertext_blob_b64, revision_counter, # validate mac (the mac validates the two fields before the mac) if not _ciphertext_mac_is_valid(mac_key, salt, ciphertext, mac): - raise ValueError("Bad MAC!!!") + raise ValueError('Bad MAC!!!') return decrypted -def decrypt_outter_layer(superencrypted_blob_b64, revision_counter, - public_identity_key, blinded_key, subcredential): + +def decrypt_outter_layer(superencrypted_blob_b64, revision_counter, public_identity_key, blinded_key, subcredential): secret_data = blinded_key - string_constant = b"hsdir-superencrypted-data" + string_constant = b'hsdir-superencrypted-data' # XXX Remove the BEGIN MESSSAGE around the thing superencrypted_blob_b64_lines = superencrypted_blob_b64.split('\n') @@ -215,26 +223,22 @@ def decrypt_outter_layer(superencrypted_blob_b64, revision_counter, assert(superencrypted_blob_b64_lines[-1] == '-----END MESSAGE-----') superencrypted_blob_b64 = ''.join(superencrypted_blob_b64_lines[1:-1]) - print("====== Decrypting outter layer =======") + print('====== Decrypting outter layer =======') - return _decrypt_descriptor_layer(superencrypted_blob_b64, revision_counter, - public_identity_key, subcredential, - secret_data, string_constant) + return _decrypt_descriptor_layer(superencrypted_blob_b64, revision_counter, public_identity_key, subcredential, secret_data, string_constant) -def decrypt_inner_layer(encrypted_blob_b64, revision_counter, - public_identity_key, blinded_key, subcredential): + +def decrypt_inner_layer(encrypted_blob_b64, revision_counter, public_identity_key, blinded_key, subcredential): secret_data = blinded_key - string_constant = b"hsdir-encrypted-data" + string_constant = b'hsdir-encrypted-data' + + print('====== Decrypting inner layer =======') - print("====== Decrypting inner layer =======") + return _decrypt_descriptor_layer(encrypted_blob_b64, revision_counter, public_identity_key, subcredential, secret_data, string_constant) - return _decrypt_descriptor_layer(encrypted_blob_b64, revision_counter, - public_identity_key, subcredential, - secret_data, string_constant) def parse_superencrypted_plaintext(outter_layer_plaintext): """Super hacky function to parse the superencrypted plaintext. This will need to be replaced by proper stem code.""" - import re START_CONSTANT = b'-----BEGIN MESSAGE-----\n' END_CONSTANT = b'\n-----END MESSAGE-----' @@ -245,4 +249,3 @@ def parse_superencrypted_plaintext(outter_layer_plaintext): start = start + len(START_CONSTANT) return outter_layer_plaintext[start:end] - diff --git a/test/settings.cfg b/test/settings.cfg index 1b971bb4..7e42b3af 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -207,6 +207,7 @@ pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.serialization. pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.modes' imported but unused pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.Cipher' imported but unused pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.algorithms' imported but unused +pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.asymmetric.ed25519.Ed25519PublicKey' imported but unused pyflakes.ignore stem/prereq.py => 'lzma' imported but unused pyflakes.ignore stem/client/datatype.py => redefinition of unused 'pop' from * pyflakes.ignore stem/descriptor/hidden_service_descriptor.py => 'stem.descriptor.hidden_service.*' imported but unused diff --git a/test/unit/descriptor/hidden_service_v3.py b/test/unit/descriptor/hidden_service_v3.py index 4fdc72bd..98de246e 100644 --- a/test/unit/descriptor/hidden_service_v3.py +++ b/test/unit/descriptor/hidden_service_v3.py @@ -45,7 +45,7 @@ class TestHiddenServiceDescriptorV3(unittest.TestCase): with open(get_resource('hidden_service_v3_test'), 'rb') as descriptor_file: desc = next(stem.descriptor.parse_file(descriptor_file, 'hidden-service-descriptor-3 1.0', validate = True, - onion_address="sltib6sxkuxh2scmtuvd5w2g7pahnzkovefxpo4e4ptnkzl5kkq5h2ad.onion")) + onion_address = 'sltib6sxkuxh2scmtuvd5w2g7pahnzkovefxpo4e4ptnkzl5kkq5h2ad.onion')) self.assertEqual(3, desc.version) self.assertEqual(180, desc.lifetime)

1 0

[stem/master] SHA3 requires python 3.6 or pysha3
by atagar＠torproject.org 06 Oct '19

06 Oct '19

commit 9e3d08e9b64d73c38c4e3d0735b739d78eaa0059 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 22 14:39:48 2019 -0700 SHA3 requires python 3.6 or pysha3 Hashlib's sha3_256() method was only recently added in python 3.6... https://docs.python.org/3/library/hashlib.html ====================================================================== ERROR: test_for_decrypt ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/atagar/Desktop/stem/test/unit/descriptor/hidden_service_v3.py", line 53, in test_for_decrypt desc._decrypt('sltib6sxkuxh2scmtuvd5w2g7pahnzkovefxpo4e4ptnkzl5kkq5h2ad.onion') File "/home/atagar/Desktop/stem/stem/descriptor/hidden_service.py", line 576, in _decrypt identity_public_key = stem.descriptor.hsv3_crypto.decode_address(onion_address) File "/home/atagar/Desktop/stem/stem/descriptor/hsv3_crypto.py", line 55, in decode_address my_checksum = hashlib.sha3_256(my_checksum_body).digest() AttributeError: 'module' object has no attribute 'sha3_256' ---------------------------------------------------------------------- If unavailable then falling back to pysha3, and if that's unavailable as well we'll need to raise an ImportError. --- stem/descriptor/hsv3_crypto.py | 27 +++++++++++++++++++++++++++ test/unit/descriptor/hidden_service_v3.py | 4 ++++ 2 files changed, 31 insertions(+) diff --git a/stem/descriptor/hsv3_crypto.py b/stem/descriptor/hsv3_crypto.py index 2e5288a1..a12ce52b 100644 --- a/stem/descriptor/hsv3_crypto.py +++ b/stem/descriptor/hsv3_crypto.py @@ -3,6 +3,22 @@ import hashlib import stem.prereq +# SHA3 requires Python 3.6+ *or* the pysha3 module... +# +# https://github.com/tiran/pysha3 +# +# If pysha3 is present then importing sha3 will monkey patch the methods we +# want onto hashlib. + +if not hasattr(hashlib, 'sha3_256') or not hasattr(hashlib, 'shake_256'): + try: + import sha3 + except ImportError: + pass + +SHA3_AVAILABLE = hasattr(hashlib, 'sha3_256') and hasattr(hashlib, 'shake_256') +SHA3_ERROR_MSG = '%s requires python 3.6+ or the pysha3 module (https://pypi.org/project/pysha3/)' + """ Onion addresses @@ -32,6 +48,8 @@ def decode_address(onion_address_str): if not stem.prereq.is_crypto_available(ed25519 = True): raise ImportError('Onion address decoding requires cryptography version 2.6') + elif not SHA3_AVAILABLE: + raise ImportError(SHA3_ERROR_MSG % 'Onion address decoding') from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey @@ -82,6 +100,9 @@ Both keys are in bytes def get_subcredential(public_identity_key, blinded_key): + if not SHA3_AVAILABLE: + raise ImportError(SHA3_ERROR_MSG % 'Hidden service subcredentials') + cred_bytes_constant = 'credential'.encode() subcred_bytes_constant = 'subcredential'.encode() @@ -141,6 +162,9 @@ def _ciphertext_mac_is_valid(key, salt, ciphertext, mac): XXX spec: H(mac_key_len | mac_key | salt_len | salt | encrypted) """ + if not SHA3_AVAILABLE: + raise ImportError(SHA3_ERROR_MSG % 'Hidden service validation') + # Construct our own MAC first key_len = len(key).to_bytes(8, 'big') salt_len = len(salt).to_bytes(8, 'big') @@ -157,6 +181,9 @@ def _ciphertext_mac_is_valid(key, salt, ciphertext, mac): def _decrypt_descriptor_layer(ciphertext_blob_b64, revision_counter, public_identity_key, subcredential, secret_data, string_constant): + if not SHA3_AVAILABLE: + raise ImportError(SHA3_ERROR_MSG % 'Hidden service descriptor decryption') + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.backends import default_backend diff --git a/test/unit/descriptor/hidden_service_v3.py b/test/unit/descriptor/hidden_service_v3.py index ac2f8ea7..d800b46f 100644 --- a/test/unit/descriptor/hidden_service_v3.py +++ b/test/unit/descriptor/hidden_service_v3.py @@ -6,6 +6,7 @@ import functools import unittest import stem.descriptor +import stem.descriptor.hsv3_crypto import stem.prereq from stem.descriptor.hidden_service import ( @@ -42,6 +43,9 @@ class TestHiddenServiceDescriptorV3(unittest.TestCase): if not stem.prereq.is_crypto_available(ed25519 = True): self.skipTest('(requires cryptography v2.6)') return + elif not stem.descriptor.hsv3_crypto.SHA3_AVAILABLE: + self.skipTest('(requires python 3.6 or pysha3)') + return with open(get_resource('hidden_service_v3_test'), 'rb') as descriptor_file: desc = next(stem.descriptor.parse_file(descriptor_file, 'hidden-service-descriptor-3 1.0', validate = True))

1 0

[stem/master] Adjust certificate validate() TODO comment
by atagar＠torproject.org 06 Oct '19

06 Oct '19

commit 22724cec45e0a2819be66a0c0def6da43d304d3b Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 22 13:42:59 2019 -0700 Adjust certificate validate() TODO comment George is absolutely right that this method is too specific to server descriptors, but I cannot find the generic "validate()" method he's referring to. Just reached out to him on irc to discuss this. If we have a generic method that applies to both server and hidden service descriptors I'd be delighted to deprecate this, but for the moment at least I'm unsure how to proceed with this. --- stem/descriptor/certificate.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/certificate.py b/stem/descriptor/certificate.py index f1f7483e..2293a7a3 100644 --- a/stem/descriptor/certificate.py +++ b/stem/descriptor/certificate.py @@ -241,9 +241,10 @@ class Ed25519CertificateV1(Ed25519Certificate): return datetime.datetime.now() > self.expiration - # ATAGAR XXX certificates are generic and not just for descriptor, however - # this function assumes they are. this needs to be moved to the descriptor - # module. the new verify() function is more generic and should be used. + # TODO: This method is too specific to server descriptors. We should + # deprecate this if possible in favor of a more generic method that covers + # hidden service v3 descriptors as well. + def validate(self, server_descriptor): """ Validates our signing key and that the given descriptor content matches its

1 0

[stem/master] Use struct for integer => byte conversions
by atagar＠torproject.org 06 Oct '19

06 Oct '19

commit ba630119da869bf65f9c6f8bd8443f3041b7228c Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 22 17:11:55 2019 -0700 Use struct for integer => byte conversions Neat! I didn't realize python 3.x added a to_bytes() method to the integer class. We can't use this just yet, but when we drop python 2.x support we will. --- stem/client/datatype.py | 9 +++++++++ stem/descriptor/hsv3_crypto.py | 8 +++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/stem/client/datatype.py b/stem/client/datatype.py index 59a0c167..fa393d30 100644 --- a/stem/client/datatype.py +++ b/stem/client/datatype.py @@ -381,6 +381,15 @@ class Size(Field): elif content < 0: raise ValueError('Packed values must be positive (attempted to pack %i as a %s)' % (content, self.name)) + # TODO: When we drop python 2.x support this can be simplified via + # integer's to_bytes() method. For example... + # + # struct.pack('>Q', my_number) + # + # ... is the same as... + # + # my_number.to_bytes(8, 'big') + try: packed = struct.pack(self.format, content) except struct.error: diff --git a/stem/descriptor/hsv3_crypto.py b/stem/descriptor/hsv3_crypto.py index 7c641363..1da07b1b 100644 --- a/stem/descriptor/hsv3_crypto.py +++ b/stem/descriptor/hsv3_crypto.py @@ -1,6 +1,8 @@ import base64 import binascii import hashlib +import struct + import stem.prereq # SHA3 requires Python 3.6+ *or* the pysha3 module... @@ -166,8 +168,8 @@ def _ciphertext_mac_is_valid(key, salt, ciphertext, mac): raise ImportError(SHA3_ERROR_MSG % 'Hidden service validation') # Construct our own MAC first - key_len = len(key).to_bytes(8, 'big') - salt_len = len(salt).to_bytes(8, 'big') + key_len = struct.pack('>Q', len(key)) + salt_len = struct.pack('>Q', len(salt)) my_mac_body = b'%s%s%s%s%s' % (key_len, key, salt_len, salt, ciphertext) my_mac = hashlib.sha3_256(my_mac_body).digest() @@ -204,7 +206,7 @@ def _decrypt_descriptor_layer(ciphertext_blob_b64, revision_counter, public_iden print('===') # INT_8(revision_counter) - rev_counter_int_8 = revision_counter.to_bytes(8, 'big') + rev_counter_int_8 = struct.pack('>Q', revision_counter) secret_input = b'%s%s%s' % (secret_data, subcredential, rev_counter_int_8) secret_input = secret_input

1 0