June 2017 - tor-commits - lists.torproject.org

[stem/master] Move sig header/footer into _generate_signature()
by atagar＠torproject.org 20 Jun '17

20 Jun '17

commit 7a3031f2e4d0922d497ad5ce693de1cfb0daf157 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jun 19 12:15:23 2017 -0700 Move sig header/footer into _generate_signature() Minor simplification to make this cleaner. --- stem/descriptor/server_descriptor.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 03008ac..2ff42e7 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -245,7 +245,9 @@ def _generate_signing_key(): def _generate_signature(content, signing_key): """ - Creates the 'router-signature' for the given descriptor content. + Creates the 'router-signature' signature block (excluding the + 'router-signature\n' prefix since that should be part of the + signed content). """ from cryptography.hazmat.primitives import hashes @@ -257,7 +259,8 @@ def _generate_signature(content, signing_key): digest = DIGEST_TYPE_INFO + (DIGEST_PADDING * (125 - len(digest))) + DIGEST_SEPARATOR + digest padding = padding.PSS(mgf = padding.MGF1(hashes.SHA256()), salt_length = padding.PSS.MAX_LENGTH) - return base64.b64encode(signing_key.private.sign(digest, padding, hashes.SHA256())) + signature = base64.b64encode(signing_key.private.sign(digest, padding, hashes.SHA256())) + return '-----BEGIN SIGNATURE-----\n' + '\n'.join(stem.util.str_tools._split_by_length(signature, 64)) + '\n-----END SIGNATURE-----\n' def _parse_router_line(descriptor, entries): @@ -878,17 +881,14 @@ class RelayDescriptor(ServerDescriptor): if attr is None: attr = {} - # create descriptor content without the router-signature line, then - # appending the signature + # create descriptor content without the router-signature, then + # appending the content signature signing_key = _generate_signing_key() attr['signing-key'] = signing_key.descriptor_signing_key content = _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER) + '\nrouter-signature\n' - signature = _generate_signature(content, signing_key) - content = '\n'.join([content + '-----BEGIN SIGNATURE-----'] + stem.util.str_tools._split_by_length(signature, 64) + ['-----END SIGNATURE-----']) + '\n' - - return content + return content + _generate_signature(content, signing_key) else: return _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER, RELAY_SERVER_FOOTER)

1 0

[stem/master] Move digest contants into base descriptor module
by atagar＠torproject.org 20 Jun '17

20 Jun '17

commit 10b6ce3fc5b129dbe30714ba1c63f2b2e190bb7f Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jun 19 12:31:00 2017 -0700 Move digest contants into base descriptor module Ah. On reflection we already used these a little for signature validation. --- stem/descriptor/__init__.py | 8 ++++++-- stem/descriptor/server_descriptor.py | 7 +++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 82b71d5..e18dfd2 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -81,6 +81,10 @@ PGP_BLOCK_START = re.compile('^-----BEGIN ([%s%s]+)-----$' % (KEYWORD_CHAR, WHIT PGP_BLOCK_END = '-----END %s-----' EMPTY_COLLECTION = ([], {}, set()) +DIGEST_TYPE_INFO = b'\x00\x01' +DIGEST_PADDING = b'\xFF' +DIGEST_SEPARATOR = b'\x00' + CRYPTO_BLOB = """ MIGJAoGBAJv5IIWQ+WDWYUdyA/0L8qbIkEVH/cwryZWoIaPAzINfrw1WfNZGtBmg skFtXhOHHqTRN4GPPrZsAIUOQGzQtGb66IQgT4tO/pj+P6QmSCCdTfhvGfgTCsC+ @@ -720,7 +724,7 @@ class Descriptor(object): ############################################################################ try: - if decrypted_bytes.index(b'\x00\x01') != 0: + if decrypted_bytes.index(DIGEST_TYPE_INFO) != 0: raise ValueError('Verification failed, identifier missing') except ValueError: raise ValueError('Verification failed, malformed data') @@ -729,7 +733,7 @@ class Descriptor(object): identifier_offset = 2 # find the separator - seperator_index = decrypted_bytes.index(b'\x00', identifier_offset) + seperator_index = decrypted_bytes.index(DIGEST_SEPARATOR, identifier_offset) except ValueError: raise ValueError('Verification failed, seperator not found') diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 2ff42e7..30599a8 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -52,6 +52,9 @@ from stem.util import str_type from stem.descriptor import ( CRYPTO_BLOB, PGP_BLOCK_END, + DIGEST_TYPE_INFO, + DIGEST_PADDING, + DIGEST_SEPARATOR, Descriptor, _descriptor_content, _descriptor_components, @@ -76,10 +79,6 @@ except ImportError: SigningKey = collections.namedtuple('SigningKey', ['public', 'private', 'descriptor_signing_key']) -DIGEST_TYPE_INFO = b'\x00\x01' -DIGEST_PADDING = b'\xFF' -DIGEST_SEPARATOR = b'\x00' - # relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( 'router',

1 0

[stem/master] Digest hashed twice
by atagar＠torproject.org 20 Jun '17

20 Jun '17

commit 7f6bbf7398fe1a9f8df00721fd09b7918e7be578 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jun 19 13:36:29 2017 -0700 Digest hashed twice Oops, we generated the sha1 hash of our digest but then cryptography it's hashed with sha1. Probably means we hash this twice. --- stem/descriptor/__init__.py | 2 +- stem/descriptor/server_descriptor.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index e18dfd2..be28c7a 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -374,7 +374,7 @@ def _descriptor_content(attr = None, exclude = (), sign = False, header_template :param tuple header_template: key/value pairs for mandatory fields before unrecognized content :param tuple footer_template: key/value pairs for mandatory fields after unrecognized content - :returns: str with the requested descriptor content + :returns: bytes with the requested descriptor content :raises: **ImportError** if cryptography is unavailable and sign is True """ diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index a0759b1..8204c44 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -249,8 +249,7 @@ def _generate_signature(content, signing_key): from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import padding - digest = hashlib.sha1(content).hexdigest().decode('hex_codec') - signature = base64.b64encode(signing_key.private.sign(digest, padding.PKCS1v15(), hashes.SHA1())) + signature = base64.b64encode(signing_key.private.sign(content, padding.PKCS1v15(), hashes.SHA1())) return '-----BEGIN SIGNATURE-----\n' + '\n'.join(stem.util.str_tools._split_by_length(signature, 64)) + '\n-----END SIGNATURE-----\n'

1 0

[stem/master] Merge 'generate' helpers into descriptor creation
by atagar＠torproject.org 20 Jun '17

20 Jun '17

commit 8129f9fa4f624b10ecdec4c47b0ec9e486597708 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jun 19 15:10:10 2017 -0700 Merge 'generate' helpers into descriptor creation On reflection these helpers aren't adding much right now. We might twiddle this more as we sign additional descriptor types but this approach lets the caller provide their own signing key. --- stem/descriptor/server_descriptor.py | 69 +++++++++++------------------------- 1 file changed, 21 insertions(+), 48 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 8204c44..3ad019a 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -33,7 +33,6 @@ etc). This information is provided from a few sources... import base64 import binascii -import collections import functools import hashlib import re @@ -74,8 +73,6 @@ try: except ImportError: from stem.util.lru_cache import lru_cache -SigningKey = collections.namedtuple('SigningKey', ['public', 'private', 'descriptor_signing_key']) - # relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( 'router', @@ -214,45 +211,6 @@ def _parse_file(descriptor_file, is_bridge = False, validate = False, **kwargs): break # done parsing descriptors -def _generate_signing_key(): - """ - Creates a key that can be used to sign server descriptors. - """ - - from cryptography.hazmat.backends import default_backend - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.primitives.asymmetric import rsa - - private_key = rsa.generate_private_key( - public_exponent = 65537, - key_size = 1024, - backend = default_backend(), - ) - - public_key = private_key.public_key() - - pem = public_key.public_bytes( - encoding = serialization.Encoding.PEM, - format = serialization.PublicFormat.PKCS1, - ).strip() - - return SigningKey(public_key, private_key, pem) - - -def _generate_signature(content, signing_key): - """ - Creates the 'router-signature' signature block (excluding the - 'router-signature\n' prefix since that should be part of the - signed content). - """ - - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.asymmetric import padding - - signature = base64.b64encode(signing_key.private.sign(content, padding.PKCS1v15(), hashes.SHA1())) - return '-----BEGIN SIGNATURE-----\n' + '\n'.join(stem.util.str_tools._split_by_length(signature, 64)) + '\n-----END SIGNATURE-----\n' - - def _parse_router_line(descriptor, entries): # "router" nickname address ORPort SocksPort DirPort @@ -859,7 +817,7 @@ class RelayDescriptor(ServerDescriptor): self.certificate.validate(self) @classmethod - def content(cls, attr = None, exclude = (), sign = False): + def content(cls, attr = None, exclude = (), sign = False, private_signing_key = None): if sign: if not stem.prereq.is_crypto_available(): raise ImportError('Signing requires the cryptography module') @@ -868,23 +826,38 @@ class RelayDescriptor(ServerDescriptor): elif attr and 'router-signature' in attr: raise ValueError('Cannot sign the descriptor if a router-signature has been provided') + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import padding, rsa + if attr is None: attr = {} + if private_signing_key is None: + private_signing_key = rsa.generate_private_key( + public_exponent = 65537, + key_size = 1024, + backend = default_backend(), + ) + # create descriptor content without the router-signature, then # appending the content signature - signing_key = _generate_signing_key() - attr['signing-key'] = '\n' + signing_key.descriptor_signing_key + attr['signing-key'] = '\n' + private_signing_key.public_key().public_bytes( + encoding = serialization.Encoding.PEM, + format = serialization.PublicFormat.PKCS1, + ).strip() + content = _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER) + '\nrouter-signature\n' + signature = base64.b64encode(private_signing_key.sign(content, padding.PKCS1v15(), hashes.SHA1())) - return content + _generate_signature(content, signing_key) + return content + '\n'.join(['-----BEGIN SIGNATURE-----'] + stem.util.str_tools._split_by_length(signature, 64) + ['-----END SIGNATURE-----\n']) else: return _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER, RELAY_SERVER_FOOTER) @classmethod - def create(cls, attr = None, exclude = (), validate = True, sign = False): - return cls(cls.content(attr, exclude, sign), validate = validate, skip_crypto_validation = not sign) + def create(cls, attr = None, exclude = (), validate = True, sign = False, private_signing_key = None): + return cls(cls.content(attr, exclude, sign, private_signing_key), validate = validate, skip_crypto_validation = not sign) @lru_cache() def digest(self):

1 0

[stem/master] Support signing server descriptors
by atagar＠torproject.org 20 Jun '17

20 Jun '17

commit 9a947c6f9aa7951f977dca2731353d564cb72381 Merge: 496d190 a41767c Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jun 20 09:03:46 2017 -0700 Support signing server descriptors Fuck yeah, got it! Signing server descriptors we create. Leekspin uses PyCrypto which is deprecated [1], so wasn't able to take advantage of it as much as I hoped. On the upside cryptography is simpler than what Isis had. There was one rough bit though - cryptography embeds a constant indicating the hashing algorithm it signs with. This required us to hack out part of its internals. Hopefully upstream is amenable to adding a flag for this. Thus far we only sign server descriptors. Gonna follow this up with other descriptor types Leekspin supports so we can migrate BridgeDB's tests to use this. [1] https://github.com/dlitz/pycrypto/issues/173 stem/descriptor/__init__.py | 32 +++++++++++---- stem/descriptor/extrainfo_descriptor.py | 8 ++-- stem/descriptor/hidden_service_descriptor.py | 8 ++-- stem/descriptor/microdescriptor.py | 4 +- stem/descriptor/networkstatus.py | 24 ++++++------ stem/descriptor/router_status_entry.py | 12 +++--- stem/descriptor/server_descriptor.py | 58 +++++++++++++++++++++++++--- stem/util/str_tools.py | 18 +++++++++ test/unit/descriptor/server_descriptor.py | 5 +++ test/unit/doctest.py | 1 + test/unit/util/str_tools.py | 8 ++++ 11 files changed, 136 insertions(+), 42 deletions(-)

1 0

[stem/master] Generate signature with PKCS1 padding
by atagar＠torproject.org 20 Jun '17

20 Jun '17

commit f8f434d5cc65460de98c41672476c4b2b6707bc0 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jun 19 13:12:20 2017 -0700 Generate signature with PKCS1 padding Blindly followed the cryptography module's example to start with but turns out it does PKCS1 padding for us. This gets us further with validation but still not working just yet. Oh, and also lets us drop our manual PKCS1 padding. --- stem/descriptor/server_descriptor.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 30599a8..a0759b1 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -52,9 +52,6 @@ from stem.util import str_type from stem.descriptor import ( CRYPTO_BLOB, PGP_BLOCK_END, - DIGEST_TYPE_INFO, - DIGEST_PADDING, - DIGEST_SEPARATOR, Descriptor, _descriptor_content, _descriptor_components, @@ -234,7 +231,7 @@ def _generate_signing_key(): public_key = private_key.public_key() - pem = '\n' + public_key.public_bytes( + pem = public_key.public_bytes( encoding = serialization.Encoding.PEM, format = serialization.PublicFormat.PKCS1, ).strip() @@ -252,13 +249,8 @@ def _generate_signature(content, signing_key): from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import padding - # generate the digest with required PKCS1 padding so it's 128 bytes - digest = hashlib.sha1(content).hexdigest().decode('hex_codec') - digest = DIGEST_TYPE_INFO + (DIGEST_PADDING * (125 - len(digest))) + DIGEST_SEPARATOR + digest - - padding = padding.PSS(mgf = padding.MGF1(hashes.SHA256()), salt_length = padding.PSS.MAX_LENGTH) - signature = base64.b64encode(signing_key.private.sign(digest, padding, hashes.SHA256())) + signature = base64.b64encode(signing_key.private.sign(digest, padding.PKCS1v15(), hashes.SHA1())) return '-----BEGIN SIGNATURE-----\n' + '\n'.join(stem.util.str_tools._split_by_length(signature, 64)) + '\n-----END SIGNATURE-----\n' @@ -884,7 +876,7 @@ class RelayDescriptor(ServerDescriptor): # appending the content signature signing_key = _generate_signing_key() - attr['signing-key'] = signing_key.descriptor_signing_key + attr['signing-key'] = '\n' + signing_key.descriptor_signing_key content = _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER) + '\nrouter-signature\n' return content + _generate_signature(content, signing_key)

1 0

[stem/master] Exclude hash constant from signatures
by atagar＠torproject.org 20 Jun '17

20 Jun '17

commit a41767c7363b2164cad7d666a927ff8eead75799 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jun 20 08:54:57 2017 -0700 Exclude hash constant from signatures Holy crap that was a pita. Oh well, got it. Python's cryptography module appends a constant indicating the hashing algorithm it uses whereas tor doesn't. This caused validation of descriptor signatures we add to fail with... ====================================================================== ERROR: test_descriptor_signing ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/atagar/Desktop/stem/test/require.py", line 58, in wrapped return func(self, *args, **kwargs) File "/home/atagar/Desktop/stem/test/unit/descriptor/server_descriptor.py", line 260, in test_descriptor_signing RelayDescriptor.create(sign = True) File "/home/atagar/Desktop/stem/stem/descriptor/server_descriptor.py", line 860, in create return cls(cls.content(attr, exclude, sign, private_signing_key), validate = validate, skip_crypto_validation = not sign) File "/home/atagar/Desktop/stem/stem/descriptor/server_descriptor.py", line 808, in __init__ raise ValueError('Decrypted digest does not match local digest (calculated: %s, local: %s)' % (signed_digest, self.digest())) ValueError: Decrypted digest does not match local digest (calculated: 3021300906052B0E03021A05000414FDE1A32B71718BFEBD610E76B6340EB4795A554A, local: FDE1A32B71718BFEBD610E76B6340EB4795A554A) ---------------------------------------------------------------------- Note that they *do* actually match. Only trouble is that the former is prefixed by the following constant... 3021300906052B0E03021A05000414 Took a full day of head scratching but finally got a hint about what this is thanks to... https://crypto.stackexchange.com/questions/8780/verify-signature-failed Cryptography embeds a constant indicating the hash algorithm used (in this case sha1). Unfortunately this is baked pretty deep so the only way to exclude this is mocking out part of its internals. I'll shoot upstream a request in a bit to allow this to be toggled. --- stem/descriptor/server_descriptor.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 3ad019a..499b53e 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -840,6 +840,17 @@ class RelayDescriptor(ServerDescriptor): backend = default_backend(), ) + # When signing the cryptography module includes a constant indicating + # the hash algorithm used. Tor doesn't. This causes signature + # validation failures and unfortunately cryptography have no nice way + # of excluding these so we need to mock out part of their internals... + # ewww. + + no_op = lambda *args, **kwargs: None + + private_signing_key._backend._lib.EVP_PKEY_CTX_set_signature_md = no_op + private_signing_key._backend.openssl_assert = no_op + # create descriptor content without the router-signature, then # appending the content signature

1 0

[tor/maint-0.3.1] mingw/windows printf lacks %zd ; use %lu and casts instead
by nickm＠torproject.org 20 Jun '17

20 Jun '17

commit 945256188ab3a9b5842455f714d51e1871b0bf69 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 20 12:12:55 2017 -0400 mingw/windows printf lacks %zd ; use %lu and casts instead (This approach can lose accuracy, but it's only in debug-level messages.) Fixes windows compilation. Bugfix on recent compress.c changes; bug not in any released Tor. --- src/common/compress.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index e65894d..c16d1b5 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -105,8 +105,8 @@ tor_compress_impl(int compress, if (stream == NULL) { log_warn(LD_GENERAL, "NULL stream while %scompressing", compress?"":"de"); - log_debug(LD_GENERAL, "method: %d level: %d at len: %zd", - method, compression_level, in_len); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); return -1; } @@ -146,15 +146,15 @@ tor_compress_impl(int compress, "Unexpected %s while %scompressing", complete_only?"end of input":"result", compress?"":"de"); - log_debug(LD_GENERAL, "method: %d level: %d at len: %zd", - method, compression_level, in_len); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); goto err; } else { if (in_len != 0) { log_fn(protocol_warn_level, LD_PROTOCOL, "Unexpected extra input while decompressing"); - log_debug(LD_GENERAL, "method: %d level: %d at len: %zd", - method, compression_level, in_len); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); goto err; } else { goto done;

1 0

[goptlib/master] Remove trailing dots from URLs.
by dcf＠torproject.org 20 Jun '17

20 Jun '17

commit eac6ab5a2fa3304695ab26451b8af8f680ddc08a Author: David Fifield <david(a)bamsoftware.com> Date: Tue Jun 20 07:54:20 2017 -0700 Remove trailing dots from URLs. --- pt.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pt.go b/pt.go index 3c018a6..f3458ec 100644 --- a/pt.go +++ b/pt.go @@ -120,10 +120,10 @@ // https://spec.torproject.org/pt-spec // // Extended ORPort: -// https://gitweb.torproject.org/torspec.git/tree/proposals/196-transport-cont…. +// https://gitweb.torproject.org/torspec.git/tree/proposals/196-transport-cont… // // Extended ORPort Authentication: -// https://gitweb.torproject.org/torspec.git/tree/proposals/217-ext-orport-aut…. +// https://gitweb.torproject.org/torspec.git/tree/proposals/217-ext-orport-aut… // // Pluggable Transport through SOCKS proxy: // https://gitweb.torproject.org/torspec.git/tree/proposals/232-pluggable-tran…

1 0

[goptlib/master] Fix a reference to prop 217.
by dcf＠torproject.org 20 Jun '17

20 Jun '17

commit 20b26079e3832b76b034fa6afe0b31603dd2052e Author: David Fifield <david(a)bamsoftware.com> Date: Tue Jun 20 08:04:43 2017 -0700 Fix a reference to prop 217. --- pt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pt.go b/pt.go index ae78f3d..f910d55 100644 --- a/pt.go +++ b/pt.go @@ -616,7 +616,7 @@ func readAuthCookie(f io.Reader) ([]byte, error) { } // Read and validate the contents of an auth cookie file. Returns the 32-byte -// cookie. See section 4.2.1.2 of pt-spec.txt. +// cookie. See section 4.2.1.2 of 217-ext-orport-auth.txt. func readAuthCookieFile(filename string) ([]byte, error) { f, err := os.Open(filename) if err != nil {

1 0