commit db5bfa4f574764fffe8610eace05cb3dea51ac36 Author: Damian Johnson atagar@torproject.org Date: Sat Feb 28 20:52:17 2015 -0800

Verifying hidden service descriptor signatures

This is identical to how server descriptors are validated, so taking this opportunity to tidy that up a bit in the process. --- stem/descriptor/__init__.py | 112 +++++++++++++++++ stem/descriptor/hidden_service_descriptor.py | 28 +++-- stem/descriptor/server_descriptor.py | 138 ++------------------- test/mocking.py | 5 +- test/unit/descriptor/hidden_service_descriptor.py | 6 +- test/unit/descriptor/server_descriptor.py | 4 +- test/unit/tutorial.py | 2 +- 7 files changed, 150 insertions(+), 145 deletions(-)

diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 87f7f73..85ff986 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -50,7 +50,10 @@ __all__ = [ 'Descriptor', ]

+import base64 +import codecs import copy +import hashlib import os import re import tarfile @@ -499,6 +502,97 @@ class Descriptor(object): def _name(self, is_plural = False): return str(type(self))

+ def _digest_for_signature(self, signing_key, signature): + """ + Provides the signed digest we should have given this key and signature. + + :param str signing_key: key block used to make this signature + :param str signature: signed digest for this descriptor content + + :returns: the digest string encoded in uppercase hex + + :raises: ValueError if unable to provide a validly signed digest + """ + + if not stem.prereq.is_crypto_available(): + raise ValueError('Generating the signed digest requires pycrypto') + + from Crypto.Util import asn1 + from Crypto.Util.number import bytes_to_long, long_to_bytes + + # get the ASN.1 sequence + + seq = asn1.DerSequence() + seq.decode(_bytes_for_block(signing_key)) + modulus, public_exponent = seq[0], seq[1] + + sig_as_bytes = _bytes_for_block(signature) + sig_as_long = bytes_to_long(sig_as_bytes) # convert signature to an int + blocksize = 128 # block size will always be 128 for a 1024 bit key + + # use the public exponent[e] & the modulus[n] to decrypt the int + + decrypted_int = pow(sig_as_long, public_exponent, modulus) + + # convert the int to a byte array + + decrypted_bytes = long_to_bytes(decrypted_int, blocksize) + + ############################################################################ + # The decrypted bytes should have a structure exactly along these lines. + # 1 byte - [null '\x00'] + # 1 byte - [block type identifier '\x01'] - Should always be 1 + # N bytes - [padding '\xFF' ] + # 1 byte - [separator '\x00' ] + # M bytes - [message] + # Total - 128 bytes + # More info here http://www.ietf.org/rfc/rfc2313.txt + # esp the Notes in section 8.1 + ############################################################################ + + try: + if decrypted_bytes.index(b'\x00\x01') != 0: + raise ValueError('Verification failed, identifier missing') + except ValueError: + raise ValueError('Verification failed, malformed data') + + try: + identifier_offset = 2 + + # find the separator + seperator_index = decrypted_bytes.index(b'\x00', identifier_offset) + except ValueError: + raise ValueError('Verification failed, seperator not found') + + digest_hex = codecs.encode(decrypted_bytes[seperator_index + 1:], 'hex_codec') + return stem.util.str_tools._to_unicode(digest_hex.upper()) + + def _digest_for_content(self, start, end): + """ + Provides the digest of our descriptor's content in a given range. + + :param bytes start: start of the range to generate a digest for + :param bytes end: end of the range to generate a digest for + + :returns: the digest string encoded in uppercase hex + + :raises: ValueError if the digest canot be calculated + """ + + raw_descriptor = self.get_bytes() + + start_index = raw_descriptor.find(start) + end_index = raw_descriptor.find(end, start_index) + + if start_index == -1: + raise ValueError("Digest is for the range starting with '%s' but that isn't in our descriptor" % start) + elif end_index == -1: + raise ValueError("Digest is for the range ending with '%s' but that isn't in our descriptor" % end) + + digest_content = raw_descriptor[start_index:end_index + len(end)] + digest_hash = hashlib.sha1(stem.util.str_tools._to_bytes(digest_content)) + return stem.util.str_tools._to_unicode(digest_hash.hexdigest().upper()) + def __getattr__(self, name): # If attribute isn't already present we might be lazy loading it...

@@ -593,6 +687,24 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi return content

+def _bytes_for_block(content): + """ + Provides the base64 decoded content of a pgp-style block. + + :param str content: block to be decoded + + :returns: decoded block content + + :raises: **TypeError** if this isn't base64 encoded content + """ + + # strip the '-----BEGIN RSA PUBLIC KEY-----' header and footer + + content = ''.join(content.split('\n')[1:-1]) + + return base64.b64decode(stem.util.str_tools._to_bytes(content)) + + def _get_pseudo_pgp_block(remaining_contents): """ Checks if given contents begins with a pseudo-Open-PGP-style block and, if diff --git a/stem/descriptor/hidden_service_descriptor.py b/stem/descriptor/hidden_service_descriptor.py index c1970a0..075c2c2 100644 --- a/stem/descriptor/hidden_service_descriptor.py +++ b/stem/descriptor/hidden_service_descriptor.py @@ -19,19 +19,17 @@ the HSDir flag. # TODO: Add a description for how to retrieve them when tor supports that # (#14847) and then update #15009.

-import base64 import collections import io

import stem.util.connection

-from stem import str_type - from stem.descriptor import ( PGP_BLOCK_END, Descriptor, _get_descriptor_components, _read_until_keywords, + _bytes_for_block, _value, _parse_simple_line, _parse_timestamp_line, @@ -138,16 +136,15 @@ def _parse_introduction_points_line(descriptor, entries): descriptor.introduction_points_encoded = block_contents

try: - blob = ''.join(block_contents.split('\n')[1:-1]) - decoded_field = base64.b64decode(stem.util.str_tools._to_bytes(blob)) + decoded_field = _bytes_for_block(block_contents) except TypeError: raise ValueError("'introduction-points' isn't base64 encoded content:\n%s" % block_contents)

auth_types = []

- while decoded_field.startswith('service-authentication ') and '\n' in decoded_field: - auth_line, decoded_field = decoded_field.split('\n', 1) - auth_line_comp = auth_line.split(' ') + while decoded_field.startswith(b'service-authentication ') and b'\n' in decoded_field: + auth_line, decoded_field = decoded_field.split(b'\n', 1) + auth_line_comp = auth_line.split(b' ')

if len(auth_line_comp) < 3: raise ValueError("Within introduction-points we expected 'service-authentication [auth_type] [auth_data]', but had '%s'" % auth_line) @@ -178,7 +175,7 @@ class HiddenServiceDescriptor(Descriptor): :var str introduction_points_encoded: raw introduction points blob :var list introduction_points_auth: ***** tuples of the form (auth_method, auth_data) for our introduction_points_content - :var str introduction_points_content: decoded introduction-points content + :var bytes introduction_points_content: decoded introduction-points content without authentication data, if using cookie authentication this is encrypted :var str signature: signature of the descriptor content @@ -228,6 +225,13 @@ class HiddenServiceDescriptor(Descriptor): raise ValueError("Hidden service descriptor must end with a 'signature' entry")

self._parse(entries, validate) + + if stem.prereq.is_crypto_available(): + signed_digest = self._digest_for_signature(self.permanent_key, self.signature) + content_digest = self._digest_for_content(b'rendezvous-service-descriptor ', b'\nsignature\n') + + if signed_digest != content_digest: + raise ValueError('Decrypted digest does not match local digest (calculated: %s, local: %s)' % (signed_digest, content_digest)) else: self._entries = entries

@@ -257,14 +261,14 @@ class HiddenServiceDescriptor(Descriptor):

if not self.introduction_points_content: return [] - elif not self.introduction_points_content.startswith('introduction-point '): + elif not self.introduction_points_content.startswith(b'introduction-point '): raise DecryptionFailure('introduction-point content is encrypted')

introduction_points = [] - content_io = io.StringIO(str_type(self.introduction_points_content)) + content_io = io.BytesIO(self.introduction_points_content)

while True: - content = ''.join(_read_until_keywords('introduction-point', content_io, ignore_first = True)) + content = b''.join(_read_until_keywords('introduction-point', content_io, ignore_first = True))

if not content: break # reached the end diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 55b4183..54ee645 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -31,8 +31,6 @@ etc). This information is provided from a few sources... +- get_annotation_lines - lines that provided the annotations """

-import base64 -import codecs import functools import hashlib import re @@ -46,13 +44,13 @@ import stem.util.tor_tools import stem.version

from stem import str_type -from stem.util import log

from stem.descriptor import ( PGP_BLOCK_END, Descriptor, _get_descriptor_components, _read_until_keywords, + _bytes_for_block, _value, _values, _parse_simple_line, @@ -670,9 +668,18 @@ class RelayDescriptor(ServerDescriptor): def __init__(self, raw_contents, validate = False, annotations = None): super(RelayDescriptor, self).__init__(raw_contents, validate, annotations)

- # validate the descriptor if required if validate: - self._validate_content() + if self.fingerprint: + key_hash = hashlib.sha1(_bytes_for_block(self.signing_key)).hexdigest() + + if key_hash != self.fingerprint.lower(): + raise ValueError('Fingerprint does not match the hash of our signing key (fingerprint: %s, signing key hash: %s)' % (self.fingerprint.lower(), key_hash)) + + if stem.prereq.is_crypto_available(): + signed_digest = self._digest_for_signature(self.signing_key, self.signature) + + if signed_digest != self.digest(): + raise ValueError('Decrypted digest does not match local digest (calculated: %s, local: %s)' % (signed_digest, self.digest()))

@lru_cache() def digest(self): @@ -684,112 +691,7 @@ class RelayDescriptor(ServerDescriptor): :raises: ValueError if the digest canot be calculated """

- # Digest is calculated from everything in the - # descriptor except the router-signature. - - raw_descriptor = self.get_bytes() - start_token = b'router ' - sig_token = b'\nrouter-signature\n' - start = raw_descriptor.find(start_token) - sig_start = raw_descriptor.find(sig_token) - end = sig_start + len(sig_token) - - if start >= 0 and sig_start > 0 and end > start: - for_digest = raw_descriptor[start:end] - digest_hash = hashlib.sha1(stem.util.str_tools._to_bytes(for_digest)) - return stem.util.str_tools._to_unicode(digest_hash.hexdigest().upper()) - else: - raise ValueError('unable to calculate digest for descriptor') - - def _validate_content(self): - """ - Validates that the descriptor content matches the signature. - - :raises: ValueError if the signature does not match the content - """ - - key_as_bytes = RelayDescriptor._get_key_bytes(self.signing_key) - - # ensure the fingerprint is a hash of the signing key - - if self.fingerprint: - # calculate the signing key hash - - key_der_as_hash = hashlib.sha1(stem.util.str_tools._to_bytes(key_as_bytes)).hexdigest() - - if key_der_as_hash != self.fingerprint.lower(): - log.warn('Signing key hash: %s != fingerprint: %s' % (key_der_as_hash, self.fingerprint.lower())) - raise ValueError('Fingerprint does not match hash') - - self._verify_digest(key_as_bytes) - - def _verify_digest(self, key_as_der): - # check that our digest matches what was signed - - if not stem.prereq.is_crypto_available(): - return - - from Crypto.Util import asn1 - from Crypto.Util.number import bytes_to_long, long_to_bytes - - # get the ASN.1 sequence - - seq = asn1.DerSequence() - seq.decode(key_as_der) - modulus = seq[0] - public_exponent = seq[1] # should always be 65537 - - sig_as_bytes = RelayDescriptor._get_key_bytes(self.signature) - - # convert the descriptor signature to an int - - sig_as_long = bytes_to_long(sig_as_bytes) - - # use the public exponent[e] & the modulus[n] to decrypt the int - - decrypted_int = pow(sig_as_long, public_exponent, modulus) - - # block size will always be 128 for a 1024 bit key - - blocksize = 128 - - # convert the int to a byte array. - - decrypted_bytes = long_to_bytes(decrypted_int, blocksize) - - ############################################################################ - # The decrypted bytes should have a structure exactly along these lines. - # 1 byte - [null '\x00'] - # 1 byte - [block type identifier '\x01'] - Should always be 1 - # N bytes - [padding '\xFF' ] - # 1 byte - [separator '\x00' ] - # M bytes - [message] - # Total - 128 bytes - # More info here http://www.ietf.org/rfc/rfc2313.txt - # esp the Notes in section 8.1 - ############################################################################ - - try: - if decrypted_bytes.index(b'\x00\x01') != 0: - raise ValueError('Verification failed, identifier missing') - except ValueError: - raise ValueError('Verification failed, malformed data') - - try: - identifier_offset = 2 - - # find the separator - seperator_index = decrypted_bytes.index(b'\x00', identifier_offset) - except ValueError: - raise ValueError('Verification failed, seperator not found') - - digest_hex = codecs.encode(decrypted_bytes[seperator_index + 1:], 'hex_codec') - digest = stem.util.str_tools._to_unicode(digest_hex.upper()) - - local_digest = self.digest() - - if digest != local_digest: - raise ValueError('Decrypted digest does not match local digest (calculated: %s, local: %s)' % (digest, local_digest)) + return self._digest_for_content(b'router ', b'\nrouter-signature\n')

def _compare(self, other, method): if not isinstance(other, RelayDescriptor): @@ -809,20 +711,6 @@ class RelayDescriptor(ServerDescriptor): def __le__(self, other): return self._compare(other, lambda s, o: s <= o)

- @staticmethod - def _get_key_bytes(key_string): - # Remove the newlines from the key string & strip off the - # '-----BEGIN RSA PUBLIC KEY-----' header and - # '-----END RSA PUBLIC KEY-----' footer - - key_as_string = ''.join(key_string.split('\n')[1:4]) - - # get the key representation in bytes - - key_bytes = base64.b64decode(stem.util.str_tools._to_bytes(key_as_string)) - - return key_bytes -

class BridgeDescriptor(ServerDescriptor): """ diff --git a/test/mocking.py b/test/mocking.py index 4a3f272..156ac4a 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -372,7 +372,7 @@ def get_relay_server_descriptor(attr = None, exclude = (), content = False, sign if sign_content: desc_content = sign_descriptor_content(desc_content)

- with patch('stem.descriptor.server_descriptor.RelayDescriptor._verify_digest', Mock()): + with patch('stem.prereq.is_crypto_available', Mock(return_value = False)): desc = stem.descriptor.server_descriptor.RelayDescriptor(desc_content, validate = True)

return desc @@ -540,7 +540,8 @@ def get_hidden_service_descriptor(attr = None, exclude = (), content = False, in if content: return desc_content else: - return stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor(desc_content, validate = True) + with patch('stem.prereq.is_crypto_available', Mock(return_value = False)): + return stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor(desc_content, validate = True)

def get_directory_authority(attr = None, exclude = (), is_vote = False, content = False): diff --git a/test/unit/descriptor/hidden_service_descriptor.py b/test/unit/descriptor/hidden_service_descriptor.py index ef8ad43..911e9fa 100644 --- a/test/unit/descriptor/hidden_service_descriptor.py +++ b/test/unit/descriptor/hidden_service_descriptor.py @@ -74,7 +74,7 @@ TkQgUlNBIFBVQkxJQyBLRVktLS0tLQoK -----END MESSAGE-----\ """

-EXPECTED_DDG_INTRODUCTION_POINTS_CONTENT = """\ +EXPECTED_DDG_INTRODUCTION_POINTS_CONTENT = b"""\ introduction-point iwki77xtbvp6qvedfrwdzncxs3ckayeu ip-address 178.62.222.129 onion-port 443 @@ -353,7 +353,7 @@ class TestHiddenServiceDescriptor(unittest.TestCase): self.assertEqual([2, 3], desc.protocol_versions) self.assertEqual('-----BEGIN MESSAGE-----\n-----END MESSAGE-----', desc.introduction_points_encoded) self.assertEqual([], desc.introduction_points_auth) - self.assertEqual('', desc.introduction_points_content) + self.assertEqual(b'', desc.introduction_points_content) self.assertTrue(CRYPTO_BLOB in desc.signature) self.assertEqual([], desc.introduction_points())

@@ -456,7 +456,7 @@ class TestHiddenServiceDescriptor(unittest.TestCase):

self.assertEqual((MESSAGE_BLOCK % '').strip(), empty_field_desc.introduction_points_encoded) self.assertEqual([], empty_field_desc.introduction_points_auth) - self.assertEqual('', empty_field_desc.introduction_points_content) + self.assertEqual(b'', empty_field_desc.introduction_points_content) self.assertEqual([], empty_field_desc.introduction_points())

def test_introduction_points_when_not_base64(self): diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 00d2bb3..fd0c032 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -452,7 +452,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= desc_text = get_relay_server_descriptor({'opt': 'protocols Link 1 2'}, content = True) self._expect_invalid_attr(desc_text, 'circuit_protocols')

- @patch('stem.descriptor.server_descriptor.RelayDescriptor._verify_digest', Mock()) + @patch('stem.prereq.is_crypto_available', Mock(return_value = False)) def test_published_leap_year(self): """ Constructs with a published entry for a leap year, and when the date is @@ -508,7 +508,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEqual(900, desc.read_history_interval) self.assertEqual([], desc.read_history_values)

- @patch('stem.descriptor.server_descriptor.RelayDescriptor._verify_digest', Mock()) + @patch('stem.prereq.is_crypto_available', Mock(return_value = False)) def test_annotations(self): """ Checks that content before a descriptor are parsed as annotations. diff --git a/test/unit/tutorial.py b/test/unit/tutorial.py index 55a170b..7d20437 100644 --- a/test/unit/tutorial.py +++ b/test/unit/tutorial.py @@ -201,7 +201,7 @@ class TestTutorial(unittest.TestCase):

@patch('sys.stdout', new_callable = StringIO) @patch('stem.descriptor.remote.DescriptorDownloader') - @patch('stem.descriptor.server_descriptor.RelayDescriptor._verify_digest', Mock()) + @patch('stem.prereq.is_crypto_available', Mock(return_value = False)) def test_mirror_mirror_on_the_wall_5(self, downloader_mock, stdout_mock): def tutorial_example(): from stem.descriptor.remote import DescriptorDownloader