commit e0095fbe54759c45cbf6d1b120d2b17b47a0ec21 Author: Eoin o Fearghail eoin.o.fearghail@gmail.com Date: Fri Nov 23 22:16:22 2012 +0000

Implementing Relay Descriptor verification

cf https://trac.torproject.org/projects/tor/ticket/5810

1) Implemented relay descriptor verification using the python-crypto lib. Code is only run if python-crypto can be imported. [cf stem.prereq.is_crypto_available()] NOTE: constructing a RelayDescriptor will now raise an exception if invalid descriptor content is used. 2) Refactored the digest() function in server_descriptor.py. 3) Added a function to the mocking lib to sign a descriptor with an auto-generated key 4) Add usage of new sign_descriptor_content() in unit tests where necessary. 5) Updated the non-ascii-descriptor file to be correctly signed. 6) Updated extra info descriptor test to use new fingerprint in non-ascii-descriptor file 7) Removed server descriptor tests that do not make sense if data is being generated dynamically. e.g. Removed test fingerprint valid test, since data now dynamically generated. --- stem/descriptor/server_descriptor.py | 144 +++++++++++++++++------ stem/prereq.py | 23 ++-- test/integ/descriptor/data/non-ascii_descriptor | 14 +- test/integ/descriptor/server_descriptor.py | 4 +- test/mocking.py | 85 +++++++++++++ test/unit/descriptor/server_descriptor.py | 23 +--- test/unit/tutorial.py | 8 +- 7 files changed, 223 insertions(+), 78 deletions(-)

diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 046dfe1..9f3dbb3 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -39,6 +39,7 @@ import stem.exit_policy import stem.version import stem.util.connection import stem.util.tor_tools +import stem.util.log as log

# relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( @@ -593,52 +594,123 @@ class RelayDescriptor(ServerDescriptor):

super(RelayDescriptor, self).__init__(raw_contents, validate, annotations)

- # if we have a fingerprint then checks that our fingerprint is a hash of - # our signing key + # validate the descriptor if required + if validate: + # ensure the digest of the descriptor has been calculated + self.digest() + self._validate_content() + + def digest(self): + # Digest is calculated from everything in the + # descriptor except the router-signature. + raw_descriptor = str(self) + start_token = "router " + sig_token = "\nrouter-signature\n" + start = raw_descriptor.find(start_token) + sig_start = raw_descriptor.find(sig_token) + end = sig_start + len(sig_token) + if start >= 0 and sig_start > 0 and end > start: + for_digest = raw_descriptor[start:end] + digest_hash = hashlib.sha1(for_digest) + self._digest = digest_hash.hexdigest() + else: + log.warn("unable to calculate digest for descriptor") + raise ValueError("unable to calculate digest for descriptor")

- if validate and self.fingerprint and stem.prereq.is_rsa_available(): - import rsa - pubkey = rsa.PublicKey.load_pkcs1(self.signing_key) - der_encoded = pubkey.save_pkcs1(format = "DER") - key_hash = hashlib.sha1(der_encoded).hexdigest() - - if key_hash != self.fingerprint.lower(): - raise ValueError("Hash of our signing key doesn't match our fingerprint. Signing key hash: %s, fingerprint: %s" % (key_hash, self.fingerprint.lower())) + return self._digest

- def is_valid(self): + def _validate_content(self): """ Validates that our content matches our signature.

- **Method implementation is incomplete, and will raise a NotImplementedError** - - :returns: **True** if our signature matches our content, **False** otherwise + :raises a ValueError if signature does not match content, """

- raise NotImplementedError # TODO: finish implementing - - # without validation we may be missing our signature - if not self.signature: return False - - # gets base64 encoded bytes of our signature without newlines nor the - # "-----[BEGIN|END] SIGNATURE-----" header/footer + if not self.signature: + log.warn("Signature missing") + raise ValueError("Signature missing")

- sig_content = self.signature.replace("\n", "")[25:-23] - sig_bytes = base64.b64decode(sig_content) + # strips off the '-----BEGIN RSA PUBLIC KEY-----' header and corresponding footer + key_as_string = ''.join(self.signing_key.split('\n')[1:4])

- # TODO: Decrypt the signature bytes with the signing key and remove - # the PKCS1 padding to get the original message, and encode the message - # in hex and compare it to the digest of the descriptor. + # calculate the signing key hash + key_as_der = base64.b64decode(key_as_string) + key_der_as_hash = hashlib.sha1(key_as_der).hexdigest()

- return True - - def digest(self): - if self._digest is None: - # our digest is calculated from everything except our signature - raw_content, ending = str(self), "\nrouter-signature\n" - raw_content = raw_content[:raw_content.find(ending) + len(ending)] - self._digest = hashlib.sha1(raw_content).hexdigest().upper() - - return self._digest + # if we have a fingerprint then check that our fingerprint is a hash of + # our signing key + if self.fingerprint: + if key_der_as_hash != self.fingerprint.lower(): + log.warn("Hash of our signing key doesn't match our fingerprint. Signing key hash: %s, fingerprint: %s" % (key_der_as_hash, self.fingerprint.lower())) + raise ValueError("Fingerprint does not match hash") + else: + log.notice("No fingerprint for this descriptor") + + try: + self._verify_descriptor(key_as_der) + log.info("Descriptor verified.") + except ValueError, e: + log.warn("Failed to verify descriptor: %s" % e) + raise e + + def _verify_descriptor(self, key_as_der): + if not stem.prereq.is_crypto_available(): + return + else: + from Crypto.Util import asn1 + from Crypto.Util.number import bytes_to_long, long_to_bytes + + # get the ASN.1 sequence + seq = asn1.DerSequence() + seq.decode(key_as_der) + modulus = seq[0] + public_exponent = seq[1] #should always be 65537 + + # convert the descriptor signature to an int before decrypting it + sig_as_string = ''.join(self.signature.split('\n')[1:4]) + sig_as_bytes = base64.b64decode(sig_as_string) + sig_as_long = bytes_to_long(sig_as_bytes) + + # use the public exponent[e] & the modulus[n] to decrypt the int + decrypted_int = pow(sig_as_long, public_exponent ,modulus) + # block size will always be 128 for a 1024 bit key + blocksize = 128 + # convert the int to a byte array. + decrypted_bytes = long_to_bytes(decrypted_int, blocksize) + + ############################################################################ + ## The decrypted bytes should have a structure exactly along these lines. + ## 1 byte - [null '\x00'] + ## 1 byte - [block type identifier '\x01'] - Should always be 1 + ## N bytes - [padding '\xFF' ] + ## 1 byte - [separator '\x00' ] + ## M bytes - [message] + ## Total - 128 bytes + ## More info here http://www.ietf.org/rfc/rfc2313.txt + ## esp the Notes in section 8.1 + ############################################################################ + try: + if decrypted_bytes.index('\x00\x01') != 0: + log.warn("Verification failed, identifier missing") + raise ValueError("Verification failed, identifier missing") + except ValueError: + log.warn("Verification failed, Malformed data") + raise ValueError("Verification failed, Malformed data") + + try: + identifier_offset = 2 + # Find the separator + seperator_index = decrypted_bytes.index('\x00', identifier_offset) + except ValueError: + log.warn("Verification failed, seperator not found") + raise ValueError("Verification failed, seperator not found") + + digest = decrypted_bytes[seperator_index+1:] + # The local digest is stored in hex so need to encode the decrypted digest + digest_hex = digest.encode('hex') + if digest_hex != self._digest: + log.warn("Decrypted digest does not match local digest") + raise ValueError("Decrypted digest does not match local digest")

def _parse(self, entries, validate): entries = dict(entries) # shallow copy since we're destructive diff --git a/stem/prereq.py b/stem/prereq.py index 67611fd..d205954 100644 --- a/stem/prereq.py +++ b/stem/prereq.py @@ -24,7 +24,7 @@ import sys

import stem.util.log as log

-IS_RSA_AVAILABLE = None +IS_CRYPTO_AVAILABLE = None

def check_requirements(): """ @@ -59,20 +59,23 @@ def is_python_27():

return _check_version(7)

-def is_rsa_available(): - global IS_RSA_AVAILABLE +def is_crypto_available(): + global IS_CRYPTO_AVAILABLE

- if IS_RSA_AVAILABLE == None: + if IS_CRYPTO_AVAILABLE == None: try: - import rsa - IS_RSA_AVAILABLE = True + from Crypto.PublicKey import RSA + from Crypto.Util import asn1 + from Crypto.Util.number import long_to_bytes + IS_CRYPTO_AVAILABLE = True except ImportError: - IS_RSA_AVAILABLE = False + IS_CRYPTO_AVAILABLE = False

- msg = "Unable to import the rsa module. Because of this we'll be unable to verify descriptor signature integrity." - log.log_once("stem.prereq.is_rsa_available", log.INFO, msg) + # the code that verifies relay descriptor signatures uses the python-crypto library + msg = "Unable to import the crypto module. Because of this we'll be unable to verify descriptor signature integrity." + log.log_once("stem.prereq.is_crypto_available", log.INFO, msg)

- return IS_RSA_AVAILABLE + return IS_CRYPTO_AVAILABLE

def _check_version(minor_req): major_version, minor_version = sys.version_info[0:2] diff --git a/test/integ/descriptor/data/non-ascii_descriptor b/test/integ/descriptor/data/non-ascii_descriptor index 2cd2a6b..eb3e31a 100644 --- a/test/integ/descriptor/data/non-ascii_descriptor +++ b/test/integ/descriptor/data/non-ascii_descriptor @@ -3,7 +3,7 @@ router torrelay389752132 130.243.230.116 9001 0 0 platform Tor 0.2.2.35 (git-4f42b0a93422f70e) on Linux x86_64 opt protocols Link 1 2 Circuit 1 published 2012-03-21 16:28:14 -opt fingerprint FEBC 7F99 2AC4 18BB E42B C13F E94E FCFE 6549 197E +opt fingerprint 5D47 E91A 1F74 21A4 E325 5F4D 04E5 34E9 A214 07BB uptime 3103848 bandwidth 81920 102400 84275 opt extra-info-digest 51E9FD0DA7C235D8C0250BAFB6E1ABB5F1EF9F04 @@ -15,16 +15,16 @@ k3Rx75up+wsuBzhfwSYr7W+T+WkDQvz49RFPpns6Ef0qFpQ1TlHxAgMBAAE= -----END RSA PUBLIC KEY----- signing-key -----BEGIN RSA PUBLIC KEY----- -MIGJAoGBAMSmtutGlXVdvh/IC4TyhQpgSajxrZItC2lS5/70Vr4uLevryPlBgVrW -35CHxKYaj0MAOfkJQ0/OvTaXe7hlaCLrDDXScaH/XEDurcWrynsdzomsCvn/6VJ+ -xZFszt2Dn5myXKMvYy3j1oevC4iDaZXwxgpwx/UMJsFn7GOUPFYbAgMBAAE= +MIGJAoGBALRHQWXGjGLNROY8At3dMnrcSxw4PF/9oLYuqCsXNAq0Gju+EBA5qfM4 +AMpeOk+7ZsZ6AsjdBPAPaOf7hm+z6Kr3Am/gC43dci+iuNHf2wYLR8TnW/C5Q6ZQ +iXpSAGrOHnIptyPHa0j9ayM4WmHWrPBKnC0QA91CGrxnnNc6DHehAgMBAAE= -----END RSA PUBLIC KEY----- opt hidden-service-dir contact 2048R/F171EC1F Johan Blåbäck こんにちは reject *:* router-signature -----BEGIN SIGNATURE----- -q3Tw41+miuycnXowX/k6CCOcHMjw0BCDjW56Wh/eHoICmVb/hBJdtuzTaorWHLWp -OoTa4Sy4OrGFL+ldzajGC8+oqMvrYudiIxbJWmH3NXFyd7ZeEdnHzHxNOU8p1+X+ -hFwdOCEvzvvTbOuS2DwDt+TU8rljZunZfcMWgXktAD0= +WqBgiomhJ+XewpbOGg1r+6KXlAkdxHRhgCB/D980yJVzXWbOCrRhwyyAH9Lx+yrK +1EFXAtfQBBx2hmsw8CSYuUT6ckjXyUBAKEdABC25yRdi+fN3NfSQd56U9MvArjo9 +Y8oz244gH4BSVp4CScL8dK0EUsUrAxjs+OU7bnV5saA= -----END SIGNATURE----- diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 55e6545..bfe13a7 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -86,7 +86,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(expected_signing_key, desc.signing_key) self.assertEquals(expected_signature, desc.signature) self.assertEquals([], desc.get_unrecognized_lines()) - self.assertEquals("2C7B27BEAB04B4E2459D89CA6D5CD1CC5F95A689", desc.digest()) + self.assertEquals("2C7B27BEAB04B4E2459D89CA6D5CD1CC5F95A689", desc.digest().upper())

def test_old_descriptor(self): """ @@ -190,7 +190,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4=

desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) self.assertEquals("torrelay389752132", desc.nickname) - self.assertEquals("FEBC7F992AC418BBE42BC13FE94EFCFE6549197E", desc.fingerprint) + self.assertEquals("5D47E91A1F7421A4E3255F4D04E534E9A21407BB", desc.fingerprint) self.assertEquals("130.243.230.116", desc.address) self.assertEquals(9001, desc.or_port) self.assertEquals(None, desc.socks_port) diff --git a/test/mocking.py b/test/mocking.py index a3d72b1..4c4ea25 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -46,6 +46,9 @@ calling :func:`test.mocking.revert_mocking`. get_router_status_entry_micro_v3 - RouterStatusEntryMicroV3 """

+ +import base64 +import hashlib import inspect import itertools import StringIO @@ -541,6 +544,7 @@ def get_relay_server_descriptor(attr = None, exclude = (), content = False): if content: return desc_content else: + desc_content = sign_descriptor_content(desc_content) return stem.descriptor.server_descriptor.RelayDescriptor(desc_content, validate = True)

def get_bridge_server_descriptor(attr = None, exclude = (), content = False): @@ -783,3 +787,84 @@ def get_network_status_document_v3(attr = None, exclude = (), authorities = None else: return stem.descriptor.networkstatus.NetworkStatusDocumentV3(desc_content, validate = True)

+def sign_descriptor_content(desc_content): + + if not stem.prereq.is_crypto_available(): + return desc_content + else: + from Crypto.PublicKey import RSA + from Crypto.Util import asn1 + from Crypto.Util.number import long_to_bytes + + # generate a key + private_key = RSA.generate(1024) + + # get a string representation of the public key + seq = asn1.DerSequence() + seq.append(private_key.n) + seq.append(private_key.e) + seq_as_string = seq.encode() + public_key_string = base64.b64encode(seq_as_string) + + # split public key into lines 64 characters long + public_key_string = public_key_string [:64] + "\n" +public_key_string[64:128] +"\n" +public_key_string[128:] + + # generate the new signing key string + signing_key_token = "\nsigning-key\n" #note the trailing '\n' is important here so as not to match the string elsewhere + signing_key_token_start = "-----BEGIN RSA PUBLIC KEY-----\n" + signing_key_token_end = "\n-----END RSA PUBLIC KEY-----\n" + new_sk = signing_key_token+ signing_key_token_start+public_key_string+signing_key_token_end + + # update the descriptor string with the new signing key + skt_start = desc_content.find(signing_key_token) + skt_end = desc_content.find(signing_key_token_end, skt_start) + desc_content = desc_content[:skt_start]+new_sk+ desc_content[skt_end+len(signing_key_token_end):] + + # generate the new fingerprint string + key_hash = hashlib.sha1(seq_as_string).hexdigest().upper() + grouped_fingerprint = "" + for x in range(0, len(key_hash), 4): + grouped_fingerprint += " " + key_hash[x:x+4] + fingerprint_token = "\nfingerprint" + new_fp = fingerprint_token + grouped_fingerprint + + # update the descriptor string with the new fingerprint + ft_start = desc_content.find(fingerprint_token) + if ft_start < 0: + fingerprint_token = "\nopt fingerprint" + ft_start = desc_content.find(fingerprint_token) + + # if the descriptor does not already contain a fingerprint do not add one + if ft_start >= 0: + ft_end = desc_content.find("\n", ft_start+1) + desc_content = desc_content[:ft_start]+new_fp+desc_content[ft_end:] + + # calculate the new digest for the descriptor + tempDesc = stem.descriptor.server_descriptor.RelayDescriptor(desc_content, validate=False) + new_digest_hex = tempDesc.digest() + # remove the hex encoding + new_digest = new_digest_hex.decode('hex') + + # Generate the digest buffer. + # block is 128 bytes in size + # 2 bytes for the type info + # 1 byte for the separator + padding = "" + for x in range(125 - len(new_digest)): + padding += '\xFF' + digestBuffer = '\x00\x01' + padding + '\x00' + new_digest + + # generate a new signature by signing the digest buffer with the private key + (signature, ) = private_key.sign(digestBuffer, None) + signature_as_bytes = long_to_bytes(signature, 128) + signature_base64 = base64.b64encode(signature_as_bytes) + signature_base64 = signature_base64 [:64] + "\n" +signature_base64[64:128] +"\n" +signature_base64[128:] + + # update the descriptor string with the new signature + router_signature_token = "\nrouter-signature\n" + router_signature_start = "-----BEGIN SIGNATURE-----\n" + router_signature_end = "\n-----END SIGNATURE-----\n" + rst_start = desc_content.find(router_signature_token) + desc_content = desc_content[:rst_start] + router_signature_token + router_signature_start + signature_base64 + router_signature_end + + return desc_content diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index fc81851..2e54e44 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -10,7 +10,7 @@ import stem.prereq import stem.descriptor.server_descriptor from stem.descriptor.server_descriptor import RelayDescriptor, BridgeDescriptor import test.runner -from test.mocking import get_relay_server_descriptor, get_bridge_server_descriptor, CRYPTO_BLOB +from test.mocking import get_relay_server_descriptor, get_bridge_server_descriptor, CRYPTO_BLOB, sign_descriptor_content

class TestServerDescriptor(unittest.TestCase): def test_minimal_relay_descriptor(self): @@ -25,8 +25,6 @@ class TestServerDescriptor(unittest.TestCase): self.assertEquals("71.35.133.197", desc.address) self.assertEquals(None, desc.fingerprint) self.assertTrue(CRYPTO_BLOB in desc.onion_key) - self.assertTrue(CRYPTO_BLOB in desc.signing_key) - self.assertTrue(CRYPTO_BLOB in desc.signature)

def test_with_opt(self): """ @@ -148,6 +146,7 @@ class TestServerDescriptor(unittest.TestCase): self._expect_invalid_attr(desc_text, "published")

desc_text = get_relay_server_descriptor({"published": "2012-02-29 04:03:19"}, content = True) + desc_text = sign_descriptor_content(desc_text) expected_published = datetime.datetime(2012, 2, 29, 4, 3, 19) self.assertEquals(expected_published, RelayDescriptor(desc_text).published)

@@ -200,6 +199,7 @@ class TestServerDescriptor(unittest.TestCase):

desc_text = "@pepperjack very tasty\n@mushrooms not so much\n" desc_text += get_relay_server_descriptor(content = True) + desc_text = sign_descriptor_content(desc_text) desc_text += "\ntrailing text that should be ignored, ho hum"

# running parse_file should provide an iterator with a single descriptor @@ -243,29 +243,12 @@ class TestServerDescriptor(unittest.TestCase): self.assertEquals(None, desc.socks_port) self.assertEquals(None, desc.dir_port)

- def test_fingerprint_valid(self): - """ - Checks that a fingerprint matching the hash of our signing key will validate. - """ - - if not stem.prereq.is_rsa_available(): - test.runner.skip(self, "(rsa module unavailable)") - return - - fingerprint = "4F0C 867D F0EF 6816 0568 C826 838F 482C EA7C FE44" - desc = get_relay_server_descriptor({"opt fingerprint": fingerprint}) - self.assertEquals(fingerprint.replace(" ", ""), desc.fingerprint) - def test_fingerprint_invalid(self): """ Checks that, with a correctly formed fingerprint, we'll fail validation if it doesn't match the hash of our signing key. """

- if not stem.prereq.is_rsa_available(): - test.runner.skip(self, "(rsa module unavailable)") - return - fingerprint = "4F0C 867D F0EF 6816 0568 C826 838F 482C EA7C FE45" desc_text = get_relay_server_descriptor({"opt fingerprint": fingerprint}, content = True) self._expect_invalid_attr(desc_text, "fingerprint", fingerprint.replace(" ", "")) diff --git a/test/unit/tutorial.py b/test/unit/tutorial.py index f8a8d09..ae1a829 100644 --- a/test/unit/tutorial.py +++ b/test/unit/tutorial.py @@ -39,9 +39,11 @@ class TestTutorial(unittest.TestCase): from stem.descriptor.reader import DescriptorReader from stem.util import str_tools

- exit_descriptor = RelayDescriptor(mocking.get_relay_server_descriptor({ - 'router': 'speedyexit 149.255.97.109 9001 0 0' - }, content = True).replace('reject *:*', 'accept *:*')) + exit_descriptor = mocking.get_relay_server_descriptor({ + 'router': 'speedyexit 149.255.97.109 9001 0 0' + }, content = True).replace('reject *:*', 'accept *:*') + exit_descriptor = mocking.sign_descriptor_content(exit_descriptor) + exit_descriptor = RelayDescriptor(exit_descriptor)

reader_wrapper = mocking.get_object(DescriptorReader, { '__enter__': lambda x: x,