June 2013 - tor-commits - lists.torproject.org

[ooni-probe/develop] Update imports for tls_handshake.py.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 34d7ebc3bcde788a2f4dfaeec105b1b4b069e368 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 17:39:04 2013 +0000 Update imports for tls_handshake.py. --- nettests/experimental/tls_handshake.py | 34 ++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 28c9990..76b9d3f 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -21,7 +21,8 @@ @copyright: © 2013 Isis Lovecruft, The Tor Project Inc. """ -from socket import error as socket_error +from socket import error as socket_error +from socket import timeout as socket_timeout from time import sleep import os @@ -30,13 +31,12 @@ import struct import sys import types +import OpenSSL + from ipaddr import IPAddress -from OpenSSL import SSL -from OpenSSL.crypto import dump_certificate, dump_privatekey -from OpenSSL.crypto import X509Name, PKey, FILETYPE_PEM -from twisted.internet import defer -from twisted.python import usage -from twisted.python.failure import Failure +from OpenSSL import SSL, crypto +from twisted.internet import defer, threads +from twisted.python import usage, failure from ooni import nettest, config from ooni.utils import log, NotRootError @@ -82,6 +82,13 @@ class HostUnreachableError(Exception): """Raised when there the host IP address appears to be unreachable.""" pass +class ConnectionTimeout(Exception): + """ + Raised when we receive a :class:`socket.timeout`, in order to pass the + Exception along to :func:`connectionFailed`. + """ + pass + class UsageOptions(usage.Options): optParameters = [ ['host', 'h', None, 'Remote host IP address (v4/v6)'], @@ -191,13 +198,14 @@ class TLSHandshakeTest(nettest.NetTestCase): def getPeerCert(connection, get_chain=False): if not get_chain: x509_cert = connection.get_peer_certificate() - pem_cert = dump_certificate(FILETYPE_PEM, x509_cert) + pem_cert = crypto.dump_certificate(crypto.FILETYPE_PEM, x509_cert) return pem_cert else: cert_chain = [] x509_cert_chain = connection.get_peer_cert_chain() for x509_cert in x509_cert_chain: - pem_cert = dump_certificate(FILETYPE_PEM, x509_cert) + pem_cert = crypto.dump_certificate(crypto.FILETYPE_PEM, + x509_cert) cert_chain.append(pem_cert) return cert_chain @@ -216,9 +224,9 @@ class TLSHandshakeTest(nettest.NetTestCase): x509_name = None try: - assert isinstance(certificate, X509Name), \ + assert isinstance(certificate, crypto.X509Name), \ "getX509Name takes OpenSSL.crypto.X509Name as first argument!" - x509_name = X509Name(certificate) + x509_name = crypto.X509Name(certificate) except AssertionError as ae: log.err(ae) except Exception as exc: @@ -240,12 +248,12 @@ class TLSHandshakeTest(nettest.NetTestCase): @param key: A :class:`OpenSSL.crypto.PKey` object. """ try: - assert isinstance(key, PKey), \ + assert isinstance(key, crypto.PKey), \ "getPublicKey expects type OpenSSL.crypto.PKey for parameter key" except AssertionError as ae: log.err(ae) else: - pubkey = dump_privatekey(FILETYPE_PEM, key) + pubkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, key) return pubkey def test_tlsv1_handshake(self):

1 0

[ooni-probe/develop] Make a note on the socket option settings for blocking.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 578c6703106c81171e4af5ad45fca22fe8979d76 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 17:56:29 2013 +0000 Make a note on the socket option settings for blocking. --- nettests/experimental/tls_handshake.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index fbd60d8..3c9a033 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -325,7 +325,10 @@ class TLSHandshakeTest(nettest.NetTestCase): @param connection: A :class:`OpenSSL.SSL.Connection` object. @param host: A tuple of the host IP and port, i.e. ('1.1.1.1', 443). """ + ## xxx TODO to get this to work with a non-blocking socket, see how + ## twisted.internet.tcp.Client handles socket objects. connection.setblocking(1) + ## Set the timeout on the connection: ## ## We want to set SO_RCVTIMEO and SO_SNDTIMEO, which both are

1 0

[ooni-probe/develop] Remove erroneous state string report from handshakeSuccessful().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit c2d3d3813dac2c337cac059315c990b57767abc1 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 18:00:17 2013 +0000 Remove erroneous state string report from handshakeSuccessful(). --- nettests/experimental/tls_handshake.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 47e5091..48902b3 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -692,11 +692,9 @@ class TLSHandshakeTest(nettest.NetTestCase): """ addr, port = host log.msg("Handshake with %s:%d failed!" % host) + self.report['host'] = host self.report['port'] = port - self.report['state'] = "HANDSHAKE_FAILED" - ## xxx do we need this? - #return connection @defer.inlineCallbacks def deferMakeConnection(host):

1 0

[ooni-probe/develop] Set tls_handshake.state to be connection.state_string so that it gets updated
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 2ca25a6d0d2a7320e1f508411bf6c2dea077d5c4 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 17:51:32 2013 +0000 Set tls_handshake.state to be connection.state_string so that it gets updated and we always have the latest state accessible in failure and success callbacks. --- nettests/experimental/tls_handshake.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index d9d805f..3e754ae 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -482,6 +482,7 @@ class TLSHandshakeTest(nettest.NetTestCase): """ try: while connection.want_read(): + self.state = connection.state_string() log.debug("Connection to %s HAS want_read" % host) _read_buffer = connection.pending() log.debug("Rereading %d bytes..." % _read_buffer) @@ -490,11 +491,13 @@ class TLSHandshakeTest(nettest.NetTestCase): log.debug("Received %d bytes" % rereceived) log.debug("State: %s" % connection.state_string()) else: + self.state = connection.state_string() peername, peerport = connection.getpeername() log.debug("Connection to %s:%s DOES NOT HAVE want_read" % (peername, peerport)) log.debug("State: %s" % connection.state_string()) except SSL.WantWriteError, wwe: + self.state = connection.state_string() log.debug("Got WantWriteError while handling want_read") log.debug("WantWriteError: %s" % wwe.message) log.debug("Switching to handleWantWrite()...") @@ -507,12 +510,14 @@ class TLSHandshakeTest(nettest.NetTestCase): """ try: while connection.want_write(): + self.state = connection.state_string() log.debug("Connection to %s HAS want_write" % host) sleep(1) resent = connection.send("o\r\n") log.debug("Sent: %d" % resent) log.debug("State: %s" % connection.state_string()) except SSL.WantReadError, wre: + self.state = connection.state_string() log.debug("Got WantReadError while handling want_write") log.debug("WantReadError: %s" % wre.message) log.debug("Switching to handleWantRead()...") @@ -565,23 +570,27 @@ class TLSHandshakeTest(nettest.NetTestCase): received = connection.recv(int(_read_buffer)) except SSL.WantReadError, wre: if connection.want_read(): + self.state = connection.state_string() connection = handleWantRead(connection) else: ## if we still have an SSL_ERROR_WANT_READ, then try ## to renegotiate + self.state = connection.state_string() connection = connectionRenegotiate(connection, connection.getpeername(), wre.message) except SSL.WantWriteError, wwe: - log.debug("State: %s" % connection.state_string()) + self.state = connection.state_string() + log.debug("Handshake state: %s" % self.state) if connection.want_write(): connection = handleWantWrite(connection) else: - log.msg("Connection to %s:%s timed out." - % (peername, str(peerport))) + raise ConnectionTimeout("Connection to %s:%d timed out." + % (peername, peerport)) else: log.msg("Received: %s" % received) - log.debug("State: %s" % connection.state_string()) + self.state = connection.state_string() + log.debug("Handshake state: %s" % self.state) return connection

1 0

[ooni-probe/develop] Fix #8359. Report server public key in PEM format. There is a bug in
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 96a1640e614eee8fa7f84837a84066e01261c622 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 13:31:57 2013 +0000 Fix #8359. Report server public key in PEM format. There is a bug in pyOpenSSL where the dumped public key begins with '----- BEGIN PRIVATE KEY -----' due to the OpenSSL.crypto.dump_privatekey() method hardcoding incorrect PEM headers, which would need to be fixed in the upstream if we care enough. --- nettests/experimental/tls_handshake.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 34a917c..28c9990 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -32,8 +32,8 @@ import types from ipaddr import IPAddress from OpenSSL import SSL -from OpenSSL.crypto import dump_certificate, FILETYPE_PEM -from OpenSSL.crypto import X509Name +from OpenSSL.crypto import dump_certificate, dump_privatekey +from OpenSSL.crypto import X509Name, PKey, FILETYPE_PEM from twisted.internet import defer from twisted.python import usage from twisted.python.failure import Failure @@ -232,6 +232,22 @@ class TLSHandshakeTest(nettest.NetTestCase): else: log.debug("getX509Name: got None for ivar x509_name") + @staticmethod + def getPublicKey(key): + """ + Get the PEM-encoded format of a host certificate's public key. + + @param key: A :class:`OpenSSL.crypto.PKey` object. + """ + try: + assert isinstance(key, PKey), \ + "getPublicKey expects type OpenSSL.crypto.PKey for parameter key" + except AssertionError as ae: + log.err(ae) + else: + pubkey = dump_privatekey(FILETYPE_PEM, key) + return pubkey + def test_tlsv1_handshake(self): """xxx fill me in""" @@ -570,12 +586,12 @@ class TLSHandshakeTest(nettest.NetTestCase): server_cert_chain = self.getPeerCert(connection, get_chain=True) s_cert = connection.get_peer_certificate() - cert_subject = getX509Name(s_cert.get_subject(), - get_components=True) + cert_subject = self.getX509Name(s_cert.get_subject(), + get_components=True) cert_subj_hash = s_cert.subject_name_hash() - cert_issuer = getX509Name(s_cert.get_issuer(), - get_components=True) - cert_public_key = s_cert.get_pubkey() + cert_issuer = self.getX509Name(s_cert.get_issuer(), + get_components=True) + cert_public_key = self.getPublicKey(s_cert.get_pubkey()) cert_serial_no = s_cert.get_serial_number() cert_sig_algo = s_cert.get_signature_algorithm()

1 0

[ooni-probe/develop] Changed log.msg to log.debug when we can't get the peer name in
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 3461ad38a1e817daffa7d79f873db989d5e8efe9 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 17:43:26 2013 +0000 Changed log.msg to log.debug when we can't get the peer name in connectionSucceeded(). --- nettests/experimental/tls_handshake.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 610cdff..d9d805f 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -345,8 +345,8 @@ class TLSHandshakeTest(nettest.NetTestCase): log.msg("Connected to %s" % peer_name) else: log.debug("Couldn't get peer name from connection: %s" % host) - log.msg("Connected to: %s" % host) - log.msg("Connection state: %s " % connection.state_string()) + log.msg("Connected to %s" % host) + log.debug("Connection state: %s " % connection.state_string()) return connection @@ -423,6 +423,7 @@ class TLSHandshakeTest(nettest.NetTestCase): else: log.debug("connectionShutdown: expected connection, got %s" % connection.__repr__()) + return connection def handleWantRead(connection):

1 0

[ooni-probe/develop] Add better printing of results to stdout in handshakeSuccessful().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit ba09aef6c8fd631817dafd6159003acba04c50e0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 17:58:28 2013 +0000 Add better printing of results to stdout in handshakeSuccessful(). --- nettests/experimental/tls_handshake.py | 62 ++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index e443242..47e5091 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -625,45 +625,61 @@ class TLSHandshakeTest(nettest.NetTestCase): @returns: None. """ host, port = connection.getpeername() + log.msg("Handshake with %s:%d successful!" % (host, port)) + server_cert = self.getPeerCert(connection) server_cert_chain = self.getPeerCert(connection, get_chain=True) - s_cert = connection.get_peer_certificate() - cert_subject = self.getX509Name(s_cert.get_subject(), - get_components=True) - cert_subj_hash = s_cert.subject_name_hash() - cert_issuer = self.getX509Name(s_cert.get_issuer(), - get_components=True) - cert_public_key = self.getPublicKey(s_cert.get_pubkey()) - cert_serial_no = s_cert.get_serial_number() - cert_sig_algo = s_cert.get_signature_algorithm() + renegotiations = connection.total_renegotiations() + cipher_list = connection.get_cipher_list() + session_key = connection.master_key() + rawcert = connection.get_peer_certificate() + ## xxx TODO this hash needs to be formatted as SHA1, not long + cert_subj_hash = rawcert.subject_name_hash() + cert_serial = rawcert.get_serial_number() + cert_sig_algo = rawcert.get_signature_algorithm() + cert_subject = self.getX509Name(rawcert.get_subject(), + get_components=True) + cert_issuer = self.getX509Name(rawcert.get_issuer(), + get_components=True) + cert_pubkey = self.getPublicKey(rawcert.get_pubkey()) self.report['host'] = host self.report['port'] = port - self.report['state'] = connection.state_string() - self.report['renegotiations'] = connection.total_renegotiations() + self.report['state'] = self.state + self.report['renegotiations'] = renegotiations self.report['server_cert'] = server_cert self.report['server_cert_chain'] = \ ''.join([cert for cert in server_cert_chain]) - self.report['server_ciphersuite'] = connection.get_cipher_list() - self.report['cert_subject'] = str(cert_subject) - self.report['cert_subj_hash'] = str(cert_subj_hash) - self.report['cert_issuer'] = str(cert_issuer) - ## xxx this needs to be parsed into PEM also - self.report['cert_public_key'] = str(cert_public_key) - self.report['cert_serial_no'] = str(cert_serial_no) - self.report['cert_sig_algo'] = str(cert_sig_algo) - + self.report['server_ciphersuite'] = cipher_list + self.report['cert_subject'] = cert_subject + self.report['cert_subj_hash'] = cert_subj_hash + self.report['cert_issuer'] = cert_issuer + self.report['cert_public_key'] = cert_pubkey + self.report['cert_serial_no'] = cert_serial + self.report['cert_sig_algo'] = cert_sig_algo ## The session's master key is only valid for that session, and ## will allow us to decrypt any packet captures (if they were ## collected). Because we are not requesting URLs, only host:port ## (which would be visible in pcaps anyway, since the FQDN is ## never encrypted) I do not see a way for this to log any user or ## identifying information. Correct me if I'm wrong. - self.report['session_key'] = connection.master_key() + self.report['session_key'] = session_key + + log.msg("Server certificate:\n\n%s" % server_cert) + log.msg("Server certificate chain:\n\n%s" + % ''.join([cert for cert in server_cert_chain])) + log.msg("Negotiated ciphersuite:\n%s" + % '\n\t'.join([cipher for cipher in cipher_list])) + log.msg("Certificate subject: %s" % cert_subject) + log.msg("Certificate subject hash: %d" % cert_subj_hash) + log.msg("Certificate issuer: %s" % cert_issuer) + log.msg("Certificate public key:\n\n%s" % cert_pubkey) + log.msg("Certificate signature algorithm: %s" % cert_sig_algo) + log.msg("Certificate serial number: %s" % cert_serial) + log.msg("Total renegotiations: %d" % renegotiations) - ## xxx do we need this? - #return connection + return connection def handshakeFailed(connection, host): """

1 0

[ooni-probe/develop] Appease asn's penchant for borderline obsessive compulsive docstrings.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit a67c418beb80ba8f3c8e81ddc906472fc30b3274 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 17:52:51 2013 +0000 Appease asn's penchant for borderline obsessive compulsive docstrings. * Notice, also, how the last sentence is less than 70 chars. :) --- nettests/experimental/tls_handshake.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 3e754ae..fbd60d8 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -603,8 +603,6 @@ class TLSHandshakeTest(nettest.NetTestCase): >>> server_cert.get_pubkey() <OpenSSL.crypto.PKey at 0x4985d28> >>> pk = server_cert.get_pubkey() - >>> pk.check - <function check> >>> pk.check() Segmentation fault

1 0

[ooni-probe/develop] Set the default socket timeout to 30 seconds.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 7daa7f2208542e55c617e542f0be322f52d103e4 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:02:24 2013 +0000 Set the default socket timeout to 30 seconds. --- nettests/experimental/tls_handshake.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 8bd4820..2c11122 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -141,9 +141,13 @@ class TLSHandshakeTest(nettest.NetTestCase): if getattr(config.advanced, 'default_timeout', None) is not None: self.timeout = config.advanced.default_timeout else: - timeout = 10 ## default the timeout to 10 seconds - socket.setdefaulttimeout(timeout) - self.timeout = struct.pack('ll', int(timeout), 0) + self.timeout = 30 ## default the timeout to 30 seconds + + ## xxx For debugging, set the socket timeout higher anyway: + self.timeout = 30 + + ## We have to set the default timeout on our sockets before creation: + socket.setdefaulttimeout(self.timeout) def splitInput(self, input): addr, port = input.strip().rsplit(':', 1)

1 0

[ooni-probe/develop] Fix the logic on detecting if we have a config setting.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit c0b2e453e20303780b941d90c54aa04fd8b6a1ec Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 03:56:25 2013 +0000 Fix the logic on detecting if we have a config setting. * We can't use hasattr(config.basic, 'foo') because this gives None if not set, which means the default in the test doesn't get set. Instead, we want to use: if getattr(config.basic, 'foo', None) is not None: --- nettests/experimental/tls_handshake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index c57545c..8bd4820 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -138,8 +138,8 @@ class TLSHandshakeTest(nettest.NetTestCase): self.ciphers.append(line.strip()) self.ciphersuite = ":".join(self.ciphers) - if hasattr(config.advanced, 'default_timeout'): - timeout = config.advanced.default_timeout + if getattr(config.advanced, 'default_timeout', None) is not None: + self.timeout = config.advanced.default_timeout else: timeout = 10 ## default the timeout to 10 seconds socket.setdefaulttimeout(timeout)

1 0