[tor-commits] [ooni-probe/develop] Update tls_handshake.py path

isis at torproject.org isis at torproject.org
Wed Jun 26 01:02:11 UTC 2013


commit a7c05739177de3b8ee9ad7337e846ec810af7473
Author: aagbsn <aagbsn at extc.org>
Date:   Mon Jun 17 14:51:48 2013 +0200

    Update tls_handshake.py path
---
 data/nettests/experimental/tls_handshake.py |  809 +++++++++++++++++++++++++++
 nettests/experimental/tls_handshake.py      |  809 ---------------------------
 2 files changed, 809 insertions(+), 809 deletions(-)

diff --git a/data/nettests/experimental/tls_handshake.py b/data/nettests/experimental/tls_handshake.py
new file mode 100644
index 0000000..5da2e8b
--- /dev/null
+++ b/data/nettests/experimental/tls_handshake.py
@@ -0,0 +1,809 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+  tls_handshake.py
+  ----------------
+
+  This file contains test cases for determining if a TLS handshake completes
+  successfully, including ways to test if a TLS handshake which uses Mozilla
+  Firefox's current ciphersuite list completes. Rather than using Twisted and
+  OpenSSL's methods for automatically completing a handshake, which includes
+  setting all the parameters, such as the ciphersuite list, these tests use
+  non-blocking sockets and implement asychronous error-handling transversal of
+  OpenSSL's memory BIO state machine, allowing us to determine where and why a
+  handshake fails.
+
+  This network test is a complete rewrite of a pseudonymously contributed
+  script by Hackerberry Finn, in order to fit into OONI's core network tests.
+
+  @authors: Isis Agora Lovecruft <isis at torproject.org>
+  @license: see included LICENSE file
+  @copyright: © 2013 Isis Lovecruft, The Tor Project Inc.
+"""
+
+from socket import error   as socket_error
+from socket import timeout as socket_timeout
+from time   import sleep
+
+import os
+import socket
+import struct
+import sys
+import types
+
+import ipaddr
+import OpenSSL
+
+from OpenSSL                import SSL, crypto
+from twisted.internet       import defer, threads
+from twisted.python         import usage, failure
+
+from ooni       import nettest, config
+from ooni.utils import log
+from ooni.errors import InsufficientPrivileges
+
+## For a way to obtain the current version of Firefox's default ciphersuite
+## list, see https://trac.torproject.org/projects/tor/attachment/ticket/4744/
+## and the attached file "get_mozilla_files.py".
+##
+## Note, however, that doing so requires the source code to the version of
+## firefox that you wish to emulate.
+
+firefox_ciphers = ["ECDHE-ECDSA-AES256-SHA",
+                   "ECDHE-RSA-AES256-SHA",
+                   "DHE-RSA-CAMELLIA256-SHA",
+                   "DHE-DSS-CAMELLIA256-SHA",
+                   "DHE-RSA-AES256-SHA",
+                   "DHE-DSS-AES256-SHA",
+                   "ECDH-ECDSA-AES256-CBC-SHA",
+                   "ECDH-RSA-AES256-CBC-SHA",
+                   "CAMELLIA256-SHA",
+                   "AES256-SHA",
+                   "ECDHE-ECDSA-RC4-SHA",
+                   "ECDHE-ECDSA-AES128-SHA",
+                   "ECDHE-RSA-RC4-SHA",
+                   "ECDHE-RSA-AES128-SHA",
+                   "DHE-RSA-CAMELLIA128-SHA",
+                   "DHE-DSS-CAMELLIA128-SHA",]
+
+
+class SSLContextError(usage.UsageError):
+    """Raised when we're missing the SSL context method, or incompatible
+    contexts were provided. The SSL context method should be one of the
+    following:
+
+        :attr:`OpenSSL.SSL.SSLv2_METHOD <OpenSSL.SSL.SSLv2_METHOD>`
+        :attr:`OpenSSL.SSL.SSLv23_METHOD <OpenSSL.SSL.SSLv23_METHOD>`
+        :attr:`OpenSSL.SSL.SSLv3_METHOD <OpenSSL.SSL.SSLv3_METHOD>`
+        :attr:`OpenSSL.SSL.TLSv1_METHOD <OpenSSL.SSL.TLSv1_METHOD>`
+
+    To use the pre-defined error messages, construct with one of the
+    :meth:`SSLContextError.errors.keys <keys>` as the ``message`` string, like
+    so:
+
+        ``SSLContextError('NO_CONTEXT')``
+    """
+
+    #: Pre-defined error messages.
+    errors = {
+        'NO_CONTEXT': 'No SSL/TLS context chosen! Defaulting to TLSv1.',
+        'INCOMPATIBLE': str("Testing TLSv1 (option '--tls1') is incompatible "
+                            + "with testing SSL ('--ssl2' and '--ssl3')."),
+        'MISSING_SSLV2': str("Your version of OpenSSL was compiled without "
+                             + "support for SSLv2. This is normal on newer "
+                             + "versions of OpenSSL, but it means that you "
+                             + "will be unable to test SSLv2 handshakes "
+                             + "without recompiling OpenSSL."), }
+
+    def __init__(self, message):
+        if message in self.errors.keys():
+            message = self.errors[message]
+        super(usage.UsageError, self).__init__(message)
+
+class HostUnreachable(Exception):
+    """Raised when the host IP address appears to be unreachable."""
+    pass
+
+class ConnectionTimeout(Exception):
+    """Raised when we receive a :class:`socket.timeout <timeout>`, in order to
+    pass the Exception along to
+    :func:`TLSHandshakeTest.test_handshake.connectionFailed
+    <connectionFailed>`.
+    """
+    pass
+
+class HandshakeOptions(usage.Options):
+    """ :class:`usage.Options <Options>` parser for the tls-handshake test."""
+    optParameters = [
+        ['host', 'h', None,
+         'Remote host IP address (v4/v6) and port, i.e. "1.2.3.4:443"'],
+        ['port', 'p', None,
+         'Use this port for all hosts, regardless of port specified in file'],
+        ['ciphersuite', 'c', None ,
+         'File containing ciphersuite list, one per line'],]
+    optFlags = [
+        ['ssl2', '2', 'Use SSLv2'],
+        ['ssl3', '3', 'Use SSLv3'],
+        ['tls1', 't', 'Use TLSv1'],]
+
+class HandshakeTest(nettest.NetTestCase):
+    """An ooniprobe NetTestCase for determining if we can complete a TLS/SSL
+    handshake with a remote host.
+    """
+    name         = 'tls-handshake'
+    author       = 'Isis Lovecruft <isis at torproject.org>'
+    description  = 'A test to determing if we can complete a TLS hankshake.'
+    version      = '0.0.3'
+
+    requiresRoot = False
+    usageOptions = HandshakeOptions
+
+    host = None
+    inputFile = ['file', 'f', None, 'List of <IP>:<PORT>s to test']
+
+    #: Default SSL/TLS context method.
+    context = SSL.Context(SSL.TLSv1_METHOD)
+
+    def setUp(self, *args, **kwargs):
+        """Set defaults for a :class:`HandshakeTest <HandshakeTest>`."""
+
+        self.ciphers = list()
+
+        if self.localOptions:
+            options = self.localOptions
+
+            ## check that we're testing an IP:PORT, else exit gracefully:
+            if not (options['host']  or options['file']):
+                raise SystemExit("Need --host or --file!")
+            if options['host']:
+                self.host = options['host']
+
+            ## If no context was chosen, explain our default to the user:
+            if not (options['ssl2'] or options['ssl3'] or options['tls1']):
+                try: raise SSLContextError('NO_CONTEXT')
+                except SSLContextError as sce: log.err(sce.message)
+            else:
+                ## If incompatible contexts were chosen, inform the user:
+                if options['tls1'] and (options['ssl2'] or options['ssl3']):
+                    try: raise SSLContextError('INCOMPATIBLE')
+                    except SSLContextError as sce: log.err(sce.message)
+                    finally: log.msg('Defaulting to testing only TLSv1.')
+                elif options['ssl2']:
+                    try:
+                        if not options['ssl3']:
+                            context = SSL.Context(SSL.SSLv2_METHOD)
+                        else:
+                            context = SSL.Context(SSL.SSLv23_METHOD)
+                    except ValueError as ve:
+                        log.err(ve.message)
+                        try: raise SSLContextError('MISSING_SSLV2')
+                        except SSLContextError as sce:
+                            log.err(sce.message)
+                            log.msg("Falling back to testing only TLSv1.")
+                            context = SSL.Context(SSL.TLSv1_METHOD)
+                elif options['ssl3']:
+                    context = SSL.Context(SSL.SSLv3_METHOD)
+            ## finally, reset the context if the user's choice was okay:
+            if context: self.context = context
+
+            ## if we weren't given a file with a list of ciphersuites to use,
+            ## then use the firefox default list:
+            if not options['ciphersuite']:
+                self.ciphers = firefox_ciphers
+                log.msg('Using default Firefox ciphersuite list.')
+            else:
+                if os.path.isfile(options['ciphersuite']):
+                    log.msg('Using ciphersuite list from "%s"'
+                            % options['ciphersuite'])
+                    with open(options['ciphersuite']) as cipherfile:
+                        for line in cipherfile.readlines():
+                            self.ciphers.append(line.strip())
+            self.ciphersuite = ":".join(self.ciphers)
+
+        if getattr(config.advanced, 'default_timeout', None) is not None:
+            self.timeout = config.advanced.default_timeout
+        else:
+            self.timeout = 30   ## default the timeout to 30 seconds
+
+        ## xxx For debugging, set the socket timeout higher anyway:
+        self.timeout = 30
+
+        ## We have to set the default timeout on our sockets before creation:
+        socket.setdefaulttimeout(self.timeout)
+
+    def splitInput(self, input):
+        addr, port = input.strip().rsplit(':', 1)
+        if self.localOptions['port']:
+            port = self.localOptions['port']
+        return (str(addr), int(port))
+
+    def inputProcessor(self, file=None):
+        if self.host:
+            yield self.splitInput(self.host)
+        if os.path.isfile(file):
+            with open(file) as fh:
+                for line in fh.readlines():
+                    if line.startswith('#'):
+                        continue
+                    yield self.splitInput(line)
+
+    def buildSocket(self, addr):
+        global s
+        ip = ipaddr.IPAddress(addr) ## learn if we're IPv4 or IPv6
+        if ip.version == 4:
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        elif ip.version == 6:
+            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        return s
+
+    def getContext(self):
+        self.context.set_cipher_list(self.ciphersuite)
+        return self.context
+
+    @staticmethod
+    def getPeerCert(connection, get_chain=False):
+        """Get the PEM-encoded certificate or cert chain of the remote host.
+
+        :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
+        :param bool get_chain: If True, get the all certificates in the
+            chain. Otherwise, only get the remote host's certificate.
+        :returns: A PEM-encoded x509 certificate. If
+            :param:`getPeerCert.get_chain <get_chain>` is True, returns a list
+            of PEM-encoded x509 certificates.
+        """
+        if not get_chain:
+            x509_cert = connection.get_peer_certificate()
+            pem_cert = crypto.dump_certificate(crypto.FILETYPE_PEM, x509_cert)
+            return pem_cert
+        else:
+            cert_chain = []
+            x509_cert_chain = connection.get_peer_cert_chain()
+            for x509_cert in x509_cert_chain:
+                pem_cert = crypto.dump_certificate(crypto.FILETYPE_PEM,
+                                                   x509_cert)
+                cert_chain.append(pem_cert)
+            return cert_chain
+
+    @staticmethod
+    def getX509Name(certificate, get_components=False):
+        """Get the DER-encoded form of the Name fields of an X509 certificate.
+
+        @param certificate: A :class:`OpenSSL.crypto.X509Name` object.
+        @param get_components: A boolean. If True, returns a list of tuples of
+                               the (name, value)s of each Name field in the
+                               :param:`certificate`. If False, returns the DER
+                               encoded form of the Name fields of the
+                               :param:`certificate`.
+        """
+        x509_name = None
+
+        try:
+            assert isinstance(certificate, crypto.X509Name), \
+                "getX509Name takes OpenSSL.crypto.X509Name as first argument!"
+            x509_name = crypto.X509Name(certificate)
+        except AssertionError as ae:
+            log.err(ae)
+        except Exception as exc:
+            log.exception(exc)
+
+        if not x509_name is None:
+            if not get_components:
+                return x509_name.der()
+            else:
+                return x509_name.get_components()
+        else:
+            log.debug("getX509Name: got None for ivar x509_name")
+
+    @staticmethod
+    def getPublicKey(key):
+        """Get the PEM-encoded format of a host certificate's public key.
+
+        :param key: A :class:`OpenSSL.crypto.PKey <crypto.PKey>` object.
+        """
+        try:
+            assert isinstance(key, crypto.PKey), \
+                "getPublicKey expects type OpenSSL.crypto.PKey for parameter key"
+        except AssertionError as ae:
+            log.err(ae)
+        else:
+            pubkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, key)
+            return pubkey
+
+    def test_handshake(self):
+        """xxx fill me in"""
+
+        def makeConnection(host):
+            """Create a socket to the remote host's IP address, then get the
+            TLS/SSL context method and ciphersuite list. Lastly, initiate a
+            connection to the host.
+
+            :param tuple host: A tuple of the remote host's IP address as a
+                string, and an integer specifying the remote host port, i.e.
+                ('1.1.1.1',443)
+            :raises: :exc:`ConnectionTimeout` if the socket timed out.
+            :returns: A :class:`OpenSSL.SSL.Connection <Connection>`.
+            """
+            addr, port = host
+            sckt = self.buildSocket(addr)
+            context = self.getContext()
+            connection = SSL.Connection(context, sckt)
+            try:
+               connection.connect(host)
+            except socket_timeout as stmo:
+               error = ConnectionTimeout(stmo.message)
+               return failure.Failure(error)
+            else:
+               return connection
+
+        def connectionFailed(connection, host):
+            """Handle errors raised while attempting to create the socket and
+            :class:`OpenSSL.SSL.Connection <Connection>`, and setting the
+            TLS/SSL context.
+
+            :type connection: :exc:Exception
+            :param connection: The exception that was raised in
+                :func:`HandshakeTest.test_handshake.makeConnection
+                <makeConnection>`.
+            :param tuple host: A tuple of the host IP address as a string, and
+                an int specifying the host port, i.e. ('1.1.1.1', 443)
+            :rtype: :exc:Exception
+            :returns: The original exception.
+            """
+            addr, port = host
+
+            if not isinstance(connection, SSL.Connection):
+                if isinstance(connection, IOError):
+                    ## On some *nix distros, /dev/random is 0600 root:root and
+                    ## we get a permissions error when trying to read
+                    if connection.message.find("[Errno 13]"):
+                        raise InsufficientPrivileges(
+                            "%s" % connection.message.split("[Errno 13]", 1)[1])
+                elif isinstance(connection, socket_error):
+                    if connection.message.find("[Errno 101]"):
+                        raise HostUnreachableError(
+                            "Host unreachable: %s:%s" % (addr, port))
+                elif isinstance(connection, Exception):
+                    log.debug("connectionFailed: got Exception:")
+                    log.err("Connection failed with reason: %s"
+                            % connection.message)
+                else:
+                    log.err("Connection failed with reason: %s" % str(connection))
+
+            self.report['host'] = addr
+            self.report['port'] = port
+            self.report['state'] = 'CONNECTION_FAILED'
+
+            return connection
+
+        def connectionSucceeded(connection, host, timeout):
+            """If we have created a connection, set the socket options, and log
+            the connection state and peer name.
+
+            :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
+            :param tuple host: A tuple of the remote host's IP address as a
+                string, and an integer specifying the remote host port, i.e.
+                ('1.1.1.1',443)
+            """
+
+            ## xxx TODO to get this to work with a non-blocking socket, see how
+            ##     twisted.internet.tcp.Client handles socket objects.
+            connection.setblocking(1)
+
+            ## Set the timeout on the connection:
+            ##
+            ## We want to set SO_RCVTIMEO and SO_SNDTIMEO, which both are
+            ## defined in the socket option definitions in <sys/socket.h>, and
+            ## which both take as their value, according to socket(7), a
+            ## struct timeval, which is defined in the libc manual:
+            ## https://www.gnu.org/software/libc/manual/html_node/Elapsed-Time.html
+            timeval = struct.pack('ll', int(timeout), 0)
+            connection.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, timeval)
+            connection.setsockopt(socket.SOL_SOCKET, socket.SO_SNDTIMEO, timeval)
+
+            ## Set the connection state to client mode:
+            connection.set_connect_state()
+
+            peer_name, peer_port = connection.getpeername()
+            if peer_name:
+                log.msg("Connected to %s" % peer_name)
+            else:
+                log.debug("Couldn't get peer name from connection: %s" % host)
+                log.msg("Connected to %s" % host)
+            log.debug("Connection state: %s " % connection.state_string())
+
+            return connection
+
+        def connectionRenegotiate(connection, host, error_message):
+            """Handle a server-initiated SSL/TLS handshake renegotiation.
+
+            :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
+            :param tuple host: A tuple of the remote host's IP address as a
+                string, and an integer specifying the remote host port, i.e.
+                ('1.1.1.1',443)
+            """
+
+            log.msg("Server requested renegotiation from: %s" % host)
+            log.debug("Renegotiation reason: %s" % error_message)
+            log.debug("State: %s" % connection.state_string())
+
+            if connection.renegotiate():
+                log.debug("Renegotiation possible.")
+                log.msg("Retrying handshake with %s..." % host)
+                try:
+                    connection.do_handshake()
+                    while connection.renegotiate_pending():
+                        log.msg("Renegotiation with %s in progress..." % host)
+                        log.debug("State: %s" % connection.state_string())
+                        sleep(1)
+                    else:
+                        log.msg("Renegotiation with %s complete!" % host)
+                except SSL.WantReadError, wre:
+                    connection = handleWantRead(connection)
+                    log.debug("State: %s" % connection.state_string())
+                except SSL.WantWriteError, wwe:
+                    connection = handleWantWrite(connection)
+                    log.debug("State: %s" % connection.state_string())
+            return connection
+
+        def connectionShutdown(connection, host):
+            """Handle shutting down a :class:`OpenSSL.SSL.Connection
+            <Connection>`, including correct handling of halfway shutdown
+            connections.
+
+            Calls to :meth:`OpenSSL.SSL.Connection.shutdown
+            <Connection.shutdown()>` return a boolean value -- if the
+            connection is already shutdown, it returns True, else it returns
+            false. Thus we loop through a block which detects if the connection
+            is an a partial shutdown state and corrects that if that is the
+            case, else it waits for one second, then attempts shutting down the
+            connection again.
+
+            Detection of a partial shutdown state is done through
+            :meth:`OpenSSL.SSL.Connection.get_shutdown
+            <Connection.get_shutdown()>` which queries OpenSSL for a bitvector
+            of the server and client shutdown states. For example, the binary
+            string '0b00' is an open connection, and '0b10' is a partially
+            closed connection that has been shutdown on the serverside.
+
+            :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
+            :param tuple host: A tuple of the remote host's IP address as a
+                string, and an integer specifying the remote host port, i.e.
+                ('1.1.1.1',443)
+            """
+
+            peername, peerport = host
+
+            if isinstance(connection, SSL.Connection):
+                log.msg("Closing connection to %s:%d..." % (peername, peerport))
+                while not connection.shutdown():
+                    ## if the connection is halfway shutdown, we have to
+                    ## wait for a ZeroReturnError on connection.recv():
+                    if (bin(connection.get_shutdown()) == '0b01') \
+                            or (bin(connection.get_shutdown()) == '0b10'):
+                        try:
+                            _read_buffer = connection.pending()
+                            connection.recv(_read_buffer)
+                        except SSL.ZeroReturnError, zre: continue
+                    else:
+                        sleep(1)
+                else:
+                    log.msg("Closed connection to %s:%d"
+                            % (peername, peerport))
+            elif isinstance(connection, types.NoneType):
+                log.debug("connectionShutdown: got NoneType for connection")
+                return
+            else:
+                log.debug("connectionShutdown: expected connection, got %r"
+                          % connection.__repr__())
+
+            return connection
+
+        def handleWantRead(connection):
+            """From OpenSSL memory BIO documentation on ssl_read():
+
+                If the underlying BIO is blocking, SSL_read() will only
+                return, once the read operation has been finished or an error
+                occurred, except when a renegotiation take place, in which
+                case a SSL_ERROR_WANT_READ may occur. This behaviour can be
+                controlled with the SSL_MODE_AUTO_RETRY flag of the
+                SSL_CTX_set_mode(3) call.
+
+                If the underlying BIO is non-blocking, SSL_read() will also
+                return when the underlying BIO could not satisfy the needs of
+                SSL_read() to continue the operation. In this case a call to
+                SSL_get_error(3) with the return value of SSL_read() will
+                yield SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE. As at any
+                time a re-negotiation is possible, a call to SSL_read() can
+                also cause write operations!  The calling process then must
+                repeat the call after taking appropriate action to satisfy the
+                needs of SSL_read(). The action depends on the underlying
+                BIO. When using a non-blocking socket, nothing is to be done,
+                but select() can be used to check for the required condition.
+
+            And from the OpenSSL memory BIO documentation on ssl_get_error():
+
+                SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE
+
+                The operation did not complete; the same TLS/SSL I/O function
+                should be called again later. If, by then, the underlying BIO
+                has data available for reading (if the result code is
+                SSL_ERROR_WANT_READ) or allows writing data
+                (SSL_ERROR_WANT_WRITE), then some TLS/SSL protocol progress
+                will take place, i.e. at least part of an TLS/SSL record will
+                be read or written. Note that the retry may again lead to a
+                SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE condition. There
+                is no fixed upper limit for the number of iterations that may
+                be necessary until progress becomes visible at application
+                protocol level.
+
+                For socket BIOs (e.g. when SSL_set_fd() was used), select() or
+                poll() on the underlying socket can be used to find out when
+                the TLS/SSL I/O function should be retried.
+
+                Caveat: Any TLS/SSL I/O function can lead to either of
+                SSL_ERROR_WANT_READ and SSL_ERROR_WANT_WRITE. In particular,
+                SSL_read() or SSL_peek() may want to write data and
+                SSL_write() may want to read data. This is mainly because
+                TLS/SSL handshakes may occur at any time during the protocol
+                (initiated by either the client or the server); SSL_read(),
+                SSL_peek(), and SSL_write() will handle any pending
+                handshakes.
+
+            Also, see http://stackoverflow.com/q/3952104
+            """
+            try:
+                while connection.want_read():
+                    self.state = connection.state_string()
+                    log.debug("Connection to %s HAS want_read" % host)
+                    _read_buffer = connection.pending()
+                    log.debug("Rereading %d bytes..." % _read_buffer)
+                    sleep(1)
+                    rereceived = connection.recv(int(_read_buffer))
+                    log.debug("Received %d bytes" % rereceived)
+                    log.debug("State: %s" % connection.state_string())
+                else:
+                    self.state = connection.state_string()
+                    peername, peerport = connection.getpeername()
+                    log.debug("Connection to %s:%s DOES NOT HAVE want_read"
+                              % (peername, peerport))
+                    log.debug("State: %s" % connection.state_string())
+            except SSL.WantWriteError, wwe:
+                self.state = connection.state_string()
+                log.debug("Got WantWriteError while handling want_read")
+                log.debug("WantWriteError: %s" % wwe.message)
+                log.debug("Switching to handleWantWrite()...")
+                handleWantWrite(connection)
+            return connection
+
+        def handleWantWrite(connection):
+            """See :func:HandshakeTest.test_hanshake.handleWantRead """
+            try:
+                while connection.want_write():
+                    self.state = connection.state_string()
+                    log.debug("Connection to %s HAS want_write" % host)
+                    sleep(1)
+                    resent = connection.send("o\r\n")
+                    log.debug("Sent: %d" % resent)
+                    log.debug("State: %s" % connection.state_string())
+            except SSL.WantReadError, wre:
+                self.state = connection.state_string()
+                log.debug("Got WantReadError while handling want_write")
+                log.debug("WantReadError: %s" % wre.message)
+                log.debug("Switching to handleWantRead()...")
+                handleWantRead(connection)
+            return connection
+
+        def doHandshake(connection):
+            """Attempt a TLS/SSL handshake with the host.
+
+            If, after the first attempt at handshaking, OpenSSL's memory BIO
+            state machine does not report success, then try reading and
+            writing from the connection, and handle any SSL_ERROR_WANT_READ or
+            SSL_ERROR_WANT_WRITE which occurs.
+
+            If multiple want_reads occur, then try renegotiation with the
+            host, and start over. If multiple want_writes occur, then it is
+            possible that the connection has timed out, and move on to the
+            connectionShutdown step.
+
+            :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
+            :ivar peername: The host IP address, as reported by
+                :meth:`Connection.getpeername <connection.getpeername()>`.
+            :ivar peerport: The host port, reported by
+                :meth:`Connection.getpeername <connection.getpeername()>`.
+            :ivar int sent: The number of bytes sent to to the remote host.
+            :ivar int received: The number of bytes received from the remote
+                                host.
+            :ivar int _read_buffer: The max bytes that can be read from the
+                                    connection.
+            :returns: The :param:`doHandshake.connection <connection>` with
+                      handshake completed, else the unhandled error that was
+                      raised.
+            """
+            peername, peerport = connection.getpeername()
+
+            try:
+                log.msg("Attempting handshake: %s" % peername)
+                connection.do_handshake()
+            except OpenSSL.SSL.WantReadError() as wre:
+                self.state = connection.state_string()
+                log.debug("Handshake state: %s" % self.state)
+                log.debug("doHandshake: WantReadError on first handshake attempt.")
+                connection = handleWantRead(connection)
+            except OpenSSL.SSL.WantWriteError() as wwe:
+                self.state = connection.state_string()
+                log.debug("Handshake state: %s" % self.state)
+                log.debug("doHandshake: WantWriteError on first handshake attempt.")
+                connection = handleWantWrite(connection)
+            else:
+                self.state = connection.state_string()
+
+            if self.state == 'SSL negotiation finished successfully':
+                ## jump to handshakeSuccessful and get certchain
+                return connection
+            else:
+                sent = connection.send("o\r\n")
+                self.state = connection.state_string()
+                log.debug("Handshake state: %s" % self.state)
+                log.debug("Transmitted %d bytes" % sent)
+
+                _read_buffer = connection.pending()
+                log.debug("Max bytes in receive buffer: %d" % _read_buffer)
+
+                try:
+                    received = connection.recv(int(_read_buffer))
+                except SSL.WantReadError, wre:
+                    if connection.want_read():
+                        self.state = connection.state_string()
+                        connection = handleWantRead(connection)
+                    else:
+                        ## if we still have an SSL_ERROR_WANT_READ, then try to
+                        ## renegotiate
+                        self.state = connection.state_string()
+                        connection = connectionRenegotiate(connection,
+                                                           connection.getpeername(),
+                                                           wre.message)
+                except SSL.WantWriteError, wwe:
+                    self.state = connection.state_string()
+                    log.debug("Handshake state: %s" % self.state)
+                    if connection.want_write():
+                        connection = handleWantWrite(connection)
+                    else:
+                        raise ConnectionTimeout("Connection to %s:%d timed out."
+                                                % (peername, peerport))
+                else:
+                    log.msg("Received: %s" % received)
+                    self.state = connection.state_string()
+                    log.debug("Handshake state: %s" % self.state)
+
+            return connection
+
+        def handshakeSucceeded(connection):
+            """Get the details from the server certificate, cert chain, and
+            server ciphersuite list, and put them in our report.
+
+            WARNING: do *not* do this:
+            >>> server_cert.get_pubkey()
+                <OpenSSL.crypto.PKey at 0x4985d28>
+            >>> pk = server_cert.get_pubkey()
+            >>> pk.check()
+                Segmentation fault
+
+            :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
+            :returns: :param:`handshakeSucceeded.connection <connection>`.
+            """
+            host, port = connection.getpeername()
+            log.msg("Handshake with %s:%d successful!" % (host, port))
+
+            server_cert = self.getPeerCert(connection)
+            server_cert_chain = self.getPeerCert(connection, get_chain=True)
+
+            renegotiations = connection.total_renegotiations()
+            cipher_list    = connection.get_cipher_list()
+            session_key    = connection.master_key()
+            rawcert        = connection.get_peer_certificate()
+            ## xxx TODO this hash needs to be formatted as SHA1, not long
+            cert_subj_hash = rawcert.subject_name_hash()
+            cert_serial    = rawcert.get_serial_number()
+            cert_sig_algo  = rawcert.get_signature_algorithm()
+            cert_subject   = self.getX509Name(rawcert.get_subject(),
+                                              get_components=True)
+            cert_issuer    = self.getX509Name(rawcert.get_issuer(),
+                                              get_components=True)
+            cert_pubkey    = self.getPublicKey(rawcert.get_pubkey())
+
+            self.report['host'] = host
+            self.report['port'] = port
+            self.report['state'] = self.state
+            self.report['renegotiations'] = renegotiations
+            self.report['server_cert'] = server_cert
+            self.report['server_cert_chain'] = \
+                ''.join([cert for cert in server_cert_chain])
+            self.report['server_ciphersuite'] = cipher_list
+            self.report['cert_subject'] = cert_subject
+            self.report['cert_subj_hash'] = cert_subj_hash
+            self.report['cert_issuer'] = cert_issuer
+            self.report['cert_public_key'] = cert_pubkey
+            self.report['cert_serial_no'] = cert_serial
+            self.report['cert_sig_algo'] = cert_sig_algo
+            ## The session's master key is only valid for that session, and
+            ## will allow us to decrypt any packet captures (if they were
+            ## collected). Because we are not requesting URLs, only host:port
+            ## (which would be visible in pcaps anyway, since the FQDN is
+            ## never encrypted) I do not see a way for this to log any user or
+            ## identifying information. Correct me if I'm wrong.
+            self.report['session_key'] = session_key
+
+            log.msg("Server certificate:\n\n%s" % server_cert)
+            log.msg("Server certificate chain:\n\n%s"
+                    % ''.join([cert for cert in server_cert_chain]))
+            log.msg("Negotiated ciphersuite:\n%s"
+                    % '\n\t'.join([cipher for cipher in cipher_list]))
+            log.msg("Certificate subject: %s" % cert_subject)
+            log.msg("Certificate subject hash: %d" % cert_subj_hash)
+            log.msg("Certificate issuer: %s" % cert_issuer)
+            log.msg("Certificate public key:\n\n%s" % cert_pubkey)
+            log.msg("Certificate signature algorithm: %s" % cert_sig_algo)
+            log.msg("Certificate serial number: %s" % cert_serial)
+            log.msg("Total renegotiations: %d" % renegotiations)
+
+            return connection
+
+        def handshakeFailed(connection, host):
+            """Handle a failed handshake attempt and report the failure reason.
+
+            :type connection: :class:`twisted.python.failure.Failure <Failure>`
+                or :exc:Exception
+            :param connection: The failed connection.
+            :param tuple host: A tuple of the remote host's IP address as a
+                string, and an integer specifying the remote host port, i.e.
+                ('1.1.1.1',443)
+            :returns: None
+            """
+            addr, port = host
+            log.msg("Handshake with %s:%d failed!" % host)
+
+            self.report['host'] = host
+            self.report['port'] = port
+
+            if isinstance(connection, Exception) \
+                    or isinstance(connection, ConnectionTimeout):
+                log.msg("Handshake failed with reason: %s" % connection.message)
+                self.report['state'] = connection.message
+            elif isinstance(connection, failure.Failure):
+                log.msg("Handshake failed with reason: Socket %s"
+                        % connection.getErrorMessage())
+                self.report['state'] = connection.getErrorMessage()
+                ctmo = connection.trap(ConnectionTimeout)
+                if ctmo == ConnectionTimeout:
+                    connection.cleanFailure()
+            else:
+                log.msg("Handshake failed with reason: %s" % str(connection))
+                if not 'state' in self.report.keys():
+                    self.report['state'] = str(connection)
+
+            return None
+
+        def deferMakeConnection(host):
+            return threads.deferToThread(makeConnection, self.input)
+
+        if self.host and not self.input:
+            self.input = self.splitInput(self.host)
+        log.msg("Beginning handshake test for %s:%s" % self.input)
+
+        connection = deferMakeConnection(self.input)
+        connection.addCallbacks(connectionSucceeded, connectionFailed,
+                                callbackArgs=[self.input, self.timeout],
+                                errbackArgs=[self.input])
+
+        handshake = defer.Deferred()
+        handshake.addCallback(doHandshake)
+        handshake.addCallbacks(handshakeSucceeded, handshakeFailed,
+                               errbackArgs=[self.input])
+
+        connection.chainDeferred(handshake)
+        connection.addCallbacks(connectionShutdown, defer.passthru,
+                                callbackArgs=[self.input])
+        connection.addBoth(log.exception)
+
+        return connection
diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py
deleted file mode 100644
index 5da2e8b..0000000
--- a/nettests/experimental/tls_handshake.py
+++ /dev/null
@@ -1,809 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-"""
-  tls_handshake.py
-  ----------------
-
-  This file contains test cases for determining if a TLS handshake completes
-  successfully, including ways to test if a TLS handshake which uses Mozilla
-  Firefox's current ciphersuite list completes. Rather than using Twisted and
-  OpenSSL's methods for automatically completing a handshake, which includes
-  setting all the parameters, such as the ciphersuite list, these tests use
-  non-blocking sockets and implement asychronous error-handling transversal of
-  OpenSSL's memory BIO state machine, allowing us to determine where and why a
-  handshake fails.
-
-  This network test is a complete rewrite of a pseudonymously contributed
-  script by Hackerberry Finn, in order to fit into OONI's core network tests.
-
-  @authors: Isis Agora Lovecruft <isis at torproject.org>
-  @license: see included LICENSE file
-  @copyright: © 2013 Isis Lovecruft, The Tor Project Inc.
-"""
-
-from socket import error   as socket_error
-from socket import timeout as socket_timeout
-from time   import sleep
-
-import os
-import socket
-import struct
-import sys
-import types
-
-import ipaddr
-import OpenSSL
-
-from OpenSSL                import SSL, crypto
-from twisted.internet       import defer, threads
-from twisted.python         import usage, failure
-
-from ooni       import nettest, config
-from ooni.utils import log
-from ooni.errors import InsufficientPrivileges
-
-## For a way to obtain the current version of Firefox's default ciphersuite
-## list, see https://trac.torproject.org/projects/tor/attachment/ticket/4744/
-## and the attached file "get_mozilla_files.py".
-##
-## Note, however, that doing so requires the source code to the version of
-## firefox that you wish to emulate.
-
-firefox_ciphers = ["ECDHE-ECDSA-AES256-SHA",
-                   "ECDHE-RSA-AES256-SHA",
-                   "DHE-RSA-CAMELLIA256-SHA",
-                   "DHE-DSS-CAMELLIA256-SHA",
-                   "DHE-RSA-AES256-SHA",
-                   "DHE-DSS-AES256-SHA",
-                   "ECDH-ECDSA-AES256-CBC-SHA",
-                   "ECDH-RSA-AES256-CBC-SHA",
-                   "CAMELLIA256-SHA",
-                   "AES256-SHA",
-                   "ECDHE-ECDSA-RC4-SHA",
-                   "ECDHE-ECDSA-AES128-SHA",
-                   "ECDHE-RSA-RC4-SHA",
-                   "ECDHE-RSA-AES128-SHA",
-                   "DHE-RSA-CAMELLIA128-SHA",
-                   "DHE-DSS-CAMELLIA128-SHA",]
-
-
-class SSLContextError(usage.UsageError):
-    """Raised when we're missing the SSL context method, or incompatible
-    contexts were provided. The SSL context method should be one of the
-    following:
-
-        :attr:`OpenSSL.SSL.SSLv2_METHOD <OpenSSL.SSL.SSLv2_METHOD>`
-        :attr:`OpenSSL.SSL.SSLv23_METHOD <OpenSSL.SSL.SSLv23_METHOD>`
-        :attr:`OpenSSL.SSL.SSLv3_METHOD <OpenSSL.SSL.SSLv3_METHOD>`
-        :attr:`OpenSSL.SSL.TLSv1_METHOD <OpenSSL.SSL.TLSv1_METHOD>`
-
-    To use the pre-defined error messages, construct with one of the
-    :meth:`SSLContextError.errors.keys <keys>` as the ``message`` string, like
-    so:
-
-        ``SSLContextError('NO_CONTEXT')``
-    """
-
-    #: Pre-defined error messages.
-    errors = {
-        'NO_CONTEXT': 'No SSL/TLS context chosen! Defaulting to TLSv1.',
-        'INCOMPATIBLE': str("Testing TLSv1 (option '--tls1') is incompatible "
-                            + "with testing SSL ('--ssl2' and '--ssl3')."),
-        'MISSING_SSLV2': str("Your version of OpenSSL was compiled without "
-                             + "support for SSLv2. This is normal on newer "
-                             + "versions of OpenSSL, but it means that you "
-                             + "will be unable to test SSLv2 handshakes "
-                             + "without recompiling OpenSSL."), }
-
-    def __init__(self, message):
-        if message in self.errors.keys():
-            message = self.errors[message]
-        super(usage.UsageError, self).__init__(message)
-
-class HostUnreachable(Exception):
-    """Raised when the host IP address appears to be unreachable."""
-    pass
-
-class ConnectionTimeout(Exception):
-    """Raised when we receive a :class:`socket.timeout <timeout>`, in order to
-    pass the Exception along to
-    :func:`TLSHandshakeTest.test_handshake.connectionFailed
-    <connectionFailed>`.
-    """
-    pass
-
-class HandshakeOptions(usage.Options):
-    """ :class:`usage.Options <Options>` parser for the tls-handshake test."""
-    optParameters = [
-        ['host', 'h', None,
-         'Remote host IP address (v4/v6) and port, i.e. "1.2.3.4:443"'],
-        ['port', 'p', None,
-         'Use this port for all hosts, regardless of port specified in file'],
-        ['ciphersuite', 'c', None ,
-         'File containing ciphersuite list, one per line'],]
-    optFlags = [
-        ['ssl2', '2', 'Use SSLv2'],
-        ['ssl3', '3', 'Use SSLv3'],
-        ['tls1', 't', 'Use TLSv1'],]
-
-class HandshakeTest(nettest.NetTestCase):
-    """An ooniprobe NetTestCase for determining if we can complete a TLS/SSL
-    handshake with a remote host.
-    """
-    name         = 'tls-handshake'
-    author       = 'Isis Lovecruft <isis at torproject.org>'
-    description  = 'A test to determing if we can complete a TLS hankshake.'
-    version      = '0.0.3'
-
-    requiresRoot = False
-    usageOptions = HandshakeOptions
-
-    host = None
-    inputFile = ['file', 'f', None, 'List of <IP>:<PORT>s to test']
-
-    #: Default SSL/TLS context method.
-    context = SSL.Context(SSL.TLSv1_METHOD)
-
-    def setUp(self, *args, **kwargs):
-        """Set defaults for a :class:`HandshakeTest <HandshakeTest>`."""
-
-        self.ciphers = list()
-
-        if self.localOptions:
-            options = self.localOptions
-
-            ## check that we're testing an IP:PORT, else exit gracefully:
-            if not (options['host']  or options['file']):
-                raise SystemExit("Need --host or --file!")
-            if options['host']:
-                self.host = options['host']
-
-            ## If no context was chosen, explain our default to the user:
-            if not (options['ssl2'] or options['ssl3'] or options['tls1']):
-                try: raise SSLContextError('NO_CONTEXT')
-                except SSLContextError as sce: log.err(sce.message)
-            else:
-                ## If incompatible contexts were chosen, inform the user:
-                if options['tls1'] and (options['ssl2'] or options['ssl3']):
-                    try: raise SSLContextError('INCOMPATIBLE')
-                    except SSLContextError as sce: log.err(sce.message)
-                    finally: log.msg('Defaulting to testing only TLSv1.')
-                elif options['ssl2']:
-                    try:
-                        if not options['ssl3']:
-                            context = SSL.Context(SSL.SSLv2_METHOD)
-                        else:
-                            context = SSL.Context(SSL.SSLv23_METHOD)
-                    except ValueError as ve:
-                        log.err(ve.message)
-                        try: raise SSLContextError('MISSING_SSLV2')
-                        except SSLContextError as sce:
-                            log.err(sce.message)
-                            log.msg("Falling back to testing only TLSv1.")
-                            context = SSL.Context(SSL.TLSv1_METHOD)
-                elif options['ssl3']:
-                    context = SSL.Context(SSL.SSLv3_METHOD)
-            ## finally, reset the context if the user's choice was okay:
-            if context: self.context = context
-
-            ## if we weren't given a file with a list of ciphersuites to use,
-            ## then use the firefox default list:
-            if not options['ciphersuite']:
-                self.ciphers = firefox_ciphers
-                log.msg('Using default Firefox ciphersuite list.')
-            else:
-                if os.path.isfile(options['ciphersuite']):
-                    log.msg('Using ciphersuite list from "%s"'
-                            % options['ciphersuite'])
-                    with open(options['ciphersuite']) as cipherfile:
-                        for line in cipherfile.readlines():
-                            self.ciphers.append(line.strip())
-            self.ciphersuite = ":".join(self.ciphers)
-
-        if getattr(config.advanced, 'default_timeout', None) is not None:
-            self.timeout = config.advanced.default_timeout
-        else:
-            self.timeout = 30   ## default the timeout to 30 seconds
-
-        ## xxx For debugging, set the socket timeout higher anyway:
-        self.timeout = 30
-
-        ## We have to set the default timeout on our sockets before creation:
-        socket.setdefaulttimeout(self.timeout)
-
-    def splitInput(self, input):
-        addr, port = input.strip().rsplit(':', 1)
-        if self.localOptions['port']:
-            port = self.localOptions['port']
-        return (str(addr), int(port))
-
-    def inputProcessor(self, file=None):
-        if self.host:
-            yield self.splitInput(self.host)
-        if os.path.isfile(file):
-            with open(file) as fh:
-                for line in fh.readlines():
-                    if line.startswith('#'):
-                        continue
-                    yield self.splitInput(line)
-
-    def buildSocket(self, addr):
-        global s
-        ip = ipaddr.IPAddress(addr) ## learn if we're IPv4 or IPv6
-        if ip.version == 4:
-            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        elif ip.version == 6:
-            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-        return s
-
-    def getContext(self):
-        self.context.set_cipher_list(self.ciphersuite)
-        return self.context
-
-    @staticmethod
-    def getPeerCert(connection, get_chain=False):
-        """Get the PEM-encoded certificate or cert chain of the remote host.
-
-        :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
-        :param bool get_chain: If True, get the all certificates in the
-            chain. Otherwise, only get the remote host's certificate.
-        :returns: A PEM-encoded x509 certificate. If
-            :param:`getPeerCert.get_chain <get_chain>` is True, returns a list
-            of PEM-encoded x509 certificates.
-        """
-        if not get_chain:
-            x509_cert = connection.get_peer_certificate()
-            pem_cert = crypto.dump_certificate(crypto.FILETYPE_PEM, x509_cert)
-            return pem_cert
-        else:
-            cert_chain = []
-            x509_cert_chain = connection.get_peer_cert_chain()
-            for x509_cert in x509_cert_chain:
-                pem_cert = crypto.dump_certificate(crypto.FILETYPE_PEM,
-                                                   x509_cert)
-                cert_chain.append(pem_cert)
-            return cert_chain
-
-    @staticmethod
-    def getX509Name(certificate, get_components=False):
-        """Get the DER-encoded form of the Name fields of an X509 certificate.
-
-        @param certificate: A :class:`OpenSSL.crypto.X509Name` object.
-        @param get_components: A boolean. If True, returns a list of tuples of
-                               the (name, value)s of each Name field in the
-                               :param:`certificate`. If False, returns the DER
-                               encoded form of the Name fields of the
-                               :param:`certificate`.
-        """
-        x509_name = None
-
-        try:
-            assert isinstance(certificate, crypto.X509Name), \
-                "getX509Name takes OpenSSL.crypto.X509Name as first argument!"
-            x509_name = crypto.X509Name(certificate)
-        except AssertionError as ae:
-            log.err(ae)
-        except Exception as exc:
-            log.exception(exc)
-
-        if not x509_name is None:
-            if not get_components:
-                return x509_name.der()
-            else:
-                return x509_name.get_components()
-        else:
-            log.debug("getX509Name: got None for ivar x509_name")
-
-    @staticmethod
-    def getPublicKey(key):
-        """Get the PEM-encoded format of a host certificate's public key.
-
-        :param key: A :class:`OpenSSL.crypto.PKey <crypto.PKey>` object.
-        """
-        try:
-            assert isinstance(key, crypto.PKey), \
-                "getPublicKey expects type OpenSSL.crypto.PKey for parameter key"
-        except AssertionError as ae:
-            log.err(ae)
-        else:
-            pubkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, key)
-            return pubkey
-
-    def test_handshake(self):
-        """xxx fill me in"""
-
-        def makeConnection(host):
-            """Create a socket to the remote host's IP address, then get the
-            TLS/SSL context method and ciphersuite list. Lastly, initiate a
-            connection to the host.
-
-            :param tuple host: A tuple of the remote host's IP address as a
-                string, and an integer specifying the remote host port, i.e.
-                ('1.1.1.1',443)
-            :raises: :exc:`ConnectionTimeout` if the socket timed out.
-            :returns: A :class:`OpenSSL.SSL.Connection <Connection>`.
-            """
-            addr, port = host
-            sckt = self.buildSocket(addr)
-            context = self.getContext()
-            connection = SSL.Connection(context, sckt)
-            try:
-               connection.connect(host)
-            except socket_timeout as stmo:
-               error = ConnectionTimeout(stmo.message)
-               return failure.Failure(error)
-            else:
-               return connection
-
-        def connectionFailed(connection, host):
-            """Handle errors raised while attempting to create the socket and
-            :class:`OpenSSL.SSL.Connection <Connection>`, and setting the
-            TLS/SSL context.
-
-            :type connection: :exc:Exception
-            :param connection: The exception that was raised in
-                :func:`HandshakeTest.test_handshake.makeConnection
-                <makeConnection>`.
-            :param tuple host: A tuple of the host IP address as a string, and
-                an int specifying the host port, i.e. ('1.1.1.1', 443)
-            :rtype: :exc:Exception
-            :returns: The original exception.
-            """
-            addr, port = host
-
-            if not isinstance(connection, SSL.Connection):
-                if isinstance(connection, IOError):
-                    ## On some *nix distros, /dev/random is 0600 root:root and
-                    ## we get a permissions error when trying to read
-                    if connection.message.find("[Errno 13]"):
-                        raise InsufficientPrivileges(
-                            "%s" % connection.message.split("[Errno 13]", 1)[1])
-                elif isinstance(connection, socket_error):
-                    if connection.message.find("[Errno 101]"):
-                        raise HostUnreachableError(
-                            "Host unreachable: %s:%s" % (addr, port))
-                elif isinstance(connection, Exception):
-                    log.debug("connectionFailed: got Exception:")
-                    log.err("Connection failed with reason: %s"
-                            % connection.message)
-                else:
-                    log.err("Connection failed with reason: %s" % str(connection))
-
-            self.report['host'] = addr
-            self.report['port'] = port
-            self.report['state'] = 'CONNECTION_FAILED'
-
-            return connection
-
-        def connectionSucceeded(connection, host, timeout):
-            """If we have created a connection, set the socket options, and log
-            the connection state and peer name.
-
-            :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
-            :param tuple host: A tuple of the remote host's IP address as a
-                string, and an integer specifying the remote host port, i.e.
-                ('1.1.1.1',443)
-            """
-
-            ## xxx TODO to get this to work with a non-blocking socket, see how
-            ##     twisted.internet.tcp.Client handles socket objects.
-            connection.setblocking(1)
-
-            ## Set the timeout on the connection:
-            ##
-            ## We want to set SO_RCVTIMEO and SO_SNDTIMEO, which both are
-            ## defined in the socket option definitions in <sys/socket.h>, and
-            ## which both take as their value, according to socket(7), a
-            ## struct timeval, which is defined in the libc manual:
-            ## https://www.gnu.org/software/libc/manual/html_node/Elapsed-Time.html
-            timeval = struct.pack('ll', int(timeout), 0)
-            connection.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, timeval)
-            connection.setsockopt(socket.SOL_SOCKET, socket.SO_SNDTIMEO, timeval)
-
-            ## Set the connection state to client mode:
-            connection.set_connect_state()
-
-            peer_name, peer_port = connection.getpeername()
-            if peer_name:
-                log.msg("Connected to %s" % peer_name)
-            else:
-                log.debug("Couldn't get peer name from connection: %s" % host)
-                log.msg("Connected to %s" % host)
-            log.debug("Connection state: %s " % connection.state_string())
-
-            return connection
-
-        def connectionRenegotiate(connection, host, error_message):
-            """Handle a server-initiated SSL/TLS handshake renegotiation.
-
-            :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
-            :param tuple host: A tuple of the remote host's IP address as a
-                string, and an integer specifying the remote host port, i.e.
-                ('1.1.1.1',443)
-            """
-
-            log.msg("Server requested renegotiation from: %s" % host)
-            log.debug("Renegotiation reason: %s" % error_message)
-            log.debug("State: %s" % connection.state_string())
-
-            if connection.renegotiate():
-                log.debug("Renegotiation possible.")
-                log.msg("Retrying handshake with %s..." % host)
-                try:
-                    connection.do_handshake()
-                    while connection.renegotiate_pending():
-                        log.msg("Renegotiation with %s in progress..." % host)
-                        log.debug("State: %s" % connection.state_string())
-                        sleep(1)
-                    else:
-                        log.msg("Renegotiation with %s complete!" % host)
-                except SSL.WantReadError, wre:
-                    connection = handleWantRead(connection)
-                    log.debug("State: %s" % connection.state_string())
-                except SSL.WantWriteError, wwe:
-                    connection = handleWantWrite(connection)
-                    log.debug("State: %s" % connection.state_string())
-            return connection
-
-        def connectionShutdown(connection, host):
-            """Handle shutting down a :class:`OpenSSL.SSL.Connection
-            <Connection>`, including correct handling of halfway shutdown
-            connections.
-
-            Calls to :meth:`OpenSSL.SSL.Connection.shutdown
-            <Connection.shutdown()>` return a boolean value -- if the
-            connection is already shutdown, it returns True, else it returns
-            false. Thus we loop through a block which detects if the connection
-            is an a partial shutdown state and corrects that if that is the
-            case, else it waits for one second, then attempts shutting down the
-            connection again.
-
-            Detection of a partial shutdown state is done through
-            :meth:`OpenSSL.SSL.Connection.get_shutdown
-            <Connection.get_shutdown()>` which queries OpenSSL for a bitvector
-            of the server and client shutdown states. For example, the binary
-            string '0b00' is an open connection, and '0b10' is a partially
-            closed connection that has been shutdown on the serverside.
-
-            :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
-            :param tuple host: A tuple of the remote host's IP address as a
-                string, and an integer specifying the remote host port, i.e.
-                ('1.1.1.1',443)
-            """
-
-            peername, peerport = host
-
-            if isinstance(connection, SSL.Connection):
-                log.msg("Closing connection to %s:%d..." % (peername, peerport))
-                while not connection.shutdown():
-                    ## if the connection is halfway shutdown, we have to
-                    ## wait for a ZeroReturnError on connection.recv():
-                    if (bin(connection.get_shutdown()) == '0b01') \
-                            or (bin(connection.get_shutdown()) == '0b10'):
-                        try:
-                            _read_buffer = connection.pending()
-                            connection.recv(_read_buffer)
-                        except SSL.ZeroReturnError, zre: continue
-                    else:
-                        sleep(1)
-                else:
-                    log.msg("Closed connection to %s:%d"
-                            % (peername, peerport))
-            elif isinstance(connection, types.NoneType):
-                log.debug("connectionShutdown: got NoneType for connection")
-                return
-            else:
-                log.debug("connectionShutdown: expected connection, got %r"
-                          % connection.__repr__())
-
-            return connection
-
-        def handleWantRead(connection):
-            """From OpenSSL memory BIO documentation on ssl_read():
-
-                If the underlying BIO is blocking, SSL_read() will only
-                return, once the read operation has been finished or an error
-                occurred, except when a renegotiation take place, in which
-                case a SSL_ERROR_WANT_READ may occur. This behaviour can be
-                controlled with the SSL_MODE_AUTO_RETRY flag of the
-                SSL_CTX_set_mode(3) call.
-
-                If the underlying BIO is non-blocking, SSL_read() will also
-                return when the underlying BIO could not satisfy the needs of
-                SSL_read() to continue the operation. In this case a call to
-                SSL_get_error(3) with the return value of SSL_read() will
-                yield SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE. As at any
-                time a re-negotiation is possible, a call to SSL_read() can
-                also cause write operations!  The calling process then must
-                repeat the call after taking appropriate action to satisfy the
-                needs of SSL_read(). The action depends on the underlying
-                BIO. When using a non-blocking socket, nothing is to be done,
-                but select() can be used to check for the required condition.
-
-            And from the OpenSSL memory BIO documentation on ssl_get_error():
-
-                SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE
-
-                The operation did not complete; the same TLS/SSL I/O function
-                should be called again later. If, by then, the underlying BIO
-                has data available for reading (if the result code is
-                SSL_ERROR_WANT_READ) or allows writing data
-                (SSL_ERROR_WANT_WRITE), then some TLS/SSL protocol progress
-                will take place, i.e. at least part of an TLS/SSL record will
-                be read or written. Note that the retry may again lead to a
-                SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE condition. There
-                is no fixed upper limit for the number of iterations that may
-                be necessary until progress becomes visible at application
-                protocol level.
-
-                For socket BIOs (e.g. when SSL_set_fd() was used), select() or
-                poll() on the underlying socket can be used to find out when
-                the TLS/SSL I/O function should be retried.
-
-                Caveat: Any TLS/SSL I/O function can lead to either of
-                SSL_ERROR_WANT_READ and SSL_ERROR_WANT_WRITE. In particular,
-                SSL_read() or SSL_peek() may want to write data and
-                SSL_write() may want to read data. This is mainly because
-                TLS/SSL handshakes may occur at any time during the protocol
-                (initiated by either the client or the server); SSL_read(),
-                SSL_peek(), and SSL_write() will handle any pending
-                handshakes.
-
-            Also, see http://stackoverflow.com/q/3952104
-            """
-            try:
-                while connection.want_read():
-                    self.state = connection.state_string()
-                    log.debug("Connection to %s HAS want_read" % host)
-                    _read_buffer = connection.pending()
-                    log.debug("Rereading %d bytes..." % _read_buffer)
-                    sleep(1)
-                    rereceived = connection.recv(int(_read_buffer))
-                    log.debug("Received %d bytes" % rereceived)
-                    log.debug("State: %s" % connection.state_string())
-                else:
-                    self.state = connection.state_string()
-                    peername, peerport = connection.getpeername()
-                    log.debug("Connection to %s:%s DOES NOT HAVE want_read"
-                              % (peername, peerport))
-                    log.debug("State: %s" % connection.state_string())
-            except SSL.WantWriteError, wwe:
-                self.state = connection.state_string()
-                log.debug("Got WantWriteError while handling want_read")
-                log.debug("WantWriteError: %s" % wwe.message)
-                log.debug("Switching to handleWantWrite()...")
-                handleWantWrite(connection)
-            return connection
-
-        def handleWantWrite(connection):
-            """See :func:HandshakeTest.test_hanshake.handleWantRead """
-            try:
-                while connection.want_write():
-                    self.state = connection.state_string()
-                    log.debug("Connection to %s HAS want_write" % host)
-                    sleep(1)
-                    resent = connection.send("o\r\n")
-                    log.debug("Sent: %d" % resent)
-                    log.debug("State: %s" % connection.state_string())
-            except SSL.WantReadError, wre:
-                self.state = connection.state_string()
-                log.debug("Got WantReadError while handling want_write")
-                log.debug("WantReadError: %s" % wre.message)
-                log.debug("Switching to handleWantRead()...")
-                handleWantRead(connection)
-            return connection
-
-        def doHandshake(connection):
-            """Attempt a TLS/SSL handshake with the host.
-
-            If, after the first attempt at handshaking, OpenSSL's memory BIO
-            state machine does not report success, then try reading and
-            writing from the connection, and handle any SSL_ERROR_WANT_READ or
-            SSL_ERROR_WANT_WRITE which occurs.
-
-            If multiple want_reads occur, then try renegotiation with the
-            host, and start over. If multiple want_writes occur, then it is
-            possible that the connection has timed out, and move on to the
-            connectionShutdown step.
-
-            :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
-            :ivar peername: The host IP address, as reported by
-                :meth:`Connection.getpeername <connection.getpeername()>`.
-            :ivar peerport: The host port, reported by
-                :meth:`Connection.getpeername <connection.getpeername()>`.
-            :ivar int sent: The number of bytes sent to to the remote host.
-            :ivar int received: The number of bytes received from the remote
-                                host.
-            :ivar int _read_buffer: The max bytes that can be read from the
-                                    connection.
-            :returns: The :param:`doHandshake.connection <connection>` with
-                      handshake completed, else the unhandled error that was
-                      raised.
-            """
-            peername, peerport = connection.getpeername()
-
-            try:
-                log.msg("Attempting handshake: %s" % peername)
-                connection.do_handshake()
-            except OpenSSL.SSL.WantReadError() as wre:
-                self.state = connection.state_string()
-                log.debug("Handshake state: %s" % self.state)
-                log.debug("doHandshake: WantReadError on first handshake attempt.")
-                connection = handleWantRead(connection)
-            except OpenSSL.SSL.WantWriteError() as wwe:
-                self.state = connection.state_string()
-                log.debug("Handshake state: %s" % self.state)
-                log.debug("doHandshake: WantWriteError on first handshake attempt.")
-                connection = handleWantWrite(connection)
-            else:
-                self.state = connection.state_string()
-
-            if self.state == 'SSL negotiation finished successfully':
-                ## jump to handshakeSuccessful and get certchain
-                return connection
-            else:
-                sent = connection.send("o\r\n")
-                self.state = connection.state_string()
-                log.debug("Handshake state: %s" % self.state)
-                log.debug("Transmitted %d bytes" % sent)
-
-                _read_buffer = connection.pending()
-                log.debug("Max bytes in receive buffer: %d" % _read_buffer)
-
-                try:
-                    received = connection.recv(int(_read_buffer))
-                except SSL.WantReadError, wre:
-                    if connection.want_read():
-                        self.state = connection.state_string()
-                        connection = handleWantRead(connection)
-                    else:
-                        ## if we still have an SSL_ERROR_WANT_READ, then try to
-                        ## renegotiate
-                        self.state = connection.state_string()
-                        connection = connectionRenegotiate(connection,
-                                                           connection.getpeername(),
-                                                           wre.message)
-                except SSL.WantWriteError, wwe:
-                    self.state = connection.state_string()
-                    log.debug("Handshake state: %s" % self.state)
-                    if connection.want_write():
-                        connection = handleWantWrite(connection)
-                    else:
-                        raise ConnectionTimeout("Connection to %s:%d timed out."
-                                                % (peername, peerport))
-                else:
-                    log.msg("Received: %s" % received)
-                    self.state = connection.state_string()
-                    log.debug("Handshake state: %s" % self.state)
-
-            return connection
-
-        def handshakeSucceeded(connection):
-            """Get the details from the server certificate, cert chain, and
-            server ciphersuite list, and put them in our report.
-
-            WARNING: do *not* do this:
-            >>> server_cert.get_pubkey()
-                <OpenSSL.crypto.PKey at 0x4985d28>
-            >>> pk = server_cert.get_pubkey()
-            >>> pk.check()
-                Segmentation fault
-
-            :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`.
-            :returns: :param:`handshakeSucceeded.connection <connection>`.
-            """
-            host, port = connection.getpeername()
-            log.msg("Handshake with %s:%d successful!" % (host, port))
-
-            server_cert = self.getPeerCert(connection)
-            server_cert_chain = self.getPeerCert(connection, get_chain=True)
-
-            renegotiations = connection.total_renegotiations()
-            cipher_list    = connection.get_cipher_list()
-            session_key    = connection.master_key()
-            rawcert        = connection.get_peer_certificate()
-            ## xxx TODO this hash needs to be formatted as SHA1, not long
-            cert_subj_hash = rawcert.subject_name_hash()
-            cert_serial    = rawcert.get_serial_number()
-            cert_sig_algo  = rawcert.get_signature_algorithm()
-            cert_subject   = self.getX509Name(rawcert.get_subject(),
-                                              get_components=True)
-            cert_issuer    = self.getX509Name(rawcert.get_issuer(),
-                                              get_components=True)
-            cert_pubkey    = self.getPublicKey(rawcert.get_pubkey())
-
-            self.report['host'] = host
-            self.report['port'] = port
-            self.report['state'] = self.state
-            self.report['renegotiations'] = renegotiations
-            self.report['server_cert'] = server_cert
-            self.report['server_cert_chain'] = \
-                ''.join([cert for cert in server_cert_chain])
-            self.report['server_ciphersuite'] = cipher_list
-            self.report['cert_subject'] = cert_subject
-            self.report['cert_subj_hash'] = cert_subj_hash
-            self.report['cert_issuer'] = cert_issuer
-            self.report['cert_public_key'] = cert_pubkey
-            self.report['cert_serial_no'] = cert_serial
-            self.report['cert_sig_algo'] = cert_sig_algo
-            ## The session's master key is only valid for that session, and
-            ## will allow us to decrypt any packet captures (if they were
-            ## collected). Because we are not requesting URLs, only host:port
-            ## (which would be visible in pcaps anyway, since the FQDN is
-            ## never encrypted) I do not see a way for this to log any user or
-            ## identifying information. Correct me if I'm wrong.
-            self.report['session_key'] = session_key
-
-            log.msg("Server certificate:\n\n%s" % server_cert)
-            log.msg("Server certificate chain:\n\n%s"
-                    % ''.join([cert for cert in server_cert_chain]))
-            log.msg("Negotiated ciphersuite:\n%s"
-                    % '\n\t'.join([cipher for cipher in cipher_list]))
-            log.msg("Certificate subject: %s" % cert_subject)
-            log.msg("Certificate subject hash: %d" % cert_subj_hash)
-            log.msg("Certificate issuer: %s" % cert_issuer)
-            log.msg("Certificate public key:\n\n%s" % cert_pubkey)
-            log.msg("Certificate signature algorithm: %s" % cert_sig_algo)
-            log.msg("Certificate serial number: %s" % cert_serial)
-            log.msg("Total renegotiations: %d" % renegotiations)
-
-            return connection
-
-        def handshakeFailed(connection, host):
-            """Handle a failed handshake attempt and report the failure reason.
-
-            :type connection: :class:`twisted.python.failure.Failure <Failure>`
-                or :exc:Exception
-            :param connection: The failed connection.
-            :param tuple host: A tuple of the remote host's IP address as a
-                string, and an integer specifying the remote host port, i.e.
-                ('1.1.1.1',443)
-            :returns: None
-            """
-            addr, port = host
-            log.msg("Handshake with %s:%d failed!" % host)
-
-            self.report['host'] = host
-            self.report['port'] = port
-
-            if isinstance(connection, Exception) \
-                    or isinstance(connection, ConnectionTimeout):
-                log.msg("Handshake failed with reason: %s" % connection.message)
-                self.report['state'] = connection.message
-            elif isinstance(connection, failure.Failure):
-                log.msg("Handshake failed with reason: Socket %s"
-                        % connection.getErrorMessage())
-                self.report['state'] = connection.getErrorMessage()
-                ctmo = connection.trap(ConnectionTimeout)
-                if ctmo == ConnectionTimeout:
-                    connection.cleanFailure()
-            else:
-                log.msg("Handshake failed with reason: %s" % str(connection))
-                if not 'state' in self.report.keys():
-                    self.report['state'] = str(connection)
-
-            return None
-
-        def deferMakeConnection(host):
-            return threads.deferToThread(makeConnection, self.input)
-
-        if self.host and not self.input:
-            self.input = self.splitInput(self.host)
-        log.msg("Beginning handshake test for %s:%s" % self.input)
-
-        connection = deferMakeConnection(self.input)
-        connection.addCallbacks(connectionSucceeded, connectionFailed,
-                                callbackArgs=[self.input, self.timeout],
-                                errbackArgs=[self.input])
-
-        handshake = defer.Deferred()
-        handshake.addCallback(doHandshake)
-        handshake.addCallbacks(handshakeSucceeded, handshakeFailed,
-                               errbackArgs=[self.input])
-
-        connection.chainDeferred(handshake)
-        connection.addCallbacks(connectionShutdown, defer.passthru,
-                                callbackArgs=[self.input])
-        connection.addBoth(log.exception)
-
-        return connection





More information about the tor-commits mailing list