April 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Add connectionRenegotiate() method for calling methods to either handle
by art＠torproject.org 30 Apr '13

30 Apr '13

commit 4ad6a7e1ea528b72731c509bddc63562f1b79f86 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:17:13 2013 +0000 Add connectionRenegotiate() method for calling methods to either handle SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE, as may be the case. --- nettests/experimental/tls_handshake.py | 25 ++++++++++++++++++++++++- 1 files changed, 24 insertions(+), 1 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index d84f174..81cab6a 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -271,7 +271,30 @@ class TLSHandshakeTest(nettest.NetTestCase): return connection - def doHandshake(connection): + def connectionRenegotiate(connection, host, error_message): + log.msg("Server requested renegotiation from: %s" % host) + log.debug("Renegotiation reason: %s" % error_message) + log.debug("State: %s" % connection.state_string()) + + if connection.renegotiate(): + log.debug("Renegotiation possible.") + log.message("Retrying handshake with %s..." % host) + try: + connection.do_handshake() + while connection.renegotiate_pending(): + log.msg("Renegotiation with %s in progress..." % host) + log.debug("State: %s" % connection.state_string()) + sleep(1) + else: + log.msg("Renegotiation with %s complete!" % host) + except SSL.WantReadError, wre: + connection = handleWantRead(connection) + log.debug("State: %s" % connection.state_string()) + except SSL.WantWriteError, wwe: + connection = handleWantWrite(connection) + log.debug("State: %s" % connection.state_string()) + return connection + try: connection.do_handshake() except SSL.WantReadError():

1 0

[ooni-probe/master] Add method connectionShutdown() for calling shutdown on the TLS/SSL handshake,
by art＠torproject.org 30 Apr '13

30 Apr '13

commit e3d14a22dc73f9f0fa64ebe01daf6c25496ae010 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:22:07 2013 +0000 Add method connectionShutdown() for calling shutdown on the TLS/SSL handshake, and the socket, and for handling instances where the connection has been halfway closed. --- nettests/experimental/tls_handshake.py | 50 ++++++++++++++++++++++++++++++++ 1 files changed, 50 insertions(+), 0 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 128812e..7e71579 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -295,6 +295,56 @@ class TLSHandshakeTest(nettest.NetTestCase): log.debug("State: %s" % connection.state_string()) return connection + def connectionShutdown(connection, host): + """ + Handle shutting down a :class:`OpenSSL.SSL.Connection`, including + correct handling of halfway shutdown connections. + + Calls to :meth:`OpenSSL.SSL.Connection.shutdown` return a boolean + value: if the connection is already shutdown, it returns True, + else it returns false. Thus we loop through a block which detects + if the connection is an a partial shutdown state and corrects that + if that is the case, else it waits for one second, then attempts + shutting down the connection again. + + Detection of a partial shutdown state is done through + :meth:`OpenSSL.SSL.Connection.get_shutdown` which queries OpenSSL + for a bitvector of the server and client shutdown states. For + example, the binary string '0b00' is an open connection, and + '0b10' is a partially closed connection that has been shutdown on + the serverside. + + @param connection: A :class:`OpenSSL.SSL.Connection`. + + @param host: A tuple of: a string representation of the remote + host's IP address, and an integer specifying the + port. + """ + peername, peerport = host + + if isinstance(connection, SSL.Connection): + log.msg("Closing connection to %s:%d..." + % (peername, peerport)) + while not connection.shutdown(): + ## if the connection is halfway shutdown, we have to + ## wait for a ZeroReturnError on connection.recv(): + if (bin(connection.get_shutdown()) == '0b01') \ + or (bin(connection.get_shutdown()) == '0b10'): + try: + _read_buffer = connection.pending() + connection.recv(_read_buffer) + except SSL.ZeroReturnError, zre: continue + else: + sleep(1) + else: + log.msg("Closed connection to %s:%d" + % (peername, peerport)) + elif isinstance(connection, types.NoneType): + log.debug("connectionShutdown: got NoneType for connection") + else: + log.debug("connectionShutdown: expected connection, got %s" + % connection.__repr__()) + return connection def handleWantRead(connection): """

1 0

[ooni-probe/master] Add logging of undetermined states reported by OpenSSL's state machine to
by art＠torproject.org 30 Apr '13

30 Apr '13

commit b016417340b8ff3e4cdd860215157f0dfa8944f7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:33:16 2013 +0000 Add logging of undetermined states reported by OpenSSL's state machine to doHandshake(). This should not ever occur, but just in case it does, we'll want to log it rather than throwing the Exception out. --- nettests/experimental/tls_handshake.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index a02cb2f..c77682d 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -500,7 +500,9 @@ class TLSHandshakeTest(nettest.NetTestCase): log.msg("Connection to %s:%s timed out." % (peername, str(peerport))) else: - log.msg("Received: %s" % recvstr) + log.msg("Received: %s" % received) + log.debug("State: %s" % connection.state_string()) + return connection def handshakeSucceeded(connection):

1 0

[ooni-probe/master] Add method handleWantRead() for handling SSL_ERROR_WANT_READ until OpenSSL's
by art＠torproject.org 30 Apr '13

30 Apr '13

commit c6f6ce0516b544b6ae25da6dc940978c11fdb544 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:19:58 2013 +0000 Add method handleWantRead() for handling SSL_ERROR_WANT_READ until OpenSSL's memory BIO state machine reports that it has been handled, and also jump to the SSL_ERROR_WANT_WRITE handler when/if that error occurs. --- nettests/experimental/tls_handshake.py | 79 ++++++++++++++++++++++++++++++-- 1 files changed, 75 insertions(+), 4 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 81cab6a..128812e 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -295,11 +295,82 @@ class TLSHandshakeTest(nettest.NetTestCase): log.debug("State: %s" % connection.state_string()) return connection + + def handleWantRead(connection): + """ + From OpenSSL memory BIO documentation on ssl_read(): + + If the underlying BIO is blocking, SSL_read() will only + return, once the read operation has been finished or an error + occurred, except when a renegotiation take place, in which + case a SSL_ERROR_WANT_READ may occur. This behaviour can be + controlled with the SSL_MODE_AUTO_RETRY flag of the + SSL_CTX_set_mode(3) call. + + If the underlying BIO is non-blocking, SSL_read() will also + return when the underlying BIO could not satisfy the needs of + SSL_read() to continue the operation. In this case a call to + SSL_get_error(3) with the return value of SSL_read() will + yield SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE. As at any + time a re-negotiation is possible, a call to SSL_read() can + also cause write operations! The calling process then must + repeat the call after taking appropriate action to satisfy the + needs of SSL_read(). The action depends on the underlying + BIO. When using a non-blocking socket, nothing is to be done, + but select() can be used to check for the required condition. + + And from the OpenSSL memory BIO documentation on ssl_get_error(): + + SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE + + The operation did not complete; the same TLS/SSL I/O function + should be called again later. If, by then, the underlying BIO + has data available for reading (if the result code is + SSL_ERROR_WANT_READ) or allows writing data + (SSL_ERROR_WANT_WRITE), then some TLS/SSL protocol progress + will take place, i.e. at least part of an TLS/SSL record will + be read or written. Note that the retry may again lead to a + SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE condition. There + is no fixed upper limit for the number of iterations that may + be necessary until progress becomes visible at application + protocol level. + + For socket BIOs (e.g. when SSL_set_fd() was used), select() or + poll() on the underlying socket can be used to find out when + the TLS/SSL I/O function should be retried. + + Caveat: Any TLS/SSL I/O function can lead to either of + SSL_ERROR_WANT_READ and SSL_ERROR_WANT_WRITE. In particular, + SSL_read() or SSL_peek() may want to write data and + SSL_write() may want to read data. This is mainly because + TLS/SSL handshakes may occur at any time during the protocol + (initiated by either the client or the server); SSL_read(), + SSL_peek(), and SSL_write() will handle any pending + handshakes. + + Also, see http://stackoverflow.com/q/3952104 + """ try: - connection.do_handshake() - except SSL.WantReadError(): - log.msg("Timeout exceeded.") - connection.shutdown() + while connection.want_read(): + log.debug("Connection to %s HAS want_read" % host) + _read_buffer = connection.pending() + log.debug("Rereading %d bytes..." % _read_buffer) + sleep(1) + rereceived = connection.recv(int(_read_buffer)) + log.debug("Received %d bytes" % rereceived) + log.debug("State: %s" % connection.state_string()) + else: + peername, peerport = connection.getpeername() + log.debug("Connection to %s:%s DOES NOT HAVE want_read" + % (peername, peerport)) + log.debug("State: %s" % connection.state_string()) + except SSL.WantWriteError, wwe: + log.debug("Got WantWriteError while handling want_read") + log.debug("WantWriteError: %s" % wwe.message) + log.debug("Switching to handleWantWrite()...") + handleWantWrite(connection) + return connection + else: log.msg("State: %s" % connection.state_string()) log.msg("Transmitted %d bytes" % connection.send("o\r\n"))

1 0

[ooni-probe/master] Add method handleWantWrite() for resending already sent data over the
by art＠torproject.org 30 Apr '13

30 Apr '13

commit 80a1dfedd7d266e83d70080bae8f73192dd09bfb Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:24:07 2013 +0000 Add method handleWantWrite() for resending already sent data over the connection, if OpenSSL's memory BIO state machine reports this error, and also jump to the SSL_ERROR_WANT_READ handler if at any point that error is raised. --- nettests/experimental/tls_handshake.py | 52 ++++++++++++++++++++++++++++++++ 1 files changed, 52 insertions(+), 0 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 7e71579..dc046a8 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -421,6 +421,58 @@ class TLSHandshakeTest(nettest.NetTestCase): handleWantWrite(connection) return connection + def handleWantWrite(connection): + """ + See :func:`handleWantRead`. + """ + try: + while connection.want_write(): + log.debug("Connection to %s HAS want_write" % host) + sleep(1) + resent = connection.send("o\r\n") + log.debug("Sent: %d" % resent) + log.debug("State: %s" % connection.state_string()) + except SSL.WantReadError, wre: + log.debug("Got WantReadError while handling want_write") + log.debug("WantReadError: %s" % wre.message) + log.debug("Switching to handleWantRead()...") + handleWantRead(connection) + return connection + + def doHandshake(connection): + """ + Attemp a TLS/SSL handshake with the host. + + If, after the first attempt at handshaking, OpenSSL's memory BIO + state machine does not report success, then try reading and + writing from the connection, and handle any SSL_ERROR_WANT_READ or + SSL_ERROR_WANT_WRITE which occurs. + + If multiple want_reads occur, then try renegotiation with the + host, and start over. If multiple want_writes occur, then it is + possible that the connection has timed out, and move on to the + connectionShutdown step. + + @param connection: A :class:`OpenSSL.SSL.Connection`. + @ivar peername: The host IP address, as reported by getpeername(). + @ivar peerport: The host port, reported by getpeername(). + @ivar sent: The number of bytes sent to to the remote host. + @ivar received: The number of bytes received from the remote host. + @ivar _read_buffer: An integer for the max bytes that can be read + from the connection. + @returns: The :param:connection with handshake completed, else + the unhandled error that was raised. + """ + peername, peerport = connection.getpeername() + + log.msg("Attempting handshake: %s" % peername) + connection.do_handshake() + log.debug("State: %s" % connection.state_string()) + if connection.state_string() == \ + 'SSL negotiation finished successfully': + ## jump to handshakeSuccessful and get certchain + return connection + else: log.msg("State: %s" % connection.state_string()) log.msg("Transmitted %d bytes" % connection.send("o\r\n"))

1 0

[ooni-probe/master] Add check for handshake timeout by sending 3 bytes along the connection.
by art＠torproject.org 30 Apr '13

30 Apr '13

commit 024755308ad8ee6c96ebd48c34f64e96beff8f4c Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:27:53 2013 +0000 Add check for handshake timeout by sending 3 bytes along the connection. * Currently we send 'o\r\n'. TODO: we should look up a more common string that is sent, perhaps one for each of the major browsers, to emulate a more normal connection, and decrease fingerprintability. * Add calls to the SSL_ERROR_WANT_READ and SSL_ERROR_WANT_WRITE handlers, handleWantRead() and handleWantWrite(), to method doHandshake(). * Add timeout notice if repetitive want_writes occur in doHandshake(). * Add call to connectionRenegotiate() if multiple want_reads occur in method doHandshake(). --- nettests/experimental/tls_handshake.py | 30 ++++++++++++++++++++++++------ 1 files changed, 24 insertions(+), 6 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index dc046a8..a02cb2f 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -474,13 +474,31 @@ class TLSHandshakeTest(nettest.NetTestCase): return connection else: - log.msg("State: %s" % connection.state_string()) - log.msg("Transmitted %d bytes" % connection.send("o\r\n")) + sent = connection.send("o\r\n") + log.debug("State: %s" % connection.state_string()) + log.debug("Transmitted %d bytes" % sent) + + _read_buffer = connection.pending() + log.debug("Max bytes in receive buffer: %d" % _read_buffer) + try: - recvstr = connection.recv(1024) - except SSL.WantReadError: - log.msg("Timeout exceeded") - connection.shutdown() + received = connection.recv(int(_read_buffer)) + except SSL.WantReadError, wre: + if connection.want_read(): + connection = handleWantRead(connection) + else: + ## if we still have an SSL_ERROR_WANT_READ, then try + ## to renegotiate + connection = connectionRenegotiate(connection, + connection.getpeername(), + wre.message) + except SSL.WantWriteError, wwe: + log.debug("State: %s" % connection.state_string()) + if connection.want_write(): + connection = handleWantWrite(connection) + else: + log.msg("Connection to %s:%s timed out." + % (peername, str(peerport))) else: log.msg("Received: %s" % recvstr) return connection

1 0

[ooni-probe/master] Add reporting of statistics for when the handshake fails.
by art＠torproject.org 30 Apr '13

30 Apr '13

commit 021f0e213c431bb7922d6f587d96c2fc4cfe5add Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:38:18 2013 +0000 Add reporting of statistics for when the handshake fails. --- nettests/experimental/tls_handshake.py | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index da6cbc0..c1e7ca5 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -561,6 +561,23 @@ class TLSHandshakeTest(nettest.NetTestCase): ## xxx do we need this? #return connection + def handshakeFailed(connection, host): + """ + xxx fill me in + + @param connection: A :class:`twisted.python.failure.Failure` or + :class:`exceptions.Exception`. + @param host: A tuple of the host IP and port, i.e. ('1.1.1.1', 443). + @returns: None. + """ + addr, port = host + log.msg("Handshake with %s:%d failed!" % host) + self.report['host'] = host + self.report['port'] = port + self.report['state'] = "HANDSHAKE_FAILED" + ## xxx do we need this? + #return connection + else: return handshakeSucceeded(connection)

1 0

[ooni-probe/master] Add certain deferreds into their own function so that we can do a
by art＠torproject.org 30 Apr '13

30 Apr '13

commit 15e8e11cb9da46b78a6089f478d28a021c798a81 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:39:49 2013 +0000 Add certain deferreds into their own function so that we can do a defer.Deferred.addCallbacks on its return value. --- nettests/experimental/tls_handshake.py | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 0bd84bc..a21b357 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -589,8 +589,15 @@ class TLSHandshakeTest(nettest.NetTestCase): ## xxx do we need this? #return connection + @defer.inlineCallbacks + def deferMakeConnection(host): + connection = yield makeConnection(host) + if isinstance(connection, Failure) \ + or isinstance(connection, Exception): + failed = connectionFailed(connection, host) + defer.returnValue(failed) else: - return handshakeSucceeded(connection) + defer.returnValue(connection) addr, port = self.input connection = defer.maybeDeferred(makeConnection, addr, port)

1 0

[ooni-probe/master] Update import statements in tls_handshake.py and bump the version number.
by art＠torproject.org 30 Apr '13

30 Apr '13

commit 56b48d7e25ee9607154657f6f117509c68f8f08c Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:39:09 2013 +0000 Update import statements in tls_handshake.py and bump the version number. --- nettests/experimental/tls_handshake.py | 37 ++++++++++++++++++++----------- 1 files changed, 24 insertions(+), 13 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index c1e7ca5..0bd84bc 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -3,31 +3,42 @@ """ tls_handshake.py ---------------- + This file contains test cases for determining if a TLS handshake completes successfully, including ways to test if a TLS handshake which uses Mozilla - Firefox's current ciphersuite list completes. + Firefox's current ciphersuite list completes. Rather than using Twisted and + OpenSSL's methods for automatically completing a handshake, which includes + setting all the parameters, such as the ciphersuite list, these tests use + non-blocking sockets and implement asychronous error-handling transversal of + OpenSSL's memory BIO state machine, allowing us to determine where and why a + handshake fails. - These NetTestCases are a rewrite of a script contributed by Hackerberry - Finn, in order to fit into OONI's core network tests. + This network test is a complete rewrite of a pseudonymously contributed + script by Hackerberry Finn, in order to fit into OONI's core network tests. - @authors: Isis Agora Lovecruft <isis(a)torproject.org>, - Hackerberry Finn <anon@localhost> + @authors: Isis Agora Lovecruft <isis(a)torproject.org> @license: see included LICENSE file + @copyright: © 2013 Isis Lovecruft, The Tor Project Inc. """ +from socket import error as socket_error +from time import sleep + import os import socket -from socket import error as serror import struct import sys +import types -from ipaddr import IPAddress -from OpenSSL import SSL -from twisted.internet import defer -from twisted.python import usage +from ipaddr import IPAddress +from OpenSSL import SSL +from OpenSSL.crypto import dump_certificate, FILETYPE_PEM +from twisted.internet import defer +from twisted.python import usage +from twisted.python.failure import Failure -from ooni import nettest, config -from ooni.utils import log +from ooni import nettest, config +from ooni.utils import log, NotRootError ## For a way to obtain the current version of Firefox's default ciphersuite ## list, see https://trac.torproject.org/projects/tor/attachment/ticket/4744/ @@ -90,7 +101,7 @@ class TLSHandshakeTest(nettest.NetTestCase): name = 'tls-handshake' author = 'Isis Lovecruft <isis(a)torproject.org>' description = 'A test to determing if we can complete a TLS hankshake.' - version = '0.0.1' + version = '0.0.2' requiresRoot = False usageOptions = UsageOptions

1 0

[ooni-probe/master] Add chained callbacks to handshake checking methods, and remove some unused
by art＠torproject.org 30 Apr '13

30 Apr '13

commit 1fa06bffa93632fb031ff690e69f50b0c2f65436 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:41:01 2013 +0000 Add chained callbacks to handshake checking methods, and remove some unused code from the end of the file. --- nettests/experimental/tls_handshake.py | 33 +++++++++++++------------------ 1 files changed, 14 insertions(+), 19 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index a21b357..d990a5b 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -599,24 +599,19 @@ class TLSHandshakeTest(nettest.NetTestCase): else: defer.returnValue(connection) - addr, port = self.input - connection = defer.maybeDeferred(makeConnection, addr, port) - connection.addCallback(doHandshake) - connection.addErrback(log.err) - connection.addCallback(handshakeSucceeded) - connection.addErrback(handshakeFailed) + connection = deferMakeConnection(self.input) + connection.addCallbacks(connectionSucceeded, connectionFailed, + callbackArgs=[self.input, self.timeout], + errbackArgs=[self.input]) - return connection + handshake = defer.Deferred() + handshake.addCallback(doHandshake) + handshake.addCallbacks(handshakeSucceeded, handshakeFailed, + errbackArgs=[self.input]) -## XXX clean me up -## old function from anonymous contribution: (saved mostly for reference of the -## success string) -## -#def checkBridgeConnection(host, port) -# cipher_arg = ":".join(ciphers) -# cmd = ["openssl", "s_client", "-connect", "%s:%s" % (host,port)] -# cmd += ["-cipher", cipher_arg] -# proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE,stdin=PIPE) -# out, error = proc.communicate() -# success = "Cipher is DHE-RSA-AES256-SHA" in out -# return success + connection.chainDeferred(handshake) + connection.addCallbacks(connectionShutdown, defer.passthru, + callbackArgs=[self.input]) + connection.addBoth(log.exception) + + return connection

1 0