June 2013 - tor-commits - lists.torproject.org

[ooni-probe/develop] Switch to setting the TLS/SSL methods in setUp(), rather than in getContext().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit a00f0700f0d6f2e3eed700f8c9bb8a5c77097d8b Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 03:38:21 2013 +0000 Switch to setting the TLS/SSL methods in setUp(), rather than in getContext(). --- nettests/experimental/tls_handshake.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 6474792..c57545c 100644 --- a/nettests/experimental/… [View More]

1 0

[ooni-probe/develop] Change getContext() to only set the ciphersuite list, and add method
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit dfc16658964a3bcbec5d94c970bde8c8927286d5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:05:42 2013 +0000 Change getContext() to only set the ciphersuite list, and add method getPeerCert() for dumping the server x509 certificate. --- nettests/experimental/tls_handshake.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 2c11122.… [View More]

1 0

[ooni-probe/develop] Add support for parsing the server certificate chain into a list of x509 certs
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 638359fa3df1901d150671c95087827fba8f4b7b Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:07:28 2013 +0000 Add support for parsing the server certificate chain into a list of x509 certs in method getPeerCert(). --- nettests/experimental/tls_handshake.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 3e22f38..af59227 100644 --- a/… [View More]

1 0

[ooni-probe/develop] Change method makeConnection to use self.input as-is, and add docstring.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 54846399504b326fa010a750d3d0c53126adf54f Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:09:25 2013 +0000 Change method makeConnection to use self.input as-is, and add docstring. --- nettests/experimental/tls_handshake.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index af59227..322ec72 100644 --- a/nettests/experimental/… [View More]

1 0

[ooni-probe/develop] In connectionFailed, handle an error from makeConnection() where on some
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 5cd30dababdf6f61187de215e2440a1917388007 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:12:02 2013 +0000 In connectionFailed, handle an error from makeConnection() where on some systems, /dev/random is apparently set to 0600 root:root, and also add handling for propagating errors due to the network/server being unreachable. --- nettests/experimental/tls_handshake.py | 70 ++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 12 … [View More]deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 322ec72..d84f174 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -209,20 +209,66 @@ class TLSHandshakeTest(nettest.NetTestCase): connection.connect(host) return connection - connection = SSL.Connection(context, socket) + def connectionFailed(connection, host): + """ + Handle errors raised while attempting to create the socket, TLS/SSL + context, and :class:`OpenSSL.SSL.Connection` object. - try: - connection.connect((addr, port)) - except serror, se: - if se.message.find("[Errno 101]"): - connection.shutdown() - log.err(se) + @param connection: The Exception that was raised in + :func:`makeConnection`. + @param host: A tuple of the host IP address as a string, and an int + specifying the host port, i.e. ('1.1.1.1', 443) + """ + addr, port = host + if isinstance(connection, IOError): + ## On some *nix distros, /dev/random is 0600 root:root and we get + ## a permissions error when trying to read + if connection.message.find("[Errno 13]"): + raise NotRootError( + "%s" % connection.message.split("[Errno 13]", 1)[1]) + + if isinstance(connection, socket_error): + if connection.message.find("[Errno 101]"): + raise HostUnreachableError( + "Host unreachable: %s:%s" % (addr, port)) + + log.err(connection) + self.report['host'] = addr + self.report['port'] = port + self.report['state'] = 'CONNECTION_FAILED' + return connection + + def connectionSucceeded(connection, host, timeout): + """ + If we have created a connection, set the socket options, and log the + connection state and peer name. + + @param connection: A :class:`OpenSSL.SSL.Connection` object. + @param host: A tuple of the host IP and port, i.e. ('1.1.1.1', 443). + """ + connection.setblocking(1) + ## Set the timeout on the connection: + ## + ## We want to set SO_RCVTIMEO and SO_SNDTIMEO, which both are + ## defined in the socket option definitions in <sys/socket.h>, and + ## which both take as their value, according to socket(7), a + ## struct timeval, which is defined in the libc manual: + ## https://www.gnu.org/software/libc/manual/html_node/Elapsed-Time.html + timeval = struct.pack('ll', int(timeout), 0) + connection.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, timeval) + connection.setsockopt(socket.SOL_SOCKET, socket.SO_SNDTIMEO, timeval) + + ## Set the connection state to client mode: + connection.set_connect_state() + + peer_name, peer_port = connection.getpeername() + if peer_name: + log.msg("Connected to %s" % peer_name) else: - connection.setblocking(1) - connection.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, - self.timeout) - log.msg("Connected to %s" % connection.getpeername()) - log.msg("Connection state: %s " % connection.state_string()) + log.debug("Couldn't get peer name from connection: %s" % host) + log.msg("Connected to: %s" % host) + log.msg("Connection state: %s " % connection.state_string()) + return connection def doHandshake(connection): [View Less]

1 0

[ooni-probe/develop] Add connectionRenegotiate() method for calling methods to either handle
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 4ad6a7e1ea528b72731c509bddc63562f1b79f86 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:17:13 2013 +0000 Add connectionRenegotiate() method for calling methods to either handle SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE, as may be the case. --- nettests/experimental/tls_handshake.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/… [View More]

1 0

[ooni-probe/develop] Add check for handshake timeout by sending 3 bytes along the connection.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 024755308ad8ee6c96ebd48c34f64e96beff8f4c Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:27:53 2013 +0000 Add check for handshake timeout by sending 3 bytes along the connection. * Currently we send 'o\r\n'. TODO: we should look up a more common string that is sent, perhaps one for each of the major browsers, to emulate a more normal connection, and decrease fingerprintability. * Add calls to the SSL_ERROR_WANT_READ and … [View More]SSL_ERROR_WANT_WRITE handlers, handleWantRead() and handleWantWrite(), to method doHandshake(). * Add timeout notice if repetitive want_writes occur in doHandshake(). * Add call to connectionRenegotiate() if multiple want_reads occur in method doHandshake(). --- nettests/experimental/tls_handshake.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index dc046a8..a02cb2f 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -474,13 +474,31 @@ class TLSHandshakeTest(nettest.NetTestCase): return connection else: - log.msg("State: %s" % connection.state_string()) - log.msg("Transmitted %d bytes" % connection.send("o\r\n")) + sent = connection.send("o\r\n") + log.debug("State: %s" % connection.state_string()) + log.debug("Transmitted %d bytes" % sent) + + _read_buffer = connection.pending() + log.debug("Max bytes in receive buffer: %d" % _read_buffer) + try: - recvstr = connection.recv(1024) - except SSL.WantReadError: - log.msg("Timeout exceeded") - connection.shutdown() + received = connection.recv(int(_read_buffer)) + except SSL.WantReadError, wre: + if connection.want_read(): + connection = handleWantRead(connection) + else: + ## if we still have an SSL_ERROR_WANT_READ, then try + ## to renegotiate + connection = connectionRenegotiate(connection, + connection.getpeername(), + wre.message) + except SSL.WantWriteError, wwe: + log.debug("State: %s" % connection.state_string()) + if connection.want_write(): + connection = handleWantWrite(connection) + else: + log.msg("Connection to %s:%s timed out." + % (peername, str(peerport))) else: log.msg("Received: %s" % recvstr) return connection [View Less]

1 0

[ooni-probe/develop] Add method handleWantWrite() for resending already sent data over the
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 80a1dfedd7d266e83d70080bae8f73192dd09bfb Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:24:07 2013 +0000 Add method handleWantWrite() for resending already sent data over the connection, if OpenSSL's memory BIO state machine reports this error, and also jump to the SSL_ERROR_WANT_READ handler if at any point that error is raised. --- nettests/experimental/tls_handshake.py | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) … [View More]diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 7e71579..dc046a8 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -421,6 +421,58 @@ class TLSHandshakeTest(nettest.NetTestCase): handleWantWrite(connection) return connection + def handleWantWrite(connection): + """ + See :func:`handleWantRead`. + """ + try: + while connection.want_write(): + log.debug("Connection to %s HAS want_write" % host) + sleep(1) + resent = connection.send("o\r\n") + log.debug("Sent: %d" % resent) + log.debug("State: %s" % connection.state_string()) + except SSL.WantReadError, wre: + log.debug("Got WantReadError while handling want_write") + log.debug("WantReadError: %s" % wre.message) + log.debug("Switching to handleWantRead()...") + handleWantRead(connection) + return connection + + def doHandshake(connection): + """ + Attemp a TLS/SSL handshake with the host. + + If, after the first attempt at handshaking, OpenSSL's memory BIO + state machine does not report success, then try reading and + writing from the connection, and handle any SSL_ERROR_WANT_READ or + SSL_ERROR_WANT_WRITE which occurs. + + If multiple want_reads occur, then try renegotiation with the + host, and start over. If multiple want_writes occur, then it is + possible that the connection has timed out, and move on to the + connectionShutdown step. + + @param connection: A :class:`OpenSSL.SSL.Connection`. + @ivar peername: The host IP address, as reported by getpeername(). + @ivar peerport: The host port, reported by getpeername(). + @ivar sent: The number of bytes sent to to the remote host. + @ivar received: The number of bytes received from the remote host. + @ivar _read_buffer: An integer for the max bytes that can be read + from the connection. + @returns: The :param:connection with handshake completed, else + the unhandled error that was raised. + """ + peername, peerport = connection.getpeername() + + log.msg("Attempting handshake: %s" % peername) + connection.do_handshake() + log.debug("State: %s" % connection.state_string()) + if connection.state_string() == \ + 'SSL negotiation finished successfully': + ## jump to handshakeSuccessful and get certchain + return connection + else: log.msg("State: %s" % connection.state_string()) log.msg("Transmitted %d bytes" % connection.send("o\r\n")) [View Less]

1 0

[ooni-probe/develop] Add method connectionShutdown() for calling shutdown on the TLS/SSL handshake,
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit e3d14a22dc73f9f0fa64ebe01daf6c25496ae010 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:22:07 2013 +0000 Add method connectionShutdown() for calling shutdown on the TLS/SSL handshake, and the socket, and for handling instances where the connection has been halfway closed. --- nettests/experimental/tls_handshake.py | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/nettests/experimental/tls_handshake.py b/… [View More]nettests/experimental/tls_handshake.py index 128812e..7e71579 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -295,6 +295,56 @@ class TLSHandshakeTest(nettest.NetTestCase): log.debug("State: %s" % connection.state_string()) return connection + def connectionShutdown(connection, host): + """ + Handle shutting down a :class:`OpenSSL.SSL.Connection`, including + correct handling of halfway shutdown connections. + + Calls to :meth:`OpenSSL.SSL.Connection.shutdown` return a boolean + value: if the connection is already shutdown, it returns True, + else it returns false. Thus we loop through a block which detects + if the connection is an a partial shutdown state and corrects that + if that is the case, else it waits for one second, then attempts + shutting down the connection again. + + Detection of a partial shutdown state is done through + :meth:`OpenSSL.SSL.Connection.get_shutdown` which queries OpenSSL + for a bitvector of the server and client shutdown states. For + example, the binary string '0b00' is an open connection, and + '0b10' is a partially closed connection that has been shutdown on + the serverside. + + @param connection: A :class:`OpenSSL.SSL.Connection`. + + @param host: A tuple of: a string representation of the remote + host's IP address, and an integer specifying the + port. + """ + peername, peerport = host + + if isinstance(connection, SSL.Connection): + log.msg("Closing connection to %s:%d..." + % (peername, peerport)) + while not connection.shutdown(): + ## if the connection is halfway shutdown, we have to + ## wait for a ZeroReturnError on connection.recv(): + if (bin(connection.get_shutdown()) == '0b01') \ + or (bin(connection.get_shutdown()) == '0b10'): + try: + _read_buffer = connection.pending() + connection.recv(_read_buffer) + except SSL.ZeroReturnError, zre: continue + else: + sleep(1) + else: + log.msg("Closed connection to %s:%d" + % (peername, peerport)) + elif isinstance(connection, types.NoneType): + log.debug("connectionShutdown: got NoneType for connection") + else: + log.debug("connectionShutdown: expected connection, got %s" + % connection.__repr__()) + return connection def handleWantRead(connection): """ [View Less]

1 0

[ooni-probe/develop] Add method handleWantRead() for handling SSL_ERROR_WANT_READ until OpenSSL's
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit c6f6ce0516b544b6ae25da6dc940978c11fdb544 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:19:58 2013 +0000 Add method handleWantRead() for handling SSL_ERROR_WANT_READ until OpenSSL's memory BIO state machine reports that it has been handled, and also jump to the SSL_ERROR_WANT_WRITE handler when/if that error occurs. --- nettests/experimental/tls_handshake.py | 79 ++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 4 deletions(-) … [View More] diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 81cab6a..128812e 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -295,11 +295,82 @@ class TLSHandshakeTest(nettest.NetTestCase): log.debug("State: %s" % connection.state_string()) return connection + + def handleWantRead(connection): + """ + From OpenSSL memory BIO documentation on ssl_read(): + + If the underlying BIO is blocking, SSL_read() will only + return, once the read operation has been finished or an error + occurred, except when a renegotiation take place, in which + case a SSL_ERROR_WANT_READ may occur. This behaviour can be + controlled with the SSL_MODE_AUTO_RETRY flag of the + SSL_CTX_set_mode(3) call. + + If the underlying BIO is non-blocking, SSL_read() will also + return when the underlying BIO could not satisfy the needs of + SSL_read() to continue the operation. In this case a call to + SSL_get_error(3) with the return value of SSL_read() will + yield SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE. As at any + time a re-negotiation is possible, a call to SSL_read() can + also cause write operations! The calling process then must + repeat the call after taking appropriate action to satisfy the + needs of SSL_read(). The action depends on the underlying + BIO. When using a non-blocking socket, nothing is to be done, + but select() can be used to check for the required condition. + + And from the OpenSSL memory BIO documentation on ssl_get_error(): + + SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE + + The operation did not complete; the same TLS/SSL I/O function + should be called again later. If, by then, the underlying BIO + has data available for reading (if the result code is + SSL_ERROR_WANT_READ) or allows writing data + (SSL_ERROR_WANT_WRITE), then some TLS/SSL protocol progress + will take place, i.e. at least part of an TLS/SSL record will + be read or written. Note that the retry may again lead to a + SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE condition. There + is no fixed upper limit for the number of iterations that may + be necessary until progress becomes visible at application + protocol level. + + For socket BIOs (e.g. when SSL_set_fd() was used), select() or + poll() on the underlying socket can be used to find out when + the TLS/SSL I/O function should be retried. + + Caveat: Any TLS/SSL I/O function can lead to either of + SSL_ERROR_WANT_READ and SSL_ERROR_WANT_WRITE. In particular, + SSL_read() or SSL_peek() may want to write data and + SSL_write() may want to read data. This is mainly because + TLS/SSL handshakes may occur at any time during the protocol + (initiated by either the client or the server); SSL_read(), + SSL_peek(), and SSL_write() will handle any pending + handshakes. + + Also, see http://stackoverflow.com/q/3952104 + """ try: - connection.do_handshake() - except SSL.WantReadError(): - log.msg("Timeout exceeded.") - connection.shutdown() + while connection.want_read(): + log.debug("Connection to %s HAS want_read" % host) + _read_buffer = connection.pending() + log.debug("Rereading %d bytes..." % _read_buffer) + sleep(1) + rereceived = connection.recv(int(_read_buffer)) + log.debug("Received %d bytes" % rereceived) + log.debug("State: %s" % connection.state_string()) + else: + peername, peerport = connection.getpeername() + log.debug("Connection to %s:%s DOES NOT HAVE want_read" + % (peername, peerport)) + log.debug("State: %s" % connection.state_string()) + except SSL.WantWriteError, wwe: + log.debug("Got WantWriteError while handling want_read") + log.debug("WantWriteError: %s" % wwe.message) + log.debug("Switching to handleWantWrite()...") + handleWantWrite(connection) + return connection + else: log.msg("State: %s" % connection.state_string()) log.msg("Transmitted %d bytes" % connection.send("o\r\n")) [View Less]

1 0