June 2013 - tor-commits - lists.torproject.org

[ooni-probe/develop] Fix the logic on detecting if we have a config setting.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit c0b2e453e20303780b941d90c54aa04fd8b6a1ec Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 03:56:25 2013 +0000 Fix the logic on detecting if we have a config setting. * We can't use hasattr(config.basic, 'foo') because this gives None if not set, which means the default in the test doesn't get set. Instead, we want to use: if getattr(config.basic, 'foo', None) is not None: --- nettests/experimental/tls_handshake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index c57545c..8bd4820 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -138,8 +138,8 @@ class TLSHandshakeTest(nettest.NetTestCase): self.ciphers.append(line.strip()) self.ciphersuite = ":".join(self.ciphers) - if hasattr(config.advanced, 'default_timeout'): - timeout = config.advanced.default_timeout + if getattr(config.advanced, 'default_timeout', None) is not None: + self.timeout = config.advanced.default_timeout else: timeout = 10 ## default the timeout to 10 seconds socket.setdefaulttimeout(timeout)

1 0

[ooni-probe/develop] Switch to setting the TLS/SSL methods in setUp(), rather than in getContext().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit a00f0700f0d6f2e3eed700f8c9bb8a5c77097d8b Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 03:38:21 2013 +0000 Switch to setting the TLS/SSL methods in setUp(), rather than in getContext(). --- nettests/experimental/tls_handshake.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 6474792..c57545c 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -101,7 +101,6 @@ class TLSHandshakeTest(nettest.NetTestCase): if self.localOptions: options = self.localOptions self.ciphers = [] - self.methods = [] ## check that we're testing an IP:PORT, else exit gracefully: if not ((options['host'] and options['port']) or options['file']): @@ -111,9 +110,22 @@ class TLSHandshakeTest(nettest.NetTestCase): ## one of the TLS/SSL methods at a time. ## set the SSL/TLS method to use: - for method in ['ssl2', 'ssl3', 'tls1']: - if options[method]: - self.methods.append(method) + if options['ssl2']: + if not options['ssl3']: + self.context = SSL.Context(SSL.SSLv2_METHOD) + else: + self.context = SSL.Context(SSL.SSLv23_METHOD) + elif options['ssl3']: + self.context = SSL.Context(SSL.SSLv3_METHOD) + elif options['tls1']: + self.context = SSL.Context(SSL.TLSv1_METHOD) + else: + try: + raise NoSSLContextError( + "No SSL/TLS context chosen! Defaulting to TLSv1...") + except NoSSLContextError, ncse: + log.err(ncse.message) + self.context = SSL.Context(SSL.TLSv1_METHOD) if not options['ciphersuite']: self.ciphers = firefox_ciphers

1 0

[ooni-probe/develop] Change getContext() to only set the ciphersuite list, and add method
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit dfc16658964a3bcbec5d94c970bde8c8927286d5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:05:42 2013 +0000 Change getContext() to only set the ciphersuite list, and add method getPeerCert() for dumping the server x509 certificate. --- nettests/experimental/tls_handshake.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 2c11122..3e22f38 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -172,15 +172,15 @@ class TLSHandshakeTest(nettest.NetTestCase): return s def getContext(self): - if 'ssl2' in self.methods: - if not 'ssl3' in self.methods: - context = SSL.Context(SSL.SSLv2_METHOD) - else: - context = SSL.Context(SSL.SSLv23_METHOD) - elif 'ssl3' in self.methods: - context = SSL.Context(SSL.SSLv3_METHOD) - elif 'tls1' in self.methods: - context = SSL.Context(SSL.TLSv1_METHOD) + self.context.set_cipher_list(self.ciphersuite) + return self.context + + @staticmethod + def getPeerCert(connection, get_chain=False): + if not get_chain: + x509_cert = connection.get_peer_certificate() + pem_cert = dump_certificate(FILETYPE_PEM, x509_cert) + return pem_cert else: raise Exception("No SSL/TLS method chosen!") context.set_cipher_list(self.ciphersuite)

1 0

[ooni-probe/develop] Add support for parsing the server certificate chain into a list of x509 certs
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 638359fa3df1901d150671c95087827fba8f4b7b Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:07:28 2013 +0000 Add support for parsing the server certificate chain into a list of x509 certs in method getPeerCert(). --- nettests/experimental/tls_handshake.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 3e22f38..af59227 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -182,9 +182,12 @@ class TLSHandshakeTest(nettest.NetTestCase): pem_cert = dump_certificate(FILETYPE_PEM, x509_cert) return pem_cert else: - raise Exception("No SSL/TLS method chosen!") - context.set_cipher_list(self.ciphersuite) - return context + cert_chain = [] + x509_cert_chain = connection.get_peer_cert_chain() + for x509_cert in x509_cert_chain: + pem_cert = dump_certificate(FILETYPE_PEM, x509_cert) + cert_chain.append(pem_cert) + return cert_chain def test_tlsv1_handshake(self):

1 0

[ooni-probe/develop] Change method makeConnection to use self.input as-is, and add docstring.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 54846399504b326fa010a750d3d0c53126adf54f Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:09:25 2013 +0000 Change method makeConnection to use self.input as-is, and add docstring. --- nettests/experimental/tls_handshake.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index af59227..322ec72 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -190,10 +190,24 @@ class TLSHandshakeTest(nettest.NetTestCase): return cert_chain def test_tlsv1_handshake(self): - - def makeConnection(addr, port): - socket = self.buildSocket(addr) + """xxx fill me in""" + + def makeConnection(host): + """ + Create a socket to the host's IP address, then get the TLS/SSL context + method and ciphersuite list. Lastly, initiate a connection to the + host. + + @param host: A tuple of the host IP and port, i.e. (addr, port). + @returns: A :class:`OpenSSL.SSL.Connection` object (or any Exception + that was raised), and the :param:`host`. + """ + addr, port = host + sckt = self.buildSocket(addr) context = self.getContext() + connection = SSL.Connection(context, sckt) + connection.connect(host) + return connection connection = SSL.Connection(context, socket)

1 0

[ooni-probe/develop] In connectionFailed, handle an error from makeConnection() where on some
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 5cd30dababdf6f61187de215e2440a1917388007 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:12:02 2013 +0000 In connectionFailed, handle an error from makeConnection() where on some systems, /dev/random is apparently set to 0600 root:root, and also add handling for propagating errors due to the network/server being unreachable. --- nettests/experimental/tls_handshake.py | 70 ++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 322ec72..d84f174 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -209,20 +209,66 @@ class TLSHandshakeTest(nettest.NetTestCase): connection.connect(host) return connection - connection = SSL.Connection(context, socket) + def connectionFailed(connection, host): + """ + Handle errors raised while attempting to create the socket, TLS/SSL + context, and :class:`OpenSSL.SSL.Connection` object. - try: - connection.connect((addr, port)) - except serror, se: - if se.message.find("[Errno 101]"): - connection.shutdown() - log.err(se) + @param connection: The Exception that was raised in + :func:`makeConnection`. + @param host: A tuple of the host IP address as a string, and an int + specifying the host port, i.e. ('1.1.1.1', 443) + """ + addr, port = host + if isinstance(connection, IOError): + ## On some *nix distros, /dev/random is 0600 root:root and we get + ## a permissions error when trying to read + if connection.message.find("[Errno 13]"): + raise NotRootError( + "%s" % connection.message.split("[Errno 13]", 1)[1]) + + if isinstance(connection, socket_error): + if connection.message.find("[Errno 101]"): + raise HostUnreachableError( + "Host unreachable: %s:%s" % (addr, port)) + + log.err(connection) + self.report['host'] = addr + self.report['port'] = port + self.report['state'] = 'CONNECTION_FAILED' + return connection + + def connectionSucceeded(connection, host, timeout): + """ + If we have created a connection, set the socket options, and log the + connection state and peer name. + + @param connection: A :class:`OpenSSL.SSL.Connection` object. + @param host: A tuple of the host IP and port, i.e. ('1.1.1.1', 443). + """ + connection.setblocking(1) + ## Set the timeout on the connection: + ## + ## We want to set SO_RCVTIMEO and SO_SNDTIMEO, which both are + ## defined in the socket option definitions in <sys/socket.h>, and + ## which both take as their value, according to socket(7), a + ## struct timeval, which is defined in the libc manual: + ## https://www.gnu.org/software/libc/manual/html_node/Elapsed-Time.html + timeval = struct.pack('ll', int(timeout), 0) + connection.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, timeval) + connection.setsockopt(socket.SOL_SOCKET, socket.SO_SNDTIMEO, timeval) + + ## Set the connection state to client mode: + connection.set_connect_state() + + peer_name, peer_port = connection.getpeername() + if peer_name: + log.msg("Connected to %s" % peer_name) else: - connection.setblocking(1) - connection.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, - self.timeout) - log.msg("Connected to %s" % connection.getpeername()) - log.msg("Connection state: %s " % connection.state_string()) + log.debug("Couldn't get peer name from connection: %s" % host) + log.msg("Connected to: %s" % host) + log.msg("Connection state: %s " % connection.state_string()) + return connection def doHandshake(connection):

1 0

[ooni-probe/develop] Add connectionRenegotiate() method for calling methods to either handle
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 4ad6a7e1ea528b72731c509bddc63562f1b79f86 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:17:13 2013 +0000 Add connectionRenegotiate() method for calling methods to either handle SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE, as may be the case. --- nettests/experimental/tls_handshake.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index d84f174..81cab6a 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -271,7 +271,30 @@ class TLSHandshakeTest(nettest.NetTestCase): return connection - def doHandshake(connection): + def connectionRenegotiate(connection, host, error_message): + log.msg("Server requested renegotiation from: %s" % host) + log.debug("Renegotiation reason: %s" % error_message) + log.debug("State: %s" % connection.state_string()) + + if connection.renegotiate(): + log.debug("Renegotiation possible.") + log.message("Retrying handshake with %s..." % host) + try: + connection.do_handshake() + while connection.renegotiate_pending(): + log.msg("Renegotiation with %s in progress..." % host) + log.debug("State: %s" % connection.state_string()) + sleep(1) + else: + log.msg("Renegotiation with %s complete!" % host) + except SSL.WantReadError, wre: + connection = handleWantRead(connection) + log.debug("State: %s" % connection.state_string()) + except SSL.WantWriteError, wwe: + connection = handleWantWrite(connection) + log.debug("State: %s" % connection.state_string()) + return connection + try: connection.do_handshake() except SSL.WantReadError():

1 0

[ooni-probe/develop] Add check for handshake timeout by sending 3 bytes along the connection.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 024755308ad8ee6c96ebd48c34f64e96beff8f4c Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:27:53 2013 +0000 Add check for handshake timeout by sending 3 bytes along the connection. * Currently we send 'o\r\n'. TODO: we should look up a more common string that is sent, perhaps one for each of the major browsers, to emulate a more normal connection, and decrease fingerprintability. * Add calls to the SSL_ERROR_WANT_READ and SSL_ERROR_WANT_WRITE handlers, handleWantRead() and handleWantWrite(), to method doHandshake(). * Add timeout notice if repetitive want_writes occur in doHandshake(). * Add call to connectionRenegotiate() if multiple want_reads occur in method doHandshake(). --- nettests/experimental/tls_handshake.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index dc046a8..a02cb2f 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -474,13 +474,31 @@ class TLSHandshakeTest(nettest.NetTestCase): return connection else: - log.msg("State: %s" % connection.state_string()) - log.msg("Transmitted %d bytes" % connection.send("o\r\n")) + sent = connection.send("o\r\n") + log.debug("State: %s" % connection.state_string()) + log.debug("Transmitted %d bytes" % sent) + + _read_buffer = connection.pending() + log.debug("Max bytes in receive buffer: %d" % _read_buffer) + try: - recvstr = connection.recv(1024) - except SSL.WantReadError: - log.msg("Timeout exceeded") - connection.shutdown() + received = connection.recv(int(_read_buffer)) + except SSL.WantReadError, wre: + if connection.want_read(): + connection = handleWantRead(connection) + else: + ## if we still have an SSL_ERROR_WANT_READ, then try + ## to renegotiate + connection = connectionRenegotiate(connection, + connection.getpeername(), + wre.message) + except SSL.WantWriteError, wwe: + log.debug("State: %s" % connection.state_string()) + if connection.want_write(): + connection = handleWantWrite(connection) + else: + log.msg("Connection to %s:%s timed out." + % (peername, str(peerport))) else: log.msg("Received: %s" % recvstr) return connection

1 0

[ooni-probe/develop] Add method handleWantWrite() for resending already sent data over the
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 80a1dfedd7d266e83d70080bae8f73192dd09bfb Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:24:07 2013 +0000 Add method handleWantWrite() for resending already sent data over the connection, if OpenSSL's memory BIO state machine reports this error, and also jump to the SSL_ERROR_WANT_READ handler if at any point that error is raised. --- nettests/experimental/tls_handshake.py | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 7e71579..dc046a8 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -421,6 +421,58 @@ class TLSHandshakeTest(nettest.NetTestCase): handleWantWrite(connection) return connection + def handleWantWrite(connection): + """ + See :func:`handleWantRead`. + """ + try: + while connection.want_write(): + log.debug("Connection to %s HAS want_write" % host) + sleep(1) + resent = connection.send("o\r\n") + log.debug("Sent: %d" % resent) + log.debug("State: %s" % connection.state_string()) + except SSL.WantReadError, wre: + log.debug("Got WantReadError while handling want_write") + log.debug("WantReadError: %s" % wre.message) + log.debug("Switching to handleWantRead()...") + handleWantRead(connection) + return connection + + def doHandshake(connection): + """ + Attemp a TLS/SSL handshake with the host. + + If, after the first attempt at handshaking, OpenSSL's memory BIO + state machine does not report success, then try reading and + writing from the connection, and handle any SSL_ERROR_WANT_READ or + SSL_ERROR_WANT_WRITE which occurs. + + If multiple want_reads occur, then try renegotiation with the + host, and start over. If multiple want_writes occur, then it is + possible that the connection has timed out, and move on to the + connectionShutdown step. + + @param connection: A :class:`OpenSSL.SSL.Connection`. + @ivar peername: The host IP address, as reported by getpeername(). + @ivar peerport: The host port, reported by getpeername(). + @ivar sent: The number of bytes sent to to the remote host. + @ivar received: The number of bytes received from the remote host. + @ivar _read_buffer: An integer for the max bytes that can be read + from the connection. + @returns: The :param:connection with handshake completed, else + the unhandled error that was raised. + """ + peername, peerport = connection.getpeername() + + log.msg("Attempting handshake: %s" % peername) + connection.do_handshake() + log.debug("State: %s" % connection.state_string()) + if connection.state_string() == \ + 'SSL negotiation finished successfully': + ## jump to handshakeSuccessful and get certchain + return connection + else: log.msg("State: %s" % connection.state_string()) log.msg("Transmitted %d bytes" % connection.send("o\r\n"))

1 0

[ooni-probe/develop] Add method connectionShutdown() for calling shutdown on the TLS/SSL handshake,
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit e3d14a22dc73f9f0fa64ebe01daf6c25496ae010 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:22:07 2013 +0000 Add method connectionShutdown() for calling shutdown on the TLS/SSL handshake, and the socket, and for handling instances where the connection has been halfway closed. --- nettests/experimental/tls_handshake.py | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 128812e..7e71579 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -295,6 +295,56 @@ class TLSHandshakeTest(nettest.NetTestCase): log.debug("State: %s" % connection.state_string()) return connection + def connectionShutdown(connection, host): + """ + Handle shutting down a :class:`OpenSSL.SSL.Connection`, including + correct handling of halfway shutdown connections. + + Calls to :meth:`OpenSSL.SSL.Connection.shutdown` return a boolean + value: if the connection is already shutdown, it returns True, + else it returns false. Thus we loop through a block which detects + if the connection is an a partial shutdown state and corrects that + if that is the case, else it waits for one second, then attempts + shutting down the connection again. + + Detection of a partial shutdown state is done through + :meth:`OpenSSL.SSL.Connection.get_shutdown` which queries OpenSSL + for a bitvector of the server and client shutdown states. For + example, the binary string '0b00' is an open connection, and + '0b10' is a partially closed connection that has been shutdown on + the serverside. + + @param connection: A :class:`OpenSSL.SSL.Connection`. + + @param host: A tuple of: a string representation of the remote + host's IP address, and an integer specifying the + port. + """ + peername, peerport = host + + if isinstance(connection, SSL.Connection): + log.msg("Closing connection to %s:%d..." + % (peername, peerport)) + while not connection.shutdown(): + ## if the connection is halfway shutdown, we have to + ## wait for a ZeroReturnError on connection.recv(): + if (bin(connection.get_shutdown()) == '0b01') \ + or (bin(connection.get_shutdown()) == '0b10'): + try: + _read_buffer = connection.pending() + connection.recv(_read_buffer) + except SSL.ZeroReturnError, zre: continue + else: + sleep(1) + else: + log.msg("Closed connection to %s:%d" + % (peername, peerport)) + elif isinstance(connection, types.NoneType): + log.debug("connectionShutdown: got NoneType for connection") + else: + log.debug("connectionShutdown: expected connection, got %s" + % connection.__repr__()) + return connection def handleWantRead(connection): """

1 0