June 2013 - tor-commits - lists.torproject.org

[ooni-probe/develop] Add method handleWantRead() for handling SSL_ERROR_WANT_READ until OpenSSL's
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit c6f6ce0516b544b6ae25da6dc940978c11fdb544 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:19:58 2013 +0000 Add method handleWantRead() for handling SSL_ERROR_WANT_READ until OpenSSL's memory BIO state machine reports that it has been handled, and also jump to the SSL_ERROR_WANT_WRITE handler when/if that error occurs. --- nettests/experimental/tls_handshake.py | 79 ++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 81cab6a..128812e 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -295,11 +295,82 @@ class TLSHandshakeTest(nettest.NetTestCase): log.debug("State: %s" % connection.state_string()) return connection + + def handleWantRead(connection): + """ + From OpenSSL memory BIO documentation on ssl_read(): + + If the underlying BIO is blocking, SSL_read() will only + return, once the read operation has been finished or an error + occurred, except when a renegotiation take place, in which + case a SSL_ERROR_WANT_READ may occur. This behaviour can be + controlled with the SSL_MODE_AUTO_RETRY flag of the + SSL_CTX_set_mode(3) call. + + If the underlying BIO is non-blocking, SSL_read() will also + return when the underlying BIO could not satisfy the needs of + SSL_read() to continue the operation. In this case a call to + SSL_get_error(3) with the return value of SSL_read() will + yield SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE. As at any + time a re-negotiation is possible, a call to SSL_read() can + also cause write operations! The calling process then must + repeat the call after taking appropriate action to satisfy the + needs of SSL_read(). The action depends on the underlying + BIO. When using a non-blocking socket, nothing is to be done, + but select() can be used to check for the required condition. + + And from the OpenSSL memory BIO documentation on ssl_get_error(): + + SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE + + The operation did not complete; the same TLS/SSL I/O function + should be called again later. If, by then, the underlying BIO + has data available for reading (if the result code is + SSL_ERROR_WANT_READ) or allows writing data + (SSL_ERROR_WANT_WRITE), then some TLS/SSL protocol progress + will take place, i.e. at least part of an TLS/SSL record will + be read or written. Note that the retry may again lead to a + SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE condition. There + is no fixed upper limit for the number of iterations that may + be necessary until progress becomes visible at application + protocol level. + + For socket BIOs (e.g. when SSL_set_fd() was used), select() or + poll() on the underlying socket can be used to find out when + the TLS/SSL I/O function should be retried. + + Caveat: Any TLS/SSL I/O function can lead to either of + SSL_ERROR_WANT_READ and SSL_ERROR_WANT_WRITE. In particular, + SSL_read() or SSL_peek() may want to write data and + SSL_write() may want to read data. This is mainly because + TLS/SSL handshakes may occur at any time during the protocol + (initiated by either the client or the server); SSL_read(), + SSL_peek(), and SSL_write() will handle any pending + handshakes. + + Also, see http://stackoverflow.com/q/3952104 + """ try: - connection.do_handshake() - except SSL.WantReadError(): - log.msg("Timeout exceeded.") - connection.shutdown() + while connection.want_read(): + log.debug("Connection to %s HAS want_read" % host) + _read_buffer = connection.pending() + log.debug("Rereading %d bytes..." % _read_buffer) + sleep(1) + rereceived = connection.recv(int(_read_buffer)) + log.debug("Received %d bytes" % rereceived) + log.debug("State: %s" % connection.state_string()) + else: + peername, peerport = connection.getpeername() + log.debug("Connection to %s:%s DOES NOT HAVE want_read" + % (peername, peerport)) + log.debug("State: %s" % connection.state_string()) + except SSL.WantWriteError, wwe: + log.debug("Got WantWriteError while handling want_read") + log.debug("WantWriteError: %s" % wwe.message) + log.debug("Switching to handleWantWrite()...") + handleWantWrite(connection) + return connection + else: log.msg("State: %s" % connection.state_string()) log.msg("Transmitted %d bytes" % connection.send("o\r\n"))

1 0

[ooni-probe/develop] Add reporting of statistics for when the handshake fails.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 021f0e213c431bb7922d6f587d96c2fc4cfe5add Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:38:18 2013 +0000 Add reporting of statistics for when the handshake fails. --- nettests/experimental/tls_handshake.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index da6cbc0..c1e7ca5 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -561,6 +561,23 @@ class TLSHandshakeTest(nettest.NetTestCase): ## xxx do we need this? #return connection + def handshakeFailed(connection, host): + """ + xxx fill me in + + @param connection: A :class:`twisted.python.failure.Failure` or + :class:`exceptions.Exception`. + @param host: A tuple of the host IP and port, i.e. ('1.1.1.1', 443). + @returns: None. + """ + addr, port = host + log.msg("Handshake with %s:%d failed!" % host) + self.report['host'] = host + self.report['port'] = port + self.report['state'] = "HANDSHAKE_FAILED" + ## xxx do we need this? + #return connection + else: return handshakeSucceeded(connection)

1 0

[ooni-probe/develop] Update import statements in tls_handshake.py and bump the version number.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 56b48d7e25ee9607154657f6f117509c68f8f08c Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:39:09 2013 +0000 Update import statements in tls_handshake.py and bump the version number. --- nettests/experimental/tls_handshake.py | 37 +++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index c1e7ca5..0bd84bc 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -3,31 +3,42 @@ """ tls_handshake.py ---------------- + This file contains test cases for determining if a TLS handshake completes successfully, including ways to test if a TLS handshake which uses Mozilla - Firefox's current ciphersuite list completes. + Firefox's current ciphersuite list completes. Rather than using Twisted and + OpenSSL's methods for automatically completing a handshake, which includes + setting all the parameters, such as the ciphersuite list, these tests use + non-blocking sockets and implement asychronous error-handling transversal of + OpenSSL's memory BIO state machine, allowing us to determine where and why a + handshake fails. - These NetTestCases are a rewrite of a script contributed by Hackerberry - Finn, in order to fit into OONI's core network tests. + This network test is a complete rewrite of a pseudonymously contributed + script by Hackerberry Finn, in order to fit into OONI's core network tests. - @authors: Isis Agora Lovecruft <isis(a)torproject.org>, - Hackerberry Finn <anon@localhost> + @authors: Isis Agora Lovecruft <isis(a)torproject.org> @license: see included LICENSE file + @copyright: © 2013 Isis Lovecruft, The Tor Project Inc. """ +from socket import error as socket_error +from time import sleep + import os import socket -from socket import error as serror import struct import sys +import types -from ipaddr import IPAddress -from OpenSSL import SSL -from twisted.internet import defer -from twisted.python import usage +from ipaddr import IPAddress +from OpenSSL import SSL +from OpenSSL.crypto import dump_certificate, FILETYPE_PEM +from twisted.internet import defer +from twisted.python import usage +from twisted.python.failure import Failure -from ooni import nettest, config -from ooni.utils import log +from ooni import nettest, config +from ooni.utils import log, NotRootError ## For a way to obtain the current version of Firefox's default ciphersuite ## list, see https://trac.torproject.org/projects/tor/attachment/ticket/4744/ @@ -90,7 +101,7 @@ class TLSHandshakeTest(nettest.NetTestCase): name = 'tls-handshake' author = 'Isis Lovecruft <isis(a)torproject.org>' description = 'A test to determing if we can complete a TLS hankshake.' - version = '0.0.1' + version = '0.0.2' requiresRoot = False usageOptions = UsageOptions

1 0

[ooni-probe/develop] Add logging of undetermined states reported by OpenSSL's state machine to
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit b016417340b8ff3e4cdd860215157f0dfa8944f7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:33:16 2013 +0000 Add logging of undetermined states reported by OpenSSL's state machine to doHandshake(). This should not ever occur, but just in case it does, we'll want to log it rather than throwing the Exception out. --- nettests/experimental/tls_handshake.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index a02cb2f..c77682d 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -500,7 +500,9 @@ class TLSHandshakeTest(nettest.NetTestCase): log.msg("Connection to %s:%s timed out." % (peername, str(peerport))) else: - log.msg("Received: %s" % recvstr) + log.msg("Received: %s" % received) + log.debug("State: %s" % connection.state_string()) + return connection def handshakeSucceeded(connection):

1 0

[ooni-probe/develop] Add reporting mechanisms to handshakeSucceeded() for storing the x509
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 28bb44a8b21c8b2488580f2fdbbf10cd1a5ccd75 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:35:09 2013 +0000 Add reporting mechanisms to handshakeSucceeded() for storing the x509 certificate and chain in a way that YAML won't complain about. * TODO: we may want to look into a better method for serialising certificates that is compatible with YAML, rather than just storing them as PEM encoded certs in a string. * Also, pyOpenSSL will segfault if you try to check the cert. Added a warning in the docstring of method handshakeSucceeded() about that. --- nettests/experimental/tls_handshake.py | 66 ++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index c77682d..da6cbc0 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -506,17 +506,61 @@ class TLSHandshakeTest(nettest.NetTestCase): return connection def handshakeSucceeded(connection): - if connection: - host, port = connection.getpeername() - self.report['host'] = host - self.report['port'] = port - self.report['state'] = connection.state_string() - - def handshakeFailed(connection, addr, port): - if connection is None: - self.report['host'] = addr - self.report['port'] = port - self.report['state'] = 'FAILED' + """ + Get the details from the server certificate, cert chain, and + server ciphersuite list, and put them in our report. + + WARNING: do *not* do this: + >>> server_cert.get_pubkey() + <OpenSSL.crypto.PKey at 0x4985d28> + >>> pk = server_cert.get_pubkey() + >>> pk.check + <function check> + >>> pk.check() + Segmentation fault + + @param connection: A :class:`OpenSSL.SSL.Connection`. + @returns: None. + """ + host, port = connection.getpeername() + server_cert = self.getPeerCert(connection) + server_cert_chain = self.getPeerCert(connection, get_chain=True) + + s_cert = connection.get_peer_certificate() + cert_subject = s_cert.get_subject() + cert_subj_hash = s_cert.subject_name_hash() + cert_issuer = s_cert.get_issuer() + cert_public_key = s_cert.get_pubkey() + cert_serial_no = s_cert.get_serial_number() + cert_sig_algo = s_cert.get_signature_algorithm() + + self.report['host'] = host + self.report['port'] = port + self.report['state'] = connection.state_string() + self.report['renegotiations'] = connection.total_renegotiations() + self.report['server_cert'] = server_cert + self.report['server_cert_chain'] = \ + ''.join([cert for cert in server_cert_chain]) + self.report['server_ciphersuite'] = connection.get_cipher_list() + self.report['cert_subject'] = str(cert_subject) + self.report['cert_subj_hash'] = str(cert_subj_hash) + self.report['cert_issuer'] = str(cert_issuer) + ## xxx this needs to be parsed into PEM also + self.report['cert_public_key'] = str(cert_public_key) + self.report['cert_serial_no'] = str(cert_serial_no) + self.report['cert_sig_algo'] = str(cert_sig_algo) + + ## The session's master key is only valid for that session, and + ## will allow us to decrypt any packet captures (if they were + ## collected). Because we are not requesting URLs, only host:port + ## (which would be visible in pcaps anyway, since the FQDN is + ## never encrypted) I do not see a way for this to log any user or + ## identifying information. Correct me if I'm wrong. + self.report['session_key'] = connection.master_key() + + ## xxx do we need this? + #return connection + else: return handshakeSucceeded(connection)

1 0

[ooni-probe/develop] Switch to using threads.deferToThread() in method deferMakeConnection().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit f69452eac1cf5334ab84caab260cbfcaf3adc2dd Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 18:35:49 2013 +0000 Switch to using threads.deferToThread() in method deferMakeConnection(). --- nettests/experimental/tls_handshake.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 48902b3..ddef435 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -696,15 +696,23 @@ class TLSHandshakeTest(nettest.NetTestCase): self.report['host'] = host self.report['port'] = port - @defer.inlineCallbacks - def deferMakeConnection(host): - connection = yield makeConnection(host) - if isinstance(connection, Failure) \ - or isinstance(connection, Exception): - failed = connectionFailed(connection, host) - defer.returnValue(failed) + if isinstance(connection, Exception) \ + or isinstance(connection, ConnectionTimeout): + log.msg("Handshake failed with reason: %s" % connection.message) + self.report['state'] = connection.message + elif isinstance(connection, failure.Failure): + log.msg("Handshake failed with reason: Socket %s" + % connection.getErrorMessage()) + self.report['state'] = connection.getErrorMessage() + ctmo = connection.trap(ConnectionTimeout) + if ctmo == ConnectionTimeout: + connection.cleanFailure() else: - defer.returnValue(connection) + log.msg("Handshake failed with reason: %s" % str(connection)) + if not 'state' in self.report.keys(): + self.report['state'] = str(connection) + + return None connection = deferMakeConnection(self.input) connection.addCallbacks(connectionSucceeded, connectionFailed,

1 0

[ooni-probe/develop] Fix method doHandshake() to callback to handleWantRead() and handleWantWrite().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 27292c9c868694ca19e2d2b9aa9c32220f9d89cf Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 17:57:05 2013 +0000 Fix method doHandshake() to callback to handleWantRead() and handleWantWrite(). --- nettests/experimental/tls_handshake.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 3c9a033..e443242 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -553,17 +553,29 @@ class TLSHandshakeTest(nettest.NetTestCase): """ peername, peerport = connection.getpeername() - log.msg("Attempting handshake: %s" % peername) - connection.do_handshake() - log.debug("State: %s" % connection.state_string()) - if connection.state_string() == \ - 'SSL negotiation finished successfully': + try: + log.msg("Attempting handshake: %s" % peername) + connection.do_handshake() + except OpenSSL.SSL.WantReadError() as wre: + self.state = connection.state_string() + log.debug("Handshake state: %s" % self.state) + log.debug("doHandshake: WantReadError on first handshake attempt.") + connection = handleWantRead(connection) + except OpenSSL.SSL.WantWriteError() as wwe: + self.state = connection.state_string() + log.debug("Handshake state: %s" % self.state) + log.debug("doHandshake: WantWriteError on first handshake attempt.") + connection = handleWantWrite(connection) + else: + self.state = connection.state_string() + + if self.state == 'SSL negotiation finished successfully': ## jump to handshakeSuccessful and get certchain return connection - else: sent = connection.send("o\r\n") - log.debug("State: %s" % connection.state_string()) + self.state = connection.state_string() + log.debug("Handshake state: %s" % self.state) log.debug("Transmitted %d bytes" % sent) _read_buffer = connection.pending()

1 0

[ooni-probe/develop] Add call to new deferMakeConnection() to test_tls_handshake().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 5e1f148a6171efddc96afdd6930978387a052cb5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 18:37:18 2013 +0000 Add call to new deferMakeConnection() to test_tls_handshake(). --- nettests/experimental/tls_handshake.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index ddef435..1b66b45 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -714,6 +714,9 @@ class TLSHandshakeTest(nettest.NetTestCase): return None + def deferMakeConnection(host): + return threads.deferToThread(makeConnection, self.input) + connection = deferMakeConnection(self.input) connection.addCallbacks(connectionSucceeded, connectionFailed, callbackArgs=[self.input, self.timeout],

1 0

[ooni-probe/develop] Implement modular probe IP address lookup function.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit f544c3ef06808c441ed6e7db3cd4dfd187b04aa3 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Mar 13 20:57:19 2013 +0100 Implement modular probe IP address lookup function. --- ooni/director.py | 153 ++++++++++++++++++++++++++++++++++++++++++++---- ooni/errors.py | 5 ++ ooni/nettest.py | 8 +-- ooni/utils/__init__.py | 5 +- tests/test_nettest.py | 1 - 5 files changed, 152 insertions(+), 20 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 8365ebd..07510eb 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -1,18 +1,151 @@ +import xml.etree.ElementTree as ET +import random import sys import os +import re from ooni import config from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report -from ooni.utils import log, checkForRoot, NotRootError +from ooni.utils import log, checkForRoot from ooni.utils.net import randomFreePort from ooni.nettest import NetTest -from ooni.errors import UnableToStartTor +from ooni import errors from txtorcon import TorConfig from txtorcon import TorState, launch_tor from twisted.internet import defer, reactor +from twisted.web import client, http_headers +from ooni.utils.net import userAgents, BodyReceiver + +class HTTPGeoIPLookupper(object): + url = None + + def _response(self, response): + content_length = response.headers.getRawHeaders('content-length') + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished, content_length)) + finished.addCallback(self.parseResponse) + return finished + + def parseResponse(self, response_body): + """ + Override this with the logic for parsing the response. + + Should return the IP address of the probe. + """ + pass + + def failed(self, failure): + log.err("Failed to lookup via %s" % url) + log.exception(failure) + return failure + + def lookup(self): + agent = client.Agent(reactor) + headers = {} + headers['User-Agent'] = [random.choice(userAgents)] + + d = agent.request("GET", self.url, http_headers.Headers(headers)) + d.addCallback(self._response) + d.addErrback(self.failed) + return d + +class UbuntuGeoIP(HTTPGeoIPLookupper): + url = "http://geoip.ubuntu.com/lookup" + + def parseResponse(self, response_body): + response = ET.fromstring(response_body) + probe_ip = response.find('Ip').text + return probe_ip + +class TorProjectGeoIP(HTTPGeoIPLookupper): + url = "https://check.torproject.org/" + + def parseResponse(self, response_body): + regexp = "Your IP address appears to be: <b>((\d+\.)+(\d+))" + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class MaxMindGeoIP(HTTPGeoIPLookupper): + url = "https://www.maxmind.com/en/locate_my_ip" + + def parseResponse(self, response_body): + regexp = '<span id="my-ip-address">((\d+\.)+(\d+))</span>' + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class ProbeIP(object): + strategy = None + geoIPServices = {'ubuntu': UbuntuGeoIP, + 'torproject': TorProjectGeoIP, + 'maximind': MaxMindGeoIP + } + address = None + + @defer.inlineCallbacks + def lookup(self): + try: + yield self.askTor() + defer.returnValue(self.address) + except errors.TorStateNotFound: + log.debug("Tor is not running. Skipping IP lookup via Tor.") + except: + log.msg("Unable to lookup the probe IP via Tor.") + + try: + yield self.askTraceroute() + defer.returnValue(self.address) + except errors.InsufficientPrivileges: + log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") + except: + log.msg("Unable to lookup the probe IP via traceroute") + + try: + yield self.askGeoIPService() + defer.returnValue(self.address) + except Exception, e: + print e + log.msg("Unable to lookup the probe IP via GeoIPService") + + @defer.inlineCallbacks + def askGeoIPService(self): + for service_name, service in self.geoIPServices.items(): + s = TorProjectGeoIP() + log.msg("Looking up your IP address via %s" % service_name) + try: + self.address = yield s.lookup() + self.strategy = 'geo_ip_service-' + service_name + break + except: + log.msg("Failed to lookup your IP via %s" % service_name) + + def askTraceroute(self): + """ + Perform a UDP traceroute to determine the probes IP address. + """ + checkForRoot() + raise NotImplemented + + def askTor(self): + """ + Obtain the probes IP address by asking the Tor Control port via GET INFO + address. + + XXX this lookup method is currently broken when there are cached descriptors or consensus documents + see: https://trac.torproject.org/projects/tor/ticket/8214 + """ + if config.tor_state: + d = config.tor_state.protocol.get_info("address") + @d.addCallback + def cb(result): + self.strategy = 'tor_get_info_address' + self.address = result.values()[0] + return d + else: + raise errors.TorStateNotFound class Director(object): """ @@ -80,6 +213,7 @@ class Director(object): self.torControlProtocol = None + @defer.inlineCallbacks def start(self): if config.privacy.includepcap: log.msg("Starting") @@ -89,10 +223,10 @@ class Director(object): if config.advanced.start_tor: log.msg("Starting Tor...") - d = self.startTor() - else: - d = defer.succeed(None) - return d + yield self.startTor() + + config.probe_ip = ProbeIP() + yield config.probe_ip.lookup() @property def measurementSuccessRatio(self): @@ -200,7 +334,7 @@ class Director(object): from ooni.utils.txscapy import ScapyFactory, ScapySniffer try: checkForRoot() - except NotRootError: + except errors.InsufficientPrivileges: print "[!] Includepcap options requires root priviledges to run" print " you should run ooniprobe as root or disable the options in ooniprobe.conf" sys.exit(1) @@ -232,18 +366,15 @@ class Director(object): socks_port = yield state.protocol.get_conf("SocksPort") control_port = yield state.protocol.get_conf("ControlPort") - client_ip = yield state.protocol.get_info("address") config.tor.socks_port = int(socks_port.values()[0]) config.tor.control_port = int(control_port.values()[0]) - config.probe_ip = client_ip.values()[0] - log.debug("Obtained our IP address from a Tor Relay %s" % config.probe_ip) def setup_failed(failure): log.exception(failure) - raise UnableToStartTor + raise errors.UnableToStartTor def setup_complete(proto): """ diff --git a/ooni/errors.py b/ooni/errors.py index 36a042f..6b23727 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -132,3 +132,8 @@ class ReportNotCreated(Exception): class ReportAlreadyClosed(Exception): pass +class TorStateNotFound(Exception): + pass + +class InsufficientPrivileges(Exception): + pass diff --git a/ooni/nettest.py b/ooni/nettest.py index 6273d34..67dee9d 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -6,7 +6,7 @@ from twisted.trial.runner import filenameToModule from twisted.python import usage, reflect from ooni.tasks import Measurement -from ooni.utils import log, checkForRoot, NotRootError, geodata +from ooni.utils import log, checkForRoot, geodata from ooni import config from ooni import otime @@ -30,15 +30,15 @@ class NetTestLoader(object): from ooni import __version__ as software_version client_geodata = {} - if config.probe_ip and (config.privacy.includeip or \ + if config.probe_ip.address and (config.privacy.includeip or \ config.privacy.includeasn or \ config.privacy.includecountry or \ config.privacy.includecity): log.msg("We will include some geo data in the report") - client_geodata = geodata.IPToLocation(config.probe_ip) + client_geodata = geodata.IPToLocation(config.probe_ip.address) if config.privacy.includeip: - client_geodata['ip'] = config.probe_ip + client_geodata['ip'] = config.probe_ip.address else: client_geodata['ip'] = "127.0.0.1" diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 8510a3b..340693b 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -48,12 +48,9 @@ class Storage(dict): for (k, v) in value.items(): self[k] = v -class NotRootError(Exception): - pass - def checkForRoot(): if os.getuid() != 0: - raise NotRootError("This test requires root") + raise errors.InsufficientPrivileges def randomSTR(length, num=True): """ diff --git a/tests/test_nettest.py b/tests/test_nettest.py index a0ccb32..976757f 100644 --- a/tests/test_nettest.py +++ b/tests/test_nettest.py @@ -9,7 +9,6 @@ from twisted.python.usage import UsageError from ooni.nettest import NetTest, InvalidOption, MissingRequiredOption from ooni.nettest import NetTestLoader, FailureToLoadNetTest from ooni.tasks import BaseTask -from ooni.utils import NotRootError from ooni.director import Director

1 0

[ooni-probe/develop] Write unittests also for Tor geoip stuff
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit d73c5fa22a72413fd1c5d24eeb1e9118f0a4662a Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 15 16:31:14 2013 +0100 Write unittests also for Tor geoip stuff --- ooni/errors.py | 3 +++ ooni/geoip.py | 16 ++++++++++++---- tests/test_geoip.py | 45 +++++++++++++++++++++++++++++++++++++++------ 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 62fcf93..69af726 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -138,3 +138,6 @@ class TorStateNotFound(Exception): class InsufficientPrivileges(Exception): pass + +class ProbeIPUnknown(Exception): + pass diff --git a/ooni/geoip.py b/ooni/geoip.py index e6a221b..ae06608 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -110,19 +110,22 @@ class ProbeIP(object): 'maxmind': MaxMindGeoIP } address = None + tor_state = config.tor_state @defer.inlineCallbacks def lookup(self): try: yield self.askTor() + log.msg("Found your IP via Tor %s" % self.address) defer.returnValue(self.address) except errors.TorStateNotFound: log.debug("Tor is not running. Skipping IP lookup via Tor.") - except: + except Exception: log.msg("Unable to lookup the probe IP via Tor.") try: yield self.askTraceroute() + log.msg("Found your IP via Traceroute %s" % self.address) defer.returnValue(self.address) except errors.InsufficientPrivileges: log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") @@ -131,9 +134,11 @@ class ProbeIP(object): try: yield self.askGeoIPService() + log.msg("Found your IP via a GeoIP service: %s" % self.address) defer.returnValue(self.address) - except: + except Exception, e: log.msg("Unable to lookup the probe IP via GeoIPService") + raise e @defer.inlineCallbacks def askGeoIPService(self): @@ -150,6 +155,9 @@ class ProbeIP(object): except Exception, e: log.msg("Failed to lookup your IP via %s" % service_name) + if not self.address: + raise errors.ProbeIPUnknown + def askTraceroute(self): """ Perform a UDP traceroute to determine the probes IP address. @@ -165,8 +173,8 @@ class ProbeIP(object): XXX this lookup method is currently broken when there are cached descriptors or consensus documents see: https://trac.torproject.org/projects/tor/ticket/8214 """ - if config.tor_state: - d = config.tor_state.protocol.get_info("address") + if self.tor_state: + d = self.tor_state.protocol.get_info("address") @d.addCallback def cb(result): self.strategy = 'tor_get_info_address' diff --git a/tests/test_geoip.py b/tests/test_geoip.py index 443d262..0d7165f 100644 --- a/tests/test_geoip.py +++ b/tests/test_geoip.py @@ -1,5 +1,7 @@ +from collections import namedtuple + from twisted.web import server, static, resource -from twisted.internet import reactor +from twisted.internet import reactor, defer from twisted.trial import unittest from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory @@ -58,7 +60,7 @@ class TestGeoIPServices(GeoIPBaseTest): gip = TorProjectGeoIP() gip.url = self.getUrl('torproject') d = gip.lookup() - @d.addCallback + @d.addBoth def cb(res): self.assertEqual(res, '127.0.0.1') return d @@ -67,7 +69,7 @@ class TestGeoIPServices(GeoIPBaseTest): gip = UbuntuGeoIP() gip.url = self.getUrl('ubuntu') d = gip.lookup() - @d.addCallback + @d.addBoth def cb(res): self.assertEqual(res, '127.0.0.1') return d @@ -76,7 +78,7 @@ class TestGeoIPServices(GeoIPBaseTest): gip = MaxMindGeoIP() gip.url = self.getUrl('maxmind') d = gip.lookup() - @d.addCallback + @d.addBoth def cb(res): self.assertEqual(res, '127.0.0.1') return d @@ -93,7 +95,7 @@ class TestProbeIP(GeoIPBaseTest): def test_ask_geoip_service(self): d = self.probe_ip.askGeoIPService() - @d.addCallback + @d.addBoth def cb(res): self.assertEqual(self.probe_ip.address, '127.0.0.1') return d @@ -102,7 +104,38 @@ class TestProbeIP(GeoIPBaseTest): self.assertRaises(errors.InsufficientPrivileges, self.probe_ip.askTraceroute) def test_ask_tor(self): - pass + class MockTorState(object): + """ + This is a Mock Tor state object. It will just pretend to answer to + the get_info("address") method call. + """ + protocol = namedtuple('Protocol', 'get_info') + def __init__(self): + def get_info(key): + return defer.succeed({'XXX': '127.0.0.2'}) + self.protocol = self.protocol(get_info=get_info) + + self.probe_ip.tor_state = MockTorState() + d = self.probe_ip.lookup() + @d.addBoth + def cb(res): + self.assertEqual(self.probe_ip.address, '127.0.0.2') + return d + + def test_probe_ip(self): + d = self.probe_ip.lookup() + @d.addBoth + def cb(res): + self.assertEqual(self.probe_ip.address, '127.0.0.1') + self.assertTrue(self.probe_ip.strategy.startswith('geo_ip_service-')) + return d + + def test_failing_probe_ip(self): + self.probe_ip.geoIPServices = {} + + d = self.probe_ip.lookup() + self.assertFailure(d, errors.ProbeIPUnknown) + return d class TestIPToLocation(unittest.TestCase): pass

1 0