June 2013 - tor-commits - lists.torproject.org

[ooni-probe/develop] Add reporting of statistics for when the handshake fails.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 021f0e213c431bb7922d6f587d96c2fc4cfe5add Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:38:18 2013 +0000 Add reporting of statistics for when the handshake fails. --- nettests/experimental/tls_handshake.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index da6cbc0..c1e7ca5 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -561,6 +561,23 @@ class TLSHandshakeTest(nettest.NetTestCase): ## xxx do we need this? #return connection + def handshakeFailed(connection, host): + """ + xxx fill me in + + @param connection: A :class:`twisted.python.failure.Failure` or + :class:`exceptions.Exception`. + @param host: A tuple of the host IP and port, i.e. ('1.1.1.1', 443). + @returns: None. + """ + addr, port = host + log.msg("Handshake with %s:%d failed!" % host) + self.report['host'] = host + self.report['port'] = port + self.report['state'] = "HANDSHAKE_FAILED" + ## xxx do we need this? + #return connection + else: return handshakeSucceeded(connection)

1 0

[ooni-probe/develop] Update import statements in tls_handshake.py and bump the version number.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 56b48d7e25ee9607154657f6f117509c68f8f08c Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:39:09 2013 +0000 Update import statements in tls_handshake.py and bump the version number. --- nettests/experimental/tls_handshake.py | 37 +++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index c1e7ca5..0bd84bc 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -3,31 +3,42 @@ """ tls_handshake.py ---------------- + This file contains test cases for determining if a TLS handshake completes successfully, including ways to test if a TLS handshake which uses Mozilla - Firefox's current ciphersuite list completes. + Firefox's current ciphersuite list completes. Rather than using Twisted and + OpenSSL's methods for automatically completing a handshake, which includes + setting all the parameters, such as the ciphersuite list, these tests use + non-blocking sockets and implement asychronous error-handling transversal of + OpenSSL's memory BIO state machine, allowing us to determine where and why a + handshake fails. - These NetTestCases are a rewrite of a script contributed by Hackerberry - Finn, in order to fit into OONI's core network tests. + This network test is a complete rewrite of a pseudonymously contributed + script by Hackerberry Finn, in order to fit into OONI's core network tests. - @authors: Isis Agora Lovecruft <isis(a)torproject.org>, - Hackerberry Finn <anon@localhost> + @authors: Isis Agora Lovecruft <isis(a)torproject.org> @license: see included LICENSE file + @copyright: © 2013 Isis Lovecruft, The Tor Project Inc. """ +from socket import error as socket_error +from time import sleep + import os import socket -from socket import error as serror import struct import sys +import types -from ipaddr import IPAddress -from OpenSSL import SSL -from twisted.internet import defer -from twisted.python import usage +from ipaddr import IPAddress +from OpenSSL import SSL +from OpenSSL.crypto import dump_certificate, FILETYPE_PEM +from twisted.internet import defer +from twisted.python import usage +from twisted.python.failure import Failure -from ooni import nettest, config -from ooni.utils import log +from ooni import nettest, config +from ooni.utils import log, NotRootError ## For a way to obtain the current version of Firefox's default ciphersuite ## list, see https://trac.torproject.org/projects/tor/attachment/ticket/4744/ @@ -90,7 +101,7 @@ class TLSHandshakeTest(nettest.NetTestCase): name = 'tls-handshake' author = 'Isis Lovecruft <isis(a)torproject.org>' description = 'A test to determing if we can complete a TLS hankshake.' - version = '0.0.1' + version = '0.0.2' requiresRoot = False usageOptions = UsageOptions

1 0

[ooni-probe/develop] Add logging of undetermined states reported by OpenSSL's state machine to
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit b016417340b8ff3e4cdd860215157f0dfa8944f7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:33:16 2013 +0000 Add logging of undetermined states reported by OpenSSL's state machine to doHandshake(). This should not ever occur, but just in case it does, we'll want to log it rather than throwing the Exception out. --- nettests/experimental/tls_handshake.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index a02cb2f..c77682d 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -500,7 +500,9 @@ class TLSHandshakeTest(nettest.NetTestCase): log.msg("Connection to %s:%s timed out." % (peername, str(peerport))) else: - log.msg("Received: %s" % recvstr) + log.msg("Received: %s" % received) + log.debug("State: %s" % connection.state_string()) + return connection def handshakeSucceeded(connection):

1 0

[ooni-probe/develop] Add reporting mechanisms to handshakeSucceeded() for storing the x509
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 28bb44a8b21c8b2488580f2fdbbf10cd1a5ccd75 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 04:35:09 2013 +0000 Add reporting mechanisms to handshakeSucceeded() for storing the x509 certificate and chain in a way that YAML won't complain about. * TODO: we may want to look into a better method for serialising certificates that is compatible with YAML, rather than just storing them as PEM encoded certs in a string. * Also, pyOpenSSL will segfault if you try to check the cert. Added a warning in the docstring of method handshakeSucceeded() about that. --- nettests/experimental/tls_handshake.py | 66 ++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index c77682d..da6cbc0 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -506,17 +506,61 @@ class TLSHandshakeTest(nettest.NetTestCase): return connection def handshakeSucceeded(connection): - if connection: - host, port = connection.getpeername() - self.report['host'] = host - self.report['port'] = port - self.report['state'] = connection.state_string() - - def handshakeFailed(connection, addr, port): - if connection is None: - self.report['host'] = addr - self.report['port'] = port - self.report['state'] = 'FAILED' + """ + Get the details from the server certificate, cert chain, and + server ciphersuite list, and put them in our report. + + WARNING: do *not* do this: + >>> server_cert.get_pubkey() + <OpenSSL.crypto.PKey at 0x4985d28> + >>> pk = server_cert.get_pubkey() + >>> pk.check + <function check> + >>> pk.check() + Segmentation fault + + @param connection: A :class:`OpenSSL.SSL.Connection`. + @returns: None. + """ + host, port = connection.getpeername() + server_cert = self.getPeerCert(connection) + server_cert_chain = self.getPeerCert(connection, get_chain=True) + + s_cert = connection.get_peer_certificate() + cert_subject = s_cert.get_subject() + cert_subj_hash = s_cert.subject_name_hash() + cert_issuer = s_cert.get_issuer() + cert_public_key = s_cert.get_pubkey() + cert_serial_no = s_cert.get_serial_number() + cert_sig_algo = s_cert.get_signature_algorithm() + + self.report['host'] = host + self.report['port'] = port + self.report['state'] = connection.state_string() + self.report['renegotiations'] = connection.total_renegotiations() + self.report['server_cert'] = server_cert + self.report['server_cert_chain'] = \ + ''.join([cert for cert in server_cert_chain]) + self.report['server_ciphersuite'] = connection.get_cipher_list() + self.report['cert_subject'] = str(cert_subject) + self.report['cert_subj_hash'] = str(cert_subj_hash) + self.report['cert_issuer'] = str(cert_issuer) + ## xxx this needs to be parsed into PEM also + self.report['cert_public_key'] = str(cert_public_key) + self.report['cert_serial_no'] = str(cert_serial_no) + self.report['cert_sig_algo'] = str(cert_sig_algo) + + ## The session's master key is only valid for that session, and + ## will allow us to decrypt any packet captures (if they were + ## collected). Because we are not requesting URLs, only host:port + ## (which would be visible in pcaps anyway, since the FQDN is + ## never encrypted) I do not see a way for this to log any user or + ## identifying information. Correct me if I'm wrong. + self.report['session_key'] = connection.master_key() + + ## xxx do we need this? + #return connection + else: return handshakeSucceeded(connection)

1 0

[ooni-probe/develop] Switch to using threads.deferToThread() in method deferMakeConnection().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit f69452eac1cf5334ab84caab260cbfcaf3adc2dd Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 18:35:49 2013 +0000 Switch to using threads.deferToThread() in method deferMakeConnection(). --- nettests/experimental/tls_handshake.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 48902b3..ddef435 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -696,15 +696,23 @@ class TLSHandshakeTest(nettest.NetTestCase): self.report['host'] = host self.report['port'] = port - @defer.inlineCallbacks - def deferMakeConnection(host): - connection = yield makeConnection(host) - if isinstance(connection, Failure) \ - or isinstance(connection, Exception): - failed = connectionFailed(connection, host) - defer.returnValue(failed) + if isinstance(connection, Exception) \ + or isinstance(connection, ConnectionTimeout): + log.msg("Handshake failed with reason: %s" % connection.message) + self.report['state'] = connection.message + elif isinstance(connection, failure.Failure): + log.msg("Handshake failed with reason: Socket %s" + % connection.getErrorMessage()) + self.report['state'] = connection.getErrorMessage() + ctmo = connection.trap(ConnectionTimeout) + if ctmo == ConnectionTimeout: + connection.cleanFailure() else: - defer.returnValue(connection) + log.msg("Handshake failed with reason: %s" % str(connection)) + if not 'state' in self.report.keys(): + self.report['state'] = str(connection) + + return None connection = deferMakeConnection(self.input) connection.addCallbacks(connectionSucceeded, connectionFailed,

1 0

[ooni-probe/develop] Fix method doHandshake() to callback to handleWantRead() and handleWantWrite().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 27292c9c868694ca19e2d2b9aa9c32220f9d89cf Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 17:57:05 2013 +0000 Fix method doHandshake() to callback to handleWantRead() and handleWantWrite(). --- nettests/experimental/tls_handshake.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 3c9a033..e443242 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -553,17 +553,29 @@ class TLSHandshakeTest(nettest.NetTestCase): """ peername, peerport = connection.getpeername() - log.msg("Attempting handshake: %s" % peername) - connection.do_handshake() - log.debug("State: %s" % connection.state_string()) - if connection.state_string() == \ - 'SSL negotiation finished successfully': + try: + log.msg("Attempting handshake: %s" % peername) + connection.do_handshake() + except OpenSSL.SSL.WantReadError() as wre: + self.state = connection.state_string() + log.debug("Handshake state: %s" % self.state) + log.debug("doHandshake: WantReadError on first handshake attempt.") + connection = handleWantRead(connection) + except OpenSSL.SSL.WantWriteError() as wwe: + self.state = connection.state_string() + log.debug("Handshake state: %s" % self.state) + log.debug("doHandshake: WantWriteError on first handshake attempt.") + connection = handleWantWrite(connection) + else: + self.state = connection.state_string() + + if self.state == 'SSL negotiation finished successfully': ## jump to handshakeSuccessful and get certchain return connection - else: sent = connection.send("o\r\n") - log.debug("State: %s" % connection.state_string()) + self.state = connection.state_string() + log.debug("Handshake state: %s" % self.state) log.debug("Transmitted %d bytes" % sent) _read_buffer = connection.pending()

1 0

[ooni-probe/develop] Add call to new deferMakeConnection() to test_tls_handshake().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 5e1f148a6171efddc96afdd6930978387a052cb5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 28 18:37:18 2013 +0000 Add call to new deferMakeConnection() to test_tls_handshake(). --- nettests/experimental/tls_handshake.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index ddef435..1b66b45 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -714,6 +714,9 @@ class TLSHandshakeTest(nettest.NetTestCase): return None + def deferMakeConnection(host): + return threads.deferToThread(makeConnection, self.input) + connection = deferMakeConnection(self.input) connection.addCallbacks(connectionSucceeded, connectionFailed, callbackArgs=[self.input, self.timeout],

1 0

[ooni-probe/develop] Implement modular probe IP address lookup function.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit f544c3ef06808c441ed6e7db3cd4dfd187b04aa3 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Mar 13 20:57:19 2013 +0100 Implement modular probe IP address lookup function. --- ooni/director.py | 153 ++++++++++++++++++++++++++++++++++++++++++++---- ooni/errors.py | 5 ++ ooni/nettest.py | 8 +-- ooni/utils/__init__.py | 5 +- tests/test_nettest.py | 1 - 5 files changed, 152 insertions(+), 20 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 8365ebd..07510eb 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -1,18 +1,151 @@ +import xml.etree.ElementTree as ET +import random import sys import os +import re from ooni import config from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report -from ooni.utils import log, checkForRoot, NotRootError +from ooni.utils import log, checkForRoot from ooni.utils.net import randomFreePort from ooni.nettest import NetTest -from ooni.errors import UnableToStartTor +from ooni import errors from txtorcon import TorConfig from txtorcon import TorState, launch_tor from twisted.internet import defer, reactor +from twisted.web import client, http_headers +from ooni.utils.net import userAgents, BodyReceiver + +class HTTPGeoIPLookupper(object): + url = None + + def _response(self, response): + content_length = response.headers.getRawHeaders('content-length') + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished, content_length)) + finished.addCallback(self.parseResponse) + return finished + + def parseResponse(self, response_body): + """ + Override this with the logic for parsing the response. + + Should return the IP address of the probe. + """ + pass + + def failed(self, failure): + log.err("Failed to lookup via %s" % url) + log.exception(failure) + return failure + + def lookup(self): + agent = client.Agent(reactor) + headers = {} + headers['User-Agent'] = [random.choice(userAgents)] + + d = agent.request("GET", self.url, http_headers.Headers(headers)) + d.addCallback(self._response) + d.addErrback(self.failed) + return d + +class UbuntuGeoIP(HTTPGeoIPLookupper): + url = "http://geoip.ubuntu.com/lookup" + + def parseResponse(self, response_body): + response = ET.fromstring(response_body) + probe_ip = response.find('Ip').text + return probe_ip + +class TorProjectGeoIP(HTTPGeoIPLookupper): + url = "https://check.torproject.org/" + + def parseResponse(self, response_body): + regexp = "Your IP address appears to be: <b>((\d+\.)+(\d+))" + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class MaxMindGeoIP(HTTPGeoIPLookupper): + url = "https://www.maxmind.com/en/locate_my_ip" + + def parseResponse(self, response_body): + regexp = '<span id="my-ip-address">((\d+\.)+(\d+))</span>' + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class ProbeIP(object): + strategy = None + geoIPServices = {'ubuntu': UbuntuGeoIP, + 'torproject': TorProjectGeoIP, + 'maximind': MaxMindGeoIP + } + address = None + + @defer.inlineCallbacks + def lookup(self): + try: + yield self.askTor() + defer.returnValue(self.address) + except errors.TorStateNotFound: + log.debug("Tor is not running. Skipping IP lookup via Tor.") + except: + log.msg("Unable to lookup the probe IP via Tor.") + + try: + yield self.askTraceroute() + defer.returnValue(self.address) + except errors.InsufficientPrivileges: + log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") + except: + log.msg("Unable to lookup the probe IP via traceroute") + + try: + yield self.askGeoIPService() + defer.returnValue(self.address) + except Exception, e: + print e + log.msg("Unable to lookup the probe IP via GeoIPService") + + @defer.inlineCallbacks + def askGeoIPService(self): + for service_name, service in self.geoIPServices.items(): + s = TorProjectGeoIP() + log.msg("Looking up your IP address via %s" % service_name) + try: + self.address = yield s.lookup() + self.strategy = 'geo_ip_service-' + service_name + break + except: + log.msg("Failed to lookup your IP via %s" % service_name) + + def askTraceroute(self): + """ + Perform a UDP traceroute to determine the probes IP address. + """ + checkForRoot() + raise NotImplemented + + def askTor(self): + """ + Obtain the probes IP address by asking the Tor Control port via GET INFO + address. + + XXX this lookup method is currently broken when there are cached descriptors or consensus documents + see: https://trac.torproject.org/projects/tor/ticket/8214 + """ + if config.tor_state: + d = config.tor_state.protocol.get_info("address") + @d.addCallback + def cb(result): + self.strategy = 'tor_get_info_address' + self.address = result.values()[0] + return d + else: + raise errors.TorStateNotFound class Director(object): """ @@ -80,6 +213,7 @@ class Director(object): self.torControlProtocol = None + @defer.inlineCallbacks def start(self): if config.privacy.includepcap: log.msg("Starting") @@ -89,10 +223,10 @@ class Director(object): if config.advanced.start_tor: log.msg("Starting Tor...") - d = self.startTor() - else: - d = defer.succeed(None) - return d + yield self.startTor() + + config.probe_ip = ProbeIP() + yield config.probe_ip.lookup() @property def measurementSuccessRatio(self): @@ -200,7 +334,7 @@ class Director(object): from ooni.utils.txscapy import ScapyFactory, ScapySniffer try: checkForRoot() - except NotRootError: + except errors.InsufficientPrivileges: print "[!] Includepcap options requires root priviledges to run" print " you should run ooniprobe as root or disable the options in ooniprobe.conf" sys.exit(1) @@ -232,18 +366,15 @@ class Director(object): socks_port = yield state.protocol.get_conf("SocksPort") control_port = yield state.protocol.get_conf("ControlPort") - client_ip = yield state.protocol.get_info("address") config.tor.socks_port = int(socks_port.values()[0]) config.tor.control_port = int(control_port.values()[0]) - config.probe_ip = client_ip.values()[0] - log.debug("Obtained our IP address from a Tor Relay %s" % config.probe_ip) def setup_failed(failure): log.exception(failure) - raise UnableToStartTor + raise errors.UnableToStartTor def setup_complete(proto): """ diff --git a/ooni/errors.py b/ooni/errors.py index 36a042f..6b23727 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -132,3 +132,8 @@ class ReportNotCreated(Exception): class ReportAlreadyClosed(Exception): pass +class TorStateNotFound(Exception): + pass + +class InsufficientPrivileges(Exception): + pass diff --git a/ooni/nettest.py b/ooni/nettest.py index 6273d34..67dee9d 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -6,7 +6,7 @@ from twisted.trial.runner import filenameToModule from twisted.python import usage, reflect from ooni.tasks import Measurement -from ooni.utils import log, checkForRoot, NotRootError, geodata +from ooni.utils import log, checkForRoot, geodata from ooni import config from ooni import otime @@ -30,15 +30,15 @@ class NetTestLoader(object): from ooni import __version__ as software_version client_geodata = {} - if config.probe_ip and (config.privacy.includeip or \ + if config.probe_ip.address and (config.privacy.includeip or \ config.privacy.includeasn or \ config.privacy.includecountry or \ config.privacy.includecity): log.msg("We will include some geo data in the report") - client_geodata = geodata.IPToLocation(config.probe_ip) + client_geodata = geodata.IPToLocation(config.probe_ip.address) if config.privacy.includeip: - client_geodata['ip'] = config.probe_ip + client_geodata['ip'] = config.probe_ip.address else: client_geodata['ip'] = "127.0.0.1" diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 8510a3b..340693b 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -48,12 +48,9 @@ class Storage(dict): for (k, v) in value.items(): self[k] = v -class NotRootError(Exception): - pass - def checkForRoot(): if os.getuid() != 0: - raise NotRootError("This test requires root") + raise errors.InsufficientPrivileges def randomSTR(length, num=True): """ diff --git a/tests/test_nettest.py b/tests/test_nettest.py index a0ccb32..976757f 100644 --- a/tests/test_nettest.py +++ b/tests/test_nettest.py @@ -9,7 +9,6 @@ from twisted.python.usage import UsageError from ooni.nettest import NetTest, InvalidOption, MissingRequiredOption from ooni.nettest import NetTestLoader, FailureToLoadNetTest from ooni.tasks import BaseTask -from ooni.utils import NotRootError from ooni.director import Director

1 0

[ooni-probe/develop] Write unittests also for Tor geoip stuff
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit d73c5fa22a72413fd1c5d24eeb1e9118f0a4662a Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 15 16:31:14 2013 +0100 Write unittests also for Tor geoip stuff --- ooni/errors.py | 3 +++ ooni/geoip.py | 16 ++++++++++++---- tests/test_geoip.py | 45 +++++++++++++++++++++++++++++++++++++++------ 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 62fcf93..69af726 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -138,3 +138,6 @@ class TorStateNotFound(Exception): class InsufficientPrivileges(Exception): pass + +class ProbeIPUnknown(Exception): + pass diff --git a/ooni/geoip.py b/ooni/geoip.py index e6a221b..ae06608 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -110,19 +110,22 @@ class ProbeIP(object): 'maxmind': MaxMindGeoIP } address = None + tor_state = config.tor_state @defer.inlineCallbacks def lookup(self): try: yield self.askTor() + log.msg("Found your IP via Tor %s" % self.address) defer.returnValue(self.address) except errors.TorStateNotFound: log.debug("Tor is not running. Skipping IP lookup via Tor.") - except: + except Exception: log.msg("Unable to lookup the probe IP via Tor.") try: yield self.askTraceroute() + log.msg("Found your IP via Traceroute %s" % self.address) defer.returnValue(self.address) except errors.InsufficientPrivileges: log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") @@ -131,9 +134,11 @@ class ProbeIP(object): try: yield self.askGeoIPService() + log.msg("Found your IP via a GeoIP service: %s" % self.address) defer.returnValue(self.address) - except: + except Exception, e: log.msg("Unable to lookup the probe IP via GeoIPService") + raise e @defer.inlineCallbacks def askGeoIPService(self): @@ -150,6 +155,9 @@ class ProbeIP(object): except Exception, e: log.msg("Failed to lookup your IP via %s" % service_name) + if not self.address: + raise errors.ProbeIPUnknown + def askTraceroute(self): """ Perform a UDP traceroute to determine the probes IP address. @@ -165,8 +173,8 @@ class ProbeIP(object): XXX this lookup method is currently broken when there are cached descriptors or consensus documents see: https://trac.torproject.org/projects/tor/ticket/8214 """ - if config.tor_state: - d = config.tor_state.protocol.get_info("address") + if self.tor_state: + d = self.tor_state.protocol.get_info("address") @d.addCallback def cb(result): self.strategy = 'tor_get_info_address' diff --git a/tests/test_geoip.py b/tests/test_geoip.py index 443d262..0d7165f 100644 --- a/tests/test_geoip.py +++ b/tests/test_geoip.py @@ -1,5 +1,7 @@ +from collections import namedtuple + from twisted.web import server, static, resource -from twisted.internet import reactor +from twisted.internet import reactor, defer from twisted.trial import unittest from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory @@ -58,7 +60,7 @@ class TestGeoIPServices(GeoIPBaseTest): gip = TorProjectGeoIP() gip.url = self.getUrl('torproject') d = gip.lookup() - @d.addCallback + @d.addBoth def cb(res): self.assertEqual(res, '127.0.0.1') return d @@ -67,7 +69,7 @@ class TestGeoIPServices(GeoIPBaseTest): gip = UbuntuGeoIP() gip.url = self.getUrl('ubuntu') d = gip.lookup() - @d.addCallback + @d.addBoth def cb(res): self.assertEqual(res, '127.0.0.1') return d @@ -76,7 +78,7 @@ class TestGeoIPServices(GeoIPBaseTest): gip = MaxMindGeoIP() gip.url = self.getUrl('maxmind') d = gip.lookup() - @d.addCallback + @d.addBoth def cb(res): self.assertEqual(res, '127.0.0.1') return d @@ -93,7 +95,7 @@ class TestProbeIP(GeoIPBaseTest): def test_ask_geoip_service(self): d = self.probe_ip.askGeoIPService() - @d.addCallback + @d.addBoth def cb(res): self.assertEqual(self.probe_ip.address, '127.0.0.1') return d @@ -102,7 +104,38 @@ class TestProbeIP(GeoIPBaseTest): self.assertRaises(errors.InsufficientPrivileges, self.probe_ip.askTraceroute) def test_ask_tor(self): - pass + class MockTorState(object): + """ + This is a Mock Tor state object. It will just pretend to answer to + the get_info("address") method call. + """ + protocol = namedtuple('Protocol', 'get_info') + def __init__(self): + def get_info(key): + return defer.succeed({'XXX': '127.0.0.2'}) + self.protocol = self.protocol(get_info=get_info) + + self.probe_ip.tor_state = MockTorState() + d = self.probe_ip.lookup() + @d.addBoth + def cb(res): + self.assertEqual(self.probe_ip.address, '127.0.0.2') + return d + + def test_probe_ip(self): + d = self.probe_ip.lookup() + @d.addBoth + def cb(res): + self.assertEqual(self.probe_ip.address, '127.0.0.1') + self.assertTrue(self.probe_ip.strategy.startswith('geo_ip_service-')) + return d + + def test_failing_probe_ip(self): + self.probe_ip.geoIPServices = {} + + d = self.probe_ip.lookup() + self.assertFailure(d, errors.ProbeIPUnknown) + return d class TestIPToLocation(unittest.TestCase): pass

1 0

[ooni-probe/develop] Make reporter creation more robust
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit e620ea035144550e3ca4162a25237367919045cb Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Mar 13 20:57:44 2013 +0100 Make reporter creation more robust We want to raise errors by yielding, because we are inside of a inlineCallbacks decorated function. --- ooni/reporter.py | 12 +++++++----- ooni/runner.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 594c6b8..20193c9 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -226,6 +226,7 @@ class YAMLReporter(OReporter): self.writeReportEntry(self.testDetails) self.created.callback(self) + return defer.succeed(None) def finish(self): self._stream.close() @@ -305,7 +306,7 @@ class OONIBReporter(OReporter): self.agent = Agent(reactor, sockshost="127.0.0.1", socksport=int(config.tor.socks_port)) except Exception, e: - log.exception(e) + yield defer.fail(e) url = self.collectorAddress + '/report' @@ -338,11 +339,11 @@ class OONIBReporter(OReporter): bodyProducer=bodyProducer) except ConnectionRefusedError: log.err("Connection to reporting backend failed (ConnectionRefusedError)") - raise OONIBReportCreationError + self.created.errback(defer.fail(OONIBReportCreationError)) except Exception, e: log.exception(e) - raise OONIBReportCreationError + yield defer.fail(OONIBReportCreationError) # This is a little trix to allow us to unspool the response. We create # a deferred and call yield on it. @@ -355,7 +356,7 @@ class OONIBReporter(OReporter): parsed_response = json.loads(backend_response) except Exception, e: log.exception(e) - raise OONIBReportCreationError + yield defer.fail(e) self.reportID = parsed_response['report_id'] self.backendVersion = parsed_response['backend_version'] @@ -392,7 +393,8 @@ class Report(object): """ l = [] for reporter in self.reporters[:]: - reporter.createReport() + d = reporter.createReport() + d.addErrback(self.failedOpeningReport, reporter) reporter.created.addErrback(self.failedOpeningReport, reporter) l.append(reporter.created) log.debug("Reporters created: %s" % l) diff --git a/ooni/runner.py b/ooni/runner.py index 080db18..abdc256 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -15,7 +15,7 @@ from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError from ooni.inputunit import InputUnitFactory from ooni.nettest import NetTestCase, NoPostProcessor from ooni.utils import log, checkForRoot, pushFilenameStack -from ooni.utils import NotRootError, Storage +from ooni.utils import Storage from ooni.utils.net import randomFreePort class InvalidResumeFile(Exception):

1 0