commit f544c3ef06808c441ed6e7db3cd4dfd187b04aa3 Author: Arturo Filastò <art@fuffa.org> Date: Wed Mar 13 20:57:19 2013 +0100 Implement modular probe IP address lookup function. --- ooni/director.py | 153 ++++++++++++++++++++++++++++++++++++++++++++---- ooni/errors.py | 5 ++ ooni/nettest.py | 8 +- ooni/utils/__init__.py | 5 +- tests/test_nettest.py | 1 - 5 files changed, 152 insertions(+), 20 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 8365ebd..07510eb 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -1,18 +1,151 @@ +import xml.etree.ElementTree as ET +import random import sys import os +import re from ooni import config from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report -from ooni.utils import log, checkForRoot, NotRootError +from ooni.utils import log, checkForRoot from ooni.utils.net import randomFreePort from ooni.nettest import NetTest -from ooni.errors import UnableToStartTor +from ooni import errors from txtorcon import TorConfig from txtorcon import TorState, launch_tor from twisted.internet import defer, reactor +from twisted.web import client, http_headers +from ooni.utils.net import userAgents, BodyReceiver + +class HTTPGeoIPLookupper(object): + url = None + + def _response(self, response): + content_length = response.headers.getRawHeaders('content-length') + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished, content_length)) + finished.addCallback(self.parseResponse) + return finished + + def parseResponse(self, response_body): + """ + Override this with the logic for parsing the response. + + Should return the IP address of the probe. + """ + pass + + def failed(self, failure): + log.err("Failed to lookup via %s" % url) + log.exception(failure) + return failure + + def lookup(self): + agent = client.Agent(reactor) + headers = {} + headers['User-Agent'] = [random.choice(userAgents)] + + d = agent.request("GET", self.url, http_headers.Headers(headers)) + d.addCallback(self._response) + d.addErrback(self.failed) + return d + +class UbuntuGeoIP(HTTPGeoIPLookupper): + url = "http://geoip.ubuntu.com/lookup" + + def parseResponse(self, response_body): + response = ET.fromstring(response_body) + probe_ip = response.find('Ip').text + return probe_ip + +class TorProjectGeoIP(HTTPGeoIPLookupper): + url = "https://check.torproject.org/" + + def parseResponse(self, response_body): + regexp = "Your IP address appears to be: <b>((\d+\.)+(\d+))" + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class MaxMindGeoIP(HTTPGeoIPLookupper): + url = "https://www.maxmind.com/en/locate_my_ip" + + def parseResponse(self, response_body): + regexp = '<span id="my-ip-address">((\d+\.)+(\d+))</span>' + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class ProbeIP(object): + strategy = None + geoIPServices = {'ubuntu': UbuntuGeoIP, + 'torproject': TorProjectGeoIP, + 'maximind': MaxMindGeoIP + } + address = None + + @defer.inlineCallbacks + def lookup(self): + try: + yield self.askTor() + defer.returnValue(self.address) + except errors.TorStateNotFound: + log.debug("Tor is not running. Skipping IP lookup via Tor.") + except: + log.msg("Unable to lookup the probe IP via Tor.") + + try: + yield self.askTraceroute() + defer.returnValue(self.address) + except errors.InsufficientPrivileges: + log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") + except: + log.msg("Unable to lookup the probe IP via traceroute") + + try: + yield self.askGeoIPService() + defer.returnValue(self.address) + except Exception, e: + print e + log.msg("Unable to lookup the probe IP via GeoIPService") + + @defer.inlineCallbacks + def askGeoIPService(self): + for service_name, service in self.geoIPServices.items(): + s = TorProjectGeoIP() + log.msg("Looking up your IP address via %s" % service_name) + try: + self.address = yield s.lookup() + self.strategy = 'geo_ip_service-' + service_name + break + except: + log.msg("Failed to lookup your IP via %s" % service_name) + + def askTraceroute(self): + """ + Perform a UDP traceroute to determine the probes IP address. + """ + checkForRoot() + raise NotImplemented + + def askTor(self): + """ + Obtain the probes IP address by asking the Tor Control port via GET INFO + address. + + XXX this lookup method is currently broken when there are cached descriptors or consensus documents + see: https://trac.torproject.org/projects/tor/ticket/8214 + """ + if config.tor_state: + d = config.tor_state.protocol.get_info("address") + @d.addCallback + def cb(result): + self.strategy = 'tor_get_info_address' + self.address = result.values()[0] + return d + else: + raise errors.TorStateNotFound class Director(object): """ @@ -80,6 +213,7 @@ class Director(object): self.torControlProtocol = None + @defer.inlineCallbacks def start(self): if config.privacy.includepcap: log.msg("Starting") @@ -89,10 +223,10 @@ class Director(object): if config.advanced.start_tor: log.msg("Starting Tor...") - d = self.startTor() - else: - d = defer.succeed(None) - return d + yield self.startTor() + + config.probe_ip = ProbeIP() + yield config.probe_ip.lookup() @property def measurementSuccessRatio(self): @@ -200,7 +334,7 @@ class Director(object): from ooni.utils.txscapy import ScapyFactory, ScapySniffer try: checkForRoot() - except NotRootError: + except errors.InsufficientPrivileges: print "[!] Includepcap options requires root priviledges to run" print " you should run ooniprobe as root or disable the options in ooniprobe.conf" sys.exit(1) @@ -232,18 +366,15 @@ class Director(object): socks_port = yield state.protocol.get_conf("SocksPort") control_port = yield state.protocol.get_conf("ControlPort") - client_ip = yield state.protocol.get_info("address") config.tor.socks_port = int(socks_port.values()[0]) config.tor.control_port = int(control_port.values()[0]) - config.probe_ip = client_ip.values()[0] - log.debug("Obtained our IP address from a Tor Relay %s" % config.probe_ip) def setup_failed(failure): log.exception(failure) - raise UnableToStartTor + raise errors.UnableToStartTor def setup_complete(proto): """ diff --git a/ooni/errors.py b/ooni/errors.py index 36a042f..6b23727 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -132,3 +132,8 @@ class ReportNotCreated(Exception): class ReportAlreadyClosed(Exception): pass +class TorStateNotFound(Exception): + pass + +class InsufficientPrivileges(Exception): + pass diff --git a/ooni/nettest.py b/ooni/nettest.py index 6273d34..67dee9d 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -6,7 +6,7 @@ from twisted.trial.runner import filenameToModule from twisted.python import usage, reflect from ooni.tasks import Measurement -from ooni.utils import log, checkForRoot, NotRootError, geodata +from ooni.utils import log, checkForRoot, geodata from ooni import config from ooni import otime @@ -30,15 +30,15 @@ class NetTestLoader(object): from ooni import __version__ as software_version client_geodata = {} - if config.probe_ip and (config.privacy.includeip or \ + if config.probe_ip.address and (config.privacy.includeip or \ config.privacy.includeasn or \ config.privacy.includecountry or \ config.privacy.includecity): log.msg("We will include some geo data in the report") - client_geodata = geodata.IPToLocation(config.probe_ip) + client_geodata = geodata.IPToLocation(config.probe_ip.address) if config.privacy.includeip: - client_geodata['ip'] = config.probe_ip + client_geodata['ip'] = config.probe_ip.address else: client_geodata['ip'] = "127.0.0.1" diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 8510a3b..340693b 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -48,12 +48,9 @@ class Storage(dict): for (k, v) in value.items(): self[k] = v -class NotRootError(Exception): - pass - def checkForRoot(): if os.getuid() != 0: - raise NotRootError("This test requires root") + raise errors.InsufficientPrivileges def randomSTR(length, num=True): """ diff --git a/tests/test_nettest.py b/tests/test_nettest.py index a0ccb32..976757f 100644 --- a/tests/test_nettest.py +++ b/tests/test_nettest.py @@ -9,7 +9,6 @@ from twisted.python.usage import UsageError from ooni.nettest import NetTest, InvalidOption, MissingRequiredOption from ooni.nettest import NetTestLoader, FailureToLoadNetTest from ooni.tasks import BaseTask -from ooni.utils import NotRootError from ooni.director import Director