[tor-commits] [ooni-probe/develop] Implement modular probe IP address lookup function.

isis at torproject.org isis at torproject.org
Thu Jun 6 16:41:37 UTC 2013


commit f544c3ef06808c441ed6e7db3cd4dfd187b04aa3
Author: Arturo Filastò <art at fuffa.org>
Date:   Wed Mar 13 20:57:19 2013 +0100

    Implement modular probe IP address lookup function.
---
 ooni/director.py       |  153 ++++++++++++++++++++++++++++++++++++++++++++----
 ooni/errors.py         |    5 ++
 ooni/nettest.py        |    8 +--
 ooni/utils/__init__.py |    5 +-
 tests/test_nettest.py  |    1 -
 5 files changed, 152 insertions(+), 20 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py
index 8365ebd..07510eb 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -1,18 +1,151 @@
+import xml.etree.ElementTree as ET
+import random
 import sys
 import os
+import re
 
 from ooni import config
 from ooni.managers import ReportEntryManager, MeasurementManager
 from ooni.reporter import Report
-from ooni.utils import log, checkForRoot, NotRootError
+from ooni.utils import log, checkForRoot
 from ooni.utils.net import randomFreePort
 from ooni.nettest import NetTest
-from ooni.errors import UnableToStartTor
+from ooni import errors
 
 from txtorcon import TorConfig
 from txtorcon import TorState, launch_tor
 
 from twisted.internet import defer, reactor
+from twisted.web import client, http_headers
+from ooni.utils.net import userAgents, BodyReceiver
+
+class HTTPGeoIPLookupper(object):
+    url = None
+
+    def _response(self, response):
+        content_length = response.headers.getRawHeaders('content-length')
+
+        finished = defer.Deferred()
+        response.deliverBody(BodyReceiver(finished, content_length))
+        finished.addCallback(self.parseResponse)
+        return finished
+
+    def parseResponse(self, response_body):
+        """
+        Override this with the logic for parsing the response.
+
+        Should return the IP address of the probe.
+        """
+        pass
+
+    def failed(self, failure):
+        log.err("Failed to lookup via %s" % url)
+        log.exception(failure)
+        return failure
+
+    def lookup(self):
+        agent = client.Agent(reactor)
+        headers = {}
+        headers['User-Agent'] = [random.choice(userAgents)]
+
+        d = agent.request("GET", self.url, http_headers.Headers(headers))
+        d.addCallback(self._response)
+        d.addErrback(self.failed)
+        return d
+
+class UbuntuGeoIP(HTTPGeoIPLookupper):
+    url = "http://geoip.ubuntu.com/lookup"
+
+    def parseResponse(self, response_body):
+        response = ET.fromstring(response_body)
+        probe_ip = response.find('Ip').text
+        return probe_ip
+
+class TorProjectGeoIP(HTTPGeoIPLookupper):
+    url = "https://check.torproject.org/"
+
+    def parseResponse(self, response_body):
+        regexp = "Your IP address appears to be: <b>((\d+\.)+(\d+))"
+        probe_ip = re.search(regexp, response_body).group(1)
+        return probe_ip
+
+class MaxMindGeoIP(HTTPGeoIPLookupper):
+    url = "https://www.maxmind.com/en/locate_my_ip"
+
+    def parseResponse(self, response_body):
+        regexp = '<span id="my-ip-address">((\d+\.)+(\d+))</span>'
+        probe_ip = re.search(regexp, response_body).group(1)
+        return probe_ip
+
+class ProbeIP(object):
+    strategy = None
+    geoIPServices = {'ubuntu': UbuntuGeoIP,
+        'torproject': TorProjectGeoIP,
+        'maximind': MaxMindGeoIP
+    }
+    address = None
+
+    @defer.inlineCallbacks
+    def lookup(self):
+        try:
+            yield self.askTor()
+            defer.returnValue(self.address)
+        except errors.TorStateNotFound:
+            log.debug("Tor is not running. Skipping IP lookup via Tor.")
+        except:
+            log.msg("Unable to lookup the probe IP via Tor.")
+
+        try:
+            yield self.askTraceroute()
+            defer.returnValue(self.address)
+        except errors.InsufficientPrivileges:
+            log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
+        except:
+            log.msg("Unable to lookup the probe IP via traceroute")
+
+        try:
+            yield self.askGeoIPService()
+            defer.returnValue(self.address)
+        except Exception, e:
+            print e
+            log.msg("Unable to lookup the probe IP via GeoIPService")
+
+    @defer.inlineCallbacks
+    def askGeoIPService(self):
+        for service_name, service in self.geoIPServices.items():
+            s = TorProjectGeoIP()
+            log.msg("Looking up your IP address via %s" % service_name)
+            try:
+                self.address = yield s.lookup()
+                self.strategy = 'geo_ip_service-' + service_name
+                break
+            except:
+                log.msg("Failed to lookup your IP via %s" % service_name)
+
+    def askTraceroute(self):
+        """
+        Perform a UDP traceroute to determine the probes IP address.
+        """
+        checkForRoot()
+        raise NotImplemented
+
+    def askTor(self):
+        """
+        Obtain the probes IP address by asking the Tor Control port via GET INFO
+        address.
+
+        XXX this lookup method is currently broken when there are cached descriptors or consensus documents
+        see: https://trac.torproject.org/projects/tor/ticket/8214
+        """
+        if config.tor_state:
+            d = config.tor_state.protocol.get_info("address")
+            @d.addCallback
+            def cb(result):
+                self.strategy = 'tor_get_info_address'
+                self.address = result.values()[0]
+            return d
+        else:
+            raise errors.TorStateNotFound
 
 class Director(object):
     """
@@ -80,6 +213,7 @@ class Director(object):
 
         self.torControlProtocol = None
 
+    @defer.inlineCallbacks
     def start(self):
         if config.privacy.includepcap:
             log.msg("Starting")
@@ -89,10 +223,10 @@ class Director(object):
 
         if config.advanced.start_tor:
             log.msg("Starting Tor...")
-            d = self.startTor()
-        else:
-            d = defer.succeed(None)
-        return d
+            yield self.startTor()
+
+        config.probe_ip = ProbeIP()
+        yield config.probe_ip.lookup()
 
     @property
     def measurementSuccessRatio(self):
@@ -200,7 +334,7 @@ class Director(object):
         from ooni.utils.txscapy import ScapyFactory, ScapySniffer
         try:
             checkForRoot()
-        except NotRootError:
+        except errors.InsufficientPrivileges:
             print "[!] Includepcap options requires root priviledges to run"
             print "    you should run ooniprobe as root or disable the options in ooniprobe.conf"
             sys.exit(1)
@@ -232,18 +366,15 @@ class Director(object):
 
             socks_port = yield state.protocol.get_conf("SocksPort")
             control_port = yield state.protocol.get_conf("ControlPort")
-            client_ip = yield state.protocol.get_info("address")
 
             config.tor.socks_port = int(socks_port.values()[0])
             config.tor.control_port = int(control_port.values()[0])
 
-            config.probe_ip = client_ip.values()[0]
-
             log.debug("Obtained our IP address from a Tor Relay %s" % config.probe_ip)
 
         def setup_failed(failure):
             log.exception(failure)
-            raise UnableToStartTor
+            raise errors.UnableToStartTor
 
         def setup_complete(proto):
             """
diff --git a/ooni/errors.py b/ooni/errors.py
index 36a042f..6b23727 100644
--- a/ooni/errors.py
+++ b/ooni/errors.py
@@ -132,3 +132,8 @@ class ReportNotCreated(Exception):
 class ReportAlreadyClosed(Exception):
     pass
 
+class TorStateNotFound(Exception):
+    pass
+
+class InsufficientPrivileges(Exception):
+    pass
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 6273d34..67dee9d 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -6,7 +6,7 @@ from twisted.trial.runner import filenameToModule
 from twisted.python import usage, reflect
 
 from ooni.tasks import Measurement
-from ooni.utils import log, checkForRoot, NotRootError, geodata
+from ooni.utils import log, checkForRoot, geodata
 from ooni import config
 from ooni import otime
 
@@ -30,15 +30,15 @@ class NetTestLoader(object):
         from ooni import __version__ as software_version
 
         client_geodata = {}
-        if config.probe_ip and (config.privacy.includeip or \
+        if config.probe_ip.address and (config.privacy.includeip or \
                 config.privacy.includeasn or \
                 config.privacy.includecountry or \
                 config.privacy.includecity):
             log.msg("We will include some geo data in the report")
-            client_geodata = geodata.IPToLocation(config.probe_ip)
+            client_geodata = geodata.IPToLocation(config.probe_ip.address)
 
         if config.privacy.includeip:
-            client_geodata['ip'] = config.probe_ip
+            client_geodata['ip'] = config.probe_ip.address
         else:
             client_geodata['ip'] = "127.0.0.1"
 
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 8510a3b..340693b 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -48,12 +48,9 @@ class Storage(dict):
         for (k, v) in value.items():
             self[k] = v
 
-class NotRootError(Exception):
-    pass
-
 def checkForRoot():
     if os.getuid() != 0:
-        raise NotRootError("This test requires root")
+        raise errors.InsufficientPrivileges
 
 def randomSTR(length, num=True):
     """
diff --git a/tests/test_nettest.py b/tests/test_nettest.py
index a0ccb32..976757f 100644
--- a/tests/test_nettest.py
+++ b/tests/test_nettest.py
@@ -9,7 +9,6 @@ from twisted.python.usage import UsageError
 from ooni.nettest import NetTest, InvalidOption, MissingRequiredOption
 from ooni.nettest import NetTestLoader, FailureToLoadNetTest
 from ooni.tasks import BaseTask
-from ooni.utils import NotRootError
 
 from ooni.director import Director
 





More information about the tor-commits mailing list