[tor-commits] [ooni-probe/master] Implement modular probe IP address lookup function.
art at torproject.org
art at torproject.org
Sat Mar 30 01:17:21 UTC 2013
commit f544c3ef06808c441ed6e7db3cd4dfd187b04aa3
Author: Arturo Filastò <art at fuffa.org>
Date: Wed Mar 13 20:57:19 2013 +0100
Implement modular probe IP address lookup function.
---
ooni/director.py | 153 ++++++++++++++++++++++++++++++++++++++++++++----
ooni/errors.py | 5 ++
ooni/nettest.py | 8 +-
ooni/utils/__init__.py | 5 +-
tests/test_nettest.py | 1 -
5 files changed, 152 insertions(+), 20 deletions(-)
diff --git a/ooni/director.py b/ooni/director.py
index 8365ebd..07510eb 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -1,18 +1,151 @@
+import xml.etree.ElementTree as ET
+import random
import sys
import os
+import re
from ooni import config
from ooni.managers import ReportEntryManager, MeasurementManager
from ooni.reporter import Report
-from ooni.utils import log, checkForRoot, NotRootError
+from ooni.utils import log, checkForRoot
from ooni.utils.net import randomFreePort
from ooni.nettest import NetTest
-from ooni.errors import UnableToStartTor
+from ooni import errors
from txtorcon import TorConfig
from txtorcon import TorState, launch_tor
from twisted.internet import defer, reactor
+from twisted.web import client, http_headers
+from ooni.utils.net import userAgents, BodyReceiver
+
+class HTTPGeoIPLookupper(object):
+ url = None
+
+ def _response(self, response):
+ content_length = response.headers.getRawHeaders('content-length')
+
+ finished = defer.Deferred()
+ response.deliverBody(BodyReceiver(finished, content_length))
+ finished.addCallback(self.parseResponse)
+ return finished
+
+ def parseResponse(self, response_body):
+ """
+ Override this with the logic for parsing the response.
+
+ Should return the IP address of the probe.
+ """
+ pass
+
+ def failed(self, failure):
+ log.err("Failed to lookup via %s" % url)
+ log.exception(failure)
+ return failure
+
+ def lookup(self):
+ agent = client.Agent(reactor)
+ headers = {}
+ headers['User-Agent'] = [random.choice(userAgents)]
+
+ d = agent.request("GET", self.url, http_headers.Headers(headers))
+ d.addCallback(self._response)
+ d.addErrback(self.failed)
+ return d
+
+class UbuntuGeoIP(HTTPGeoIPLookupper):
+ url = "http://geoip.ubuntu.com/lookup"
+
+ def parseResponse(self, response_body):
+ response = ET.fromstring(response_body)
+ probe_ip = response.find('Ip').text
+ return probe_ip
+
+class TorProjectGeoIP(HTTPGeoIPLookupper):
+ url = "https://check.torproject.org/"
+
+ def parseResponse(self, response_body):
+ regexp = "Your IP address appears to be: <b>((\d+\.)+(\d+))"
+ probe_ip = re.search(regexp, response_body).group(1)
+ return probe_ip
+
+class MaxMindGeoIP(HTTPGeoIPLookupper):
+ url = "https://www.maxmind.com/en/locate_my_ip"
+
+ def parseResponse(self, response_body):
+ regexp = '<span id="my-ip-address">((\d+\.)+(\d+))</span>'
+ probe_ip = re.search(regexp, response_body).group(1)
+ return probe_ip
+
+class ProbeIP(object):
+ strategy = None
+ geoIPServices = {'ubuntu': UbuntuGeoIP,
+ 'torproject': TorProjectGeoIP,
+ 'maximind': MaxMindGeoIP
+ }
+ address = None
+
+ @defer.inlineCallbacks
+ def lookup(self):
+ try:
+ yield self.askTor()
+ defer.returnValue(self.address)
+ except errors.TorStateNotFound:
+ log.debug("Tor is not running. Skipping IP lookup via Tor.")
+ except:
+ log.msg("Unable to lookup the probe IP via Tor.")
+
+ try:
+ yield self.askTraceroute()
+ defer.returnValue(self.address)
+ except errors.InsufficientPrivileges:
+ log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
+ except:
+ log.msg("Unable to lookup the probe IP via traceroute")
+
+ try:
+ yield self.askGeoIPService()
+ defer.returnValue(self.address)
+ except Exception, e:
+ print e
+ log.msg("Unable to lookup the probe IP via GeoIPService")
+
+ @defer.inlineCallbacks
+ def askGeoIPService(self):
+ for service_name, service in self.geoIPServices.items():
+ s = TorProjectGeoIP()
+ log.msg("Looking up your IP address via %s" % service_name)
+ try:
+ self.address = yield s.lookup()
+ self.strategy = 'geo_ip_service-' + service_name
+ break
+ except:
+ log.msg("Failed to lookup your IP via %s" % service_name)
+
+ def askTraceroute(self):
+ """
+ Perform a UDP traceroute to determine the probes IP address.
+ """
+ checkForRoot()
+ raise NotImplemented
+
+ def askTor(self):
+ """
+ Obtain the probes IP address by asking the Tor Control port via GET INFO
+ address.
+
+ XXX this lookup method is currently broken when there are cached descriptors or consensus documents
+ see: https://trac.torproject.org/projects/tor/ticket/8214
+ """
+ if config.tor_state:
+ d = config.tor_state.protocol.get_info("address")
+ @d.addCallback
+ def cb(result):
+ self.strategy = 'tor_get_info_address'
+ self.address = result.values()[0]
+ return d
+ else:
+ raise errors.TorStateNotFound
class Director(object):
"""
@@ -80,6 +213,7 @@ class Director(object):
self.torControlProtocol = None
+ @defer.inlineCallbacks
def start(self):
if config.privacy.includepcap:
log.msg("Starting")
@@ -89,10 +223,10 @@ class Director(object):
if config.advanced.start_tor:
log.msg("Starting Tor...")
- d = self.startTor()
- else:
- d = defer.succeed(None)
- return d
+ yield self.startTor()
+
+ config.probe_ip = ProbeIP()
+ yield config.probe_ip.lookup()
@property
def measurementSuccessRatio(self):
@@ -200,7 +334,7 @@ class Director(object):
from ooni.utils.txscapy import ScapyFactory, ScapySniffer
try:
checkForRoot()
- except NotRootError:
+ except errors.InsufficientPrivileges:
print "[!] Includepcap options requires root priviledges to run"
print " you should run ooniprobe as root or disable the options in ooniprobe.conf"
sys.exit(1)
@@ -232,18 +366,15 @@ class Director(object):
socks_port = yield state.protocol.get_conf("SocksPort")
control_port = yield state.protocol.get_conf("ControlPort")
- client_ip = yield state.protocol.get_info("address")
config.tor.socks_port = int(socks_port.values()[0])
config.tor.control_port = int(control_port.values()[0])
- config.probe_ip = client_ip.values()[0]
-
log.debug("Obtained our IP address from a Tor Relay %s" % config.probe_ip)
def setup_failed(failure):
log.exception(failure)
- raise UnableToStartTor
+ raise errors.UnableToStartTor
def setup_complete(proto):
"""
diff --git a/ooni/errors.py b/ooni/errors.py
index 36a042f..6b23727 100644
--- a/ooni/errors.py
+++ b/ooni/errors.py
@@ -132,3 +132,8 @@ class ReportNotCreated(Exception):
class ReportAlreadyClosed(Exception):
pass
+class TorStateNotFound(Exception):
+ pass
+
+class InsufficientPrivileges(Exception):
+ pass
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 6273d34..67dee9d 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -6,7 +6,7 @@ from twisted.trial.runner import filenameToModule
from twisted.python import usage, reflect
from ooni.tasks import Measurement
-from ooni.utils import log, checkForRoot, NotRootError, geodata
+from ooni.utils import log, checkForRoot, geodata
from ooni import config
from ooni import otime
@@ -30,15 +30,15 @@ class NetTestLoader(object):
from ooni import __version__ as software_version
client_geodata = {}
- if config.probe_ip and (config.privacy.includeip or \
+ if config.probe_ip.address and (config.privacy.includeip or \
config.privacy.includeasn or \
config.privacy.includecountry or \
config.privacy.includecity):
log.msg("We will include some geo data in the report")
- client_geodata = geodata.IPToLocation(config.probe_ip)
+ client_geodata = geodata.IPToLocation(config.probe_ip.address)
if config.privacy.includeip:
- client_geodata['ip'] = config.probe_ip
+ client_geodata['ip'] = config.probe_ip.address
else:
client_geodata['ip'] = "127.0.0.1"
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 8510a3b..340693b 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -48,12 +48,9 @@ class Storage(dict):
for (k, v) in value.items():
self[k] = v
-class NotRootError(Exception):
- pass
-
def checkForRoot():
if os.getuid() != 0:
- raise NotRootError("This test requires root")
+ raise errors.InsufficientPrivileges
def randomSTR(length, num=True):
"""
diff --git a/tests/test_nettest.py b/tests/test_nettest.py
index a0ccb32..976757f 100644
--- a/tests/test_nettest.py
+++ b/tests/test_nettest.py
@@ -9,7 +9,6 @@ from twisted.python.usage import UsageError
from ooni.nettest import NetTest, InvalidOption, MissingRequiredOption
from ooni.nettest import NetTestLoader, FailureToLoadNetTest
from ooni.tasks import BaseTask
-from ooni.utils import NotRootError
from ooni.director import Director
More information about the tor-commits
mailing list