[tor-commits] [ooni-probe/master] Move geoip related classes into ooni.geoip

art at torproject.org art at torproject.org
Sat Mar 30 01:17:21 UTC 2013


commit 51fe88e01de3d559522dc2cf3ea518f65993635b
Author: Arturo Filastò <art at fuffa.org>
Date:   Fri Mar 15 09:22:13 2013 +0100

    Move geoip related classes into ooni.geoip
---
 HACKING               |    2 +-
 ooni/director.py      |  133 +--------------------------------------
 ooni/geoip.py         |  168 +++++++++++++++++++++++++++++++++++++++++++++++++
 ooni/nettest.py       |    5 +-
 ooni/reporter.py      |    2 +-
 ooni/utils/geodata.py |   40 ------------
 tests/test_nettest.py |    2 +-
 7 files changed, 176 insertions(+), 176 deletions(-)

diff --git a/HACKING b/HACKING
index b92ddbe..aad7ebb 100644
--- a/HACKING
+++ b/HACKING
@@ -87,7 +87,7 @@ Code Structure
 - ooni/utils/
   In here go internal utilities that are useful to ooniprobe
 
-- ooni/utils/geodata.py
+- ooni/geoip.py
   In here go functions related to the understanding of
   geographical information of the probe
 
diff --git a/ooni/director.py b/ooni/director.py
index 07510eb..d2f5a7c 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -5,6 +5,7 @@ import os
 import re
 
 from ooni import config
+from ooni import geoip
 from ooni.managers import ReportEntryManager, MeasurementManager
 from ooni.reporter import Report
 from ooni.utils import log, checkForRoot
@@ -16,136 +17,6 @@ from txtorcon import TorConfig
 from txtorcon import TorState, launch_tor
 
 from twisted.internet import defer, reactor
-from twisted.web import client, http_headers
-from ooni.utils.net import userAgents, BodyReceiver
-
-class HTTPGeoIPLookupper(object):
-    url = None
-
-    def _response(self, response):
-        content_length = response.headers.getRawHeaders('content-length')
-
-        finished = defer.Deferred()
-        response.deliverBody(BodyReceiver(finished, content_length))
-        finished.addCallback(self.parseResponse)
-        return finished
-
-    def parseResponse(self, response_body):
-        """
-        Override this with the logic for parsing the response.
-
-        Should return the IP address of the probe.
-        """
-        pass
-
-    def failed(self, failure):
-        log.err("Failed to lookup via %s" % url)
-        log.exception(failure)
-        return failure
-
-    def lookup(self):
-        agent = client.Agent(reactor)
-        headers = {}
-        headers['User-Agent'] = [random.choice(userAgents)]
-
-        d = agent.request("GET", self.url, http_headers.Headers(headers))
-        d.addCallback(self._response)
-        d.addErrback(self.failed)
-        return d
-
-class UbuntuGeoIP(HTTPGeoIPLookupper):
-    url = "http://geoip.ubuntu.com/lookup"
-
-    def parseResponse(self, response_body):
-        response = ET.fromstring(response_body)
-        probe_ip = response.find('Ip').text
-        return probe_ip
-
-class TorProjectGeoIP(HTTPGeoIPLookupper):
-    url = "https://check.torproject.org/"
-
-    def parseResponse(self, response_body):
-        regexp = "Your IP address appears to be: <b>((\d+\.)+(\d+))"
-        probe_ip = re.search(regexp, response_body).group(1)
-        return probe_ip
-
-class MaxMindGeoIP(HTTPGeoIPLookupper):
-    url = "https://www.maxmind.com/en/locate_my_ip"
-
-    def parseResponse(self, response_body):
-        regexp = '<span id="my-ip-address">((\d+\.)+(\d+))</span>'
-        probe_ip = re.search(regexp, response_body).group(1)
-        return probe_ip
-
-class ProbeIP(object):
-    strategy = None
-    geoIPServices = {'ubuntu': UbuntuGeoIP,
-        'torproject': TorProjectGeoIP,
-        'maximind': MaxMindGeoIP
-    }
-    address = None
-
-    @defer.inlineCallbacks
-    def lookup(self):
-        try:
-            yield self.askTor()
-            defer.returnValue(self.address)
-        except errors.TorStateNotFound:
-            log.debug("Tor is not running. Skipping IP lookup via Tor.")
-        except:
-            log.msg("Unable to lookup the probe IP via Tor.")
-
-        try:
-            yield self.askTraceroute()
-            defer.returnValue(self.address)
-        except errors.InsufficientPrivileges:
-            log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
-        except:
-            log.msg("Unable to lookup the probe IP via traceroute")
-
-        try:
-            yield self.askGeoIPService()
-            defer.returnValue(self.address)
-        except Exception, e:
-            print e
-            log.msg("Unable to lookup the probe IP via GeoIPService")
-
-    @defer.inlineCallbacks
-    def askGeoIPService(self):
-        for service_name, service in self.geoIPServices.items():
-            s = TorProjectGeoIP()
-            log.msg("Looking up your IP address via %s" % service_name)
-            try:
-                self.address = yield s.lookup()
-                self.strategy = 'geo_ip_service-' + service_name
-                break
-            except:
-                log.msg("Failed to lookup your IP via %s" % service_name)
-
-    def askTraceroute(self):
-        """
-        Perform a UDP traceroute to determine the probes IP address.
-        """
-        checkForRoot()
-        raise NotImplemented
-
-    def askTor(self):
-        """
-        Obtain the probes IP address by asking the Tor Control port via GET INFO
-        address.
-
-        XXX this lookup method is currently broken when there are cached descriptors or consensus documents
-        see: https://trac.torproject.org/projects/tor/ticket/8214
-        """
-        if config.tor_state:
-            d = config.tor_state.protocol.get_info("address")
-            @d.addCallback
-            def cb(result):
-                self.strategy = 'tor_get_info_address'
-                self.address = result.values()[0]
-            return d
-        else:
-            raise errors.TorStateNotFound
 
 class Director(object):
     """
@@ -225,7 +96,7 @@ class Director(object):
             log.msg("Starting Tor...")
             yield self.startTor()
 
-        config.probe_ip = ProbeIP()
+        config.probe_ip = geoip.ProbeIP()
         yield config.probe_ip.lookup()
 
     @property
diff --git a/ooni/geoip.py b/ooni/geoip.py
new file mode 100644
index 0000000..0fe8a6b
--- /dev/null
+++ b/ooni/geoip.py
@@ -0,0 +1,168 @@
+import re
+import os
+
+from twisted.web import client, http_headers
+from ooni.utils.net import userAgents, BodyReceiver
+from twisted.internet import reactor, defer, protocol
+
+from ooni.utils import log, net
+from ooni import config
+
+try:
+    import pygeoip
+except ImportError:
+    log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements")
+
+class GeoIPDataFilesNotFound(Exception):
+    pass
+
+def IPToLocation(ipaddr):
+    city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
+    country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
+    asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
+
+    location = {'city': None, 'countrycode': None, 'asn': None}
+    try:
+        city_dat = pygeoip.GeoIP(city_file)
+        location['city'] = city_dat.record_by_addr(ipaddr)['city']
+
+        country_dat = pygeoip.GeoIP(country_file)
+        location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
+
+        asn_dat = pygeoip.GeoIP(asn_file)
+        location['asn'] = asn_dat.org_by_addr(ipaddr)
+
+    except IOError:
+        log.err("Could not find GeoIP data files. Go into data/ "
+                "and run make geoip")
+        raise GeoIPDataFilesNotFound
+
+    return location
+
+class HTTPGeoIPLookupper(object):
+    url = None
+
+    def _response(self, response):
+        content_length = response.headers.getRawHeaders('content-length')
+
+        finished = defer.Deferred()
+        response.deliverBody(BodyReceiver(finished, content_length))
+        finished.addCallback(self.parseResponse)
+        return finished
+
+    def parseResponse(self, response_body):
+        """
+        Override this with the logic for parsing the response.
+
+        Should return the IP address of the probe.
+        """
+        pass
+
+    def failed(self, failure):
+        log.err("Failed to lookup via %s" % url)
+        log.exception(failure)
+        return failure
+
+    def lookup(self):
+        agent = client.Agent(reactor)
+        headers = {}
+        headers['User-Agent'] = [random.choice(userAgents)]
+
+        d = agent.request("GET", self.url, http_headers.Headers(headers))
+        d.addCallback(self._response)
+        d.addErrback(self.failed)
+        return d
+
+class UbuntuGeoIP(HTTPGeoIPLookupper):
+    url = "http://geoip.ubuntu.com/lookup"
+
+    def parseResponse(self, response_body):
+        response = ET.fromstring(response_body)
+        probe_ip = response.find('Ip').text
+        return probe_ip
+
+class TorProjectGeoIP(HTTPGeoIPLookupper):
+    url = "https://check.torproject.org/"
+
+    def parseResponse(self, response_body):
+        regexp = "Your IP address appears to be: <b>((\d+\.)+(\d+))"
+        probe_ip = re.search(regexp, response_body).group(1)
+        return probe_ip
+
+class MaxMindGeoIP(HTTPGeoIPLookupper):
+    url = "https://www.maxmind.com/en/locate_my_ip"
+
+    def parseResponse(self, response_body):
+        regexp = '<span id="my-ip-address">((\d+\.)+(\d+))</span>'
+        probe_ip = re.search(regexp, response_body).group(1)
+        return probe_ip
+
+class ProbeIP(object):
+    strategy = None
+    geoIPServices = {'ubuntu': UbuntuGeoIP,
+        'torproject': TorProjectGeoIP,
+        'maximind': MaxMindGeoIP
+    }
+    address = None
+
+    @defer.inlineCallbacks
+    def lookup(self):
+        try:
+            yield self.askTor()
+            defer.returnValue(self.address)
+        except errors.TorStateNotFound:
+            log.debug("Tor is not running. Skipping IP lookup via Tor.")
+        except:
+            log.msg("Unable to lookup the probe IP via Tor.")
+
+        try:
+            yield self.askTraceroute()
+            defer.returnValue(self.address)
+        except errors.InsufficientPrivileges:
+            log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
+        except:
+            log.msg("Unable to lookup the probe IP via traceroute")
+
+        try:
+            yield self.askGeoIPService()
+            defer.returnValue(self.address)
+        except Exception, e:
+            print e
+            log.msg("Unable to lookup the probe IP via GeoIPService")
+
+    @defer.inlineCallbacks
+    def askGeoIPService(self):
+        for service_name, service in self.geoIPServices.items():
+            s = TorProjectGeoIP()
+            log.msg("Looking up your IP address via %s" % service_name)
+            try:
+                self.address = yield s.lookup()
+                self.strategy = 'geo_ip_service-' + service_name
+                break
+            except:
+                log.msg("Failed to lookup your IP via %s" % service_name)
+
+    def askTraceroute(self):
+        """
+        Perform a UDP traceroute to determine the probes IP address.
+        """
+        checkForRoot()
+        raise NotImplemented
+
+    def askTor(self):
+        """
+        Obtain the probes IP address by asking the Tor Control port via GET INFO
+        address.
+
+        XXX this lookup method is currently broken when there are cached descriptors or consensus documents
+        see: https://trac.torproject.org/projects/tor/ticket/8214
+        """
+        if config.tor_state:
+            d = config.tor_state.protocol.get_info("address")
+            @d.addCallback
+            def cb(result):
+                self.strategy = 'tor_get_info_address'
+                self.address = result.values()[0]
+            return d
+        else:
+            raise errors.TorStateNotFound
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 67dee9d..bd6bd41 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -5,8 +5,9 @@ from twisted.internet import defer, reactor
 from twisted.trial.runner import filenameToModule
 from twisted.python import usage, reflect
 
+from ooni import geoip
 from ooni.tasks import Measurement
-from ooni.utils import log, checkForRoot, geodata
+from ooni.utils import log, checkForRoot
 from ooni import config
 from ooni import otime
 
@@ -35,7 +36,7 @@ class NetTestLoader(object):
                 config.privacy.includecountry or \
                 config.privacy.includecity):
             log.msg("We will include some geo data in the report")
-            client_geodata = geodata.IPToLocation(config.probe_ip.address)
+            client_geodata = geoip.IPToLocation(config.probe_ip.address)
 
         if config.privacy.includeip:
             client_geodata['ip'] = config.probe_ip.address
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 20193c9..fc73b67 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -30,7 +30,7 @@ from ooni.errors import InvalidOONIBCollectorAddress
 from ooni.errors import ReportNotCreated, ReportAlreadyClosed
 
 from ooni import otime
-from ooni.utils import geodata, pushFilenameStack
+from ooni.utils import pushFilenameStack
 from ooni.utils.net import BodyReceiver, StringProducer, userAgents
 
 from ooni import config
diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py
deleted file mode 100644
index d9883ba..0000000
--- a/ooni/utils/geodata.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import re
-import os
-
-from twisted.web.client import Agent
-from twisted.internet import reactor, defer, protocol
-
-from ooni.utils import log, net
-from ooni import config
-
-try:
-    import pygeoip
-except ImportError:
-    log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements")
-
-class GeoIPDataFilesNotFound(Exception):
-    pass
-
-def IPToLocation(ipaddr):
-    city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
-    country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
-    asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
-
-    location = {'city': None, 'countrycode': None, 'asn': None}
-    try:
-        city_dat = pygeoip.GeoIP(city_file)
-        location['city'] = city_dat.record_by_addr(ipaddr)['city']
-
-        country_dat = pygeoip.GeoIP(country_file)
-        location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
-
-        asn_dat = pygeoip.GeoIP(asn_file)
-        location['asn'] = asn_dat.org_by_addr(ipaddr)
-
-    except IOError:
-        log.err("Could not find GeoIP data files. Go into data/ "
-                "and run make geoip")
-        raise GeoIPDataFilesNotFound
-
-    return location
-
diff --git a/tests/test_nettest.py b/tests/test_nettest.py
index 976757f..2d070c3 100644
--- a/tests/test_nettest.py
+++ b/tests/test_nettest.py
@@ -241,7 +241,7 @@ class TestNetTest(unittest.TestCase):
     #    try:
     #        net_test = NetTestLoader(StringIO(net_test_root_required),
     #                dummyArgs)
-    #    except NotRootError:
+    #    except errors.InsufficientPrivileges:
     #        pass
 
     #def test_create_report_succeed(self):





More information about the tor-commits mailing list