[tor-commits] [ooni-probe/develop] Merge branch 'master' into feature/probe_ip_lookup
isis at torproject.org
isis at torproject.org
Thu Jun 6 16:41:37 UTC 2013
commit c597157a554b47f59b1ee85fae2019a0dbc8834e
Merge: 5f55b18 0b84394
Author: Arturo Filastò <art at fuffa.org>
Date: Wed Mar 20 13:57:01 2013 -0400
Merge branch 'master' into feature/probe_ip_lookup
* master:
Change None to correct YAML "Null" syntax in ooniprobe.conf.sample.
Support CPython with patch from lunar https://trac.torproject.org/projects/tor/ticket/8507
Add the 'reports' group to the sample config file.
Change ooni.director.start() to set config.reports.pcap option.
Move config processing options to new function config.processConfigFile().
Rewrite config.generatePcapFilename().
Conflicts:
ooni/utils/geodata.py
ooni/config.py | 61 ++++++++++++++++++++++++++++++++++---------------
ooni/director.py | 2 +-
ooni/geoip.py | 15 ++++++++----
ooniprobe.conf.sample | 4 +++-
4 files changed, 57 insertions(+), 25 deletions(-)
diff --cc ooni/geoip.py
index efac5e9,0000000..0f8335f
mode 100644,000000..100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@@ -1,183 -1,0 +1,188 @@@
+import re
+import os
+import random
+
+from twisted.web import client, http_headers
+from ooni.utils.net import userAgents, BodyReceiver
+from twisted.internet import reactor, defer, protocol
+
+from ooni.utils import log, net, checkForRoot
+from ooni import config, errors
+
+try:
- import pygeoip
++ from pygeoip import GeoIP
+except ImportError:
- log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements")
++ try:
++ import GeoIP as CGeoIP
++ def GeoIP(database_path, *args, **kwargs):
++ return CGeoIP.open(database_path)
++ except ImportError:
++ log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements")
+
+class GeoIPDataFilesNotFound(Exception):
+ pass
+
+def IPToLocation(ipaddr):
+ city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
+ country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
+ asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
+
+ location = {'city': None, 'countrycode': None, 'asn': None}
+ try:
- city_dat = pygeoip.GeoIP(city_file)
++ city_dat = GeoIP(city_file)
+ location['city'] = city_dat.record_by_addr(ipaddr)['city']
+
- country_dat = pygeoip.GeoIP(country_file)
++ country_dat = GeoIP(country_file)
+ location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
+
- asn_dat = pygeoip.GeoIP(asn_file)
++ asn_dat = GeoIP(asn_file)
+ location['asn'] = asn_dat.org_by_addr(ipaddr)
+
+ except IOError:
+ log.err("Could not find GeoIP data files. Go into data/ "
+ "and run make geoip")
+ raise GeoIPDataFilesNotFound
+
+ return location
+
+class HTTPGeoIPLookupper(object):
+ url = None
+
+ _agent = client.Agent
+
+ def __init__(self):
+ self.agent = self._agent(reactor)
+
+ def _response(self, response):
+ content_length = response.headers.getRawHeaders('content-length')
+
+ finished = defer.Deferred()
+ response.deliverBody(BodyReceiver(finished, content_length))
+ finished.addCallback(self.parseResponse)
+ return finished
+
+ def parseResponse(self, response_body):
+ """
+ Override this with the logic for parsing the response.
+
+ Should return the IP address of the probe.
+ """
+ pass
+
+ def failed(self, failure):
+ log.err("Failed to lookup via %s" % self.url)
+ log.exception(failure)
+ return failure
+
+ def lookup(self):
+ headers = {}
+ headers['User-Agent'] = [random.choice(userAgents)]
+
+ d = self.agent.request("GET", self.url, http_headers.Headers(headers))
+ d.addCallback(self._response)
+ d.addErrback(self.failed)
+ return d
+
+class UbuntuGeoIP(HTTPGeoIPLookupper):
+ url = "http://geoip.ubuntu.com/lookup"
+
+ def parseResponse(self, response_body):
+ m = re.match(".*<Ip>(.*)</Ip>.*", response_body)
+ probe_ip = m.group(1)
+ return probe_ip
+
+class TorProjectGeoIP(HTTPGeoIPLookupper):
+ url = "https://check.torproject.org/"
+
+ def parseResponse(self, response_body):
+ regexp = "Your IP address appears to be: <b>((\d+\.)+(\d+))"
+ probe_ip = re.search(regexp, response_body).group(1)
+ return probe_ip
+
+class MaxMindGeoIP(HTTPGeoIPLookupper):
+ url = "https://www.maxmind.com/en/locate_my_ip"
+
+ def parseResponse(self, response_body):
+ regexp = '<span id="my-ip-address">((\d+\.)+(\d+))</span>'
+ probe_ip = re.search(regexp, response_body).group(1)
+ return probe_ip
+
+class ProbeIP(object):
+ strategy = None
+ geoIPServices = {'ubuntu': UbuntuGeoIP,
+ 'torproject': TorProjectGeoIP,
+ 'maxmind': MaxMindGeoIP
+ }
+ address = None
+ tor_state = config.tor_state
+
+ @defer.inlineCallbacks
+ def lookup(self):
+ try:
+ yield self.askTor()
+ log.msg("Found your IP via Tor %s" % self.address)
+ defer.returnValue(self.address)
+ except errors.TorStateNotFound:
+ log.debug("Tor is not running. Skipping IP lookup via Tor.")
+ except Exception:
+ log.msg("Unable to lookup the probe IP via Tor.")
+
+ try:
+ yield self.askTraceroute()
+ log.msg("Found your IP via Traceroute %s" % self.address)
+ defer.returnValue(self.address)
+ except errors.InsufficientPrivileges:
+ log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
+ except:
+ log.msg("Unable to lookup the probe IP via traceroute")
+
+ try:
+ yield self.askGeoIPService()
+ log.msg("Found your IP via a GeoIP service: %s" % self.address)
+ defer.returnValue(self.address)
+ except Exception, e:
+ log.msg("Unable to lookup the probe IP via GeoIPService")
+ raise e
+
+ @defer.inlineCallbacks
+ def askGeoIPService(self):
+ # Shuffle the order in which we test the geoip services.
+ services = self.geoIPServices.items()
+ random.shuffle(services)
+ for service_name, service in services:
+ s = service()
+ log.msg("Looking up your IP address via %s" % service_name)
+ try:
+ self.address = yield s.lookup()
+ self.strategy = 'geo_ip_service-' + service_name
+ break
+ except Exception, e:
+ log.msg("Failed to lookup your IP via %s" % service_name)
+
+ if not self.address:
+ raise errors.ProbeIPUnknown
+
+ def askTraceroute(self):
+ """
+ Perform a UDP traceroute to determine the probes IP address.
+ """
+ checkForRoot()
+ raise NotImplemented
+
+ def askTor(self):
+ """
+ Obtain the probes IP address by asking the Tor Control port via GET INFO
+ address.
+
+ XXX this lookup method is currently broken when there are cached descriptors or consensus documents
+ see: https://trac.torproject.org/projects/tor/ticket/8214
+ """
+ if self.tor_state:
+ d = self.tor_state.protocol.get_info("address")
+ @d.addCallback
+ def cb(result):
+ self.strategy = 'tor_get_info_address'
+ self.address = result.values()[0]
+ return d
+ else:
+ raise errors.TorStateNotFound
More information about the tor-commits
mailing list