commit 51fe88e01de3d559522dc2cf3ea518f65993635b Author: Arturo Filastò art@fuffa.org Date: Fri Mar 15 09:22:13 2013 +0100

Move geoip related classes into ooni.geoip --- HACKING | 2 +- ooni/director.py | 133 +-------------------------------------- ooni/geoip.py | 168 +++++++++++++++++++++++++++++++++++++++++++++++++ ooni/nettest.py | 5 +- ooni/reporter.py | 2 +- ooni/utils/geodata.py | 40 ------------ tests/test_nettest.py | 2 +- 7 files changed, 176 insertions(+), 176 deletions(-)

diff --git a/HACKING b/HACKING index b92ddbe..aad7ebb 100644 --- a/HACKING +++ b/HACKING @@ -87,7 +87,7 @@ Code Structure - ooni/utils/ In here go internal utilities that are useful to ooniprobe

-- ooni/utils/geodata.py +- ooni/geoip.py In here go functions related to the understanding of geographical information of the probe

diff --git a/ooni/director.py b/ooni/director.py index 07510eb..d2f5a7c 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -5,6 +5,7 @@ import os import re

from ooni import config +from ooni import geoip from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report from ooni.utils import log, checkForRoot @@ -16,136 +17,6 @@ from txtorcon import TorConfig from txtorcon import TorState, launch_tor

from twisted.internet import defer, reactor -from twisted.web import client, http_headers -from ooni.utils.net import userAgents, BodyReceiver - -class HTTPGeoIPLookupper(object): - url = None - - def _response(self, response): - content_length = response.headers.getRawHeaders('content-length') - - finished = defer.Deferred() - response.deliverBody(BodyReceiver(finished, content_length)) - finished.addCallback(self.parseResponse) - return finished - - def parseResponse(self, response_body): - """ - Override this with the logic for parsing the response. - - Should return the IP address of the probe. - """ - pass - - def failed(self, failure): - log.err("Failed to lookup via %s" % url) - log.exception(failure) - return failure - - def lookup(self): - agent = client.Agent(reactor) - headers = {} - headers['User-Agent'] = [random.choice(userAgents)] - - d = agent.request("GET", self.url, http_headers.Headers(headers)) - d.addCallback(self._response) - d.addErrback(self.failed) - return d - -class UbuntuGeoIP(HTTPGeoIPLookupper): - url = "http://geoip.ubuntu.com/lookup" - - def parseResponse(self, response_body): - response = ET.fromstring(response_body) - probe_ip = response.find('Ip').text - return probe_ip - -class TorProjectGeoIP(HTTPGeoIPLookupper): - url = "https://check.torproject.org/" - - def parseResponse(self, response_body): - regexp = "Your IP address appears to be: <b>((\d+.)+(\d+))" - probe_ip = re.search(regexp, response_body).group(1) - return probe_ip - -class MaxMindGeoIP(HTTPGeoIPLookupper): - url = "https://www.maxmind.com/en/locate_my_ip" - - def parseResponse(self, response_body): - regexp = '<span id="my-ip-address">((\d+.)+(\d+))</span>' - probe_ip = re.search(regexp, response_body).group(1) - return probe_ip - -class ProbeIP(object): - strategy = None - geoIPServices = {'ubuntu': UbuntuGeoIP, - 'torproject': TorProjectGeoIP, - 'maximind': MaxMindGeoIP - } - address = None - - @defer.inlineCallbacks - def lookup(self): - try: - yield self.askTor() - defer.returnValue(self.address) - except errors.TorStateNotFound: - log.debug("Tor is not running. Skipping IP lookup via Tor.") - except: - log.msg("Unable to lookup the probe IP via Tor.") - - try: - yield self.askTraceroute() - defer.returnValue(self.address) - except errors.InsufficientPrivileges: - log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") - except: - log.msg("Unable to lookup the probe IP via traceroute") - - try: - yield self.askGeoIPService() - defer.returnValue(self.address) - except Exception, e: - print e - log.msg("Unable to lookup the probe IP via GeoIPService") - - @defer.inlineCallbacks - def askGeoIPService(self): - for service_name, service in self.geoIPServices.items(): - s = TorProjectGeoIP() - log.msg("Looking up your IP address via %s" % service_name) - try: - self.address = yield s.lookup() - self.strategy = 'geo_ip_service-' + service_name - break - except: - log.msg("Failed to lookup your IP via %s" % service_name) - - def askTraceroute(self): - """ - Perform a UDP traceroute to determine the probes IP address. - """ - checkForRoot() - raise NotImplemented - - def askTor(self): - """ - Obtain the probes IP address by asking the Tor Control port via GET INFO - address. - - XXX this lookup method is currently broken when there are cached descriptors or consensus documents - see: https://trac.torproject.org/projects/tor/ticket/8214 - """ - if config.tor_state: - d = config.tor_state.protocol.get_info("address") - @d.addCallback - def cb(result): - self.strategy = 'tor_get_info_address' - self.address = result.values()[0] - return d - else: - raise errors.TorStateNotFound

class Director(object): """ @@ -225,7 +96,7 @@ class Director(object): log.msg("Starting Tor...") yield self.startTor()

- config.probe_ip = ProbeIP() + config.probe_ip = geoip.ProbeIP() yield config.probe_ip.lookup()

@property diff --git a/ooni/geoip.py b/ooni/geoip.py new file mode 100644 index 0000000..0fe8a6b --- /dev/null +++ b/ooni/geoip.py @@ -0,0 +1,168 @@ +import re +import os + +from twisted.web import client, http_headers +from ooni.utils.net import userAgents, BodyReceiver +from twisted.internet import reactor, defer, protocol + +from ooni.utils import log, net +from ooni import config + +try: + import pygeoip +except ImportError: + log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") + +class GeoIPDataFilesNotFound(Exception): + pass + +def IPToLocation(ipaddr): + city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') + country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') + asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') + + location = {'city': None, 'countrycode': None, 'asn': None} + try: + city_dat = pygeoip.GeoIP(city_file) + location['city'] = city_dat.record_by_addr(ipaddr)['city'] + + country_dat = pygeoip.GeoIP(country_file) + location['countrycode'] = country_dat.country_code_by_addr(ipaddr) + + asn_dat = pygeoip.GeoIP(asn_file) + location['asn'] = asn_dat.org_by_addr(ipaddr) + + except IOError: + log.err("Could not find GeoIP data files. Go into data/ " + "and run make geoip") + raise GeoIPDataFilesNotFound + + return location + +class HTTPGeoIPLookupper(object): + url = None + + def _response(self, response): + content_length = response.headers.getRawHeaders('content-length') + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished, content_length)) + finished.addCallback(self.parseResponse) + return finished + + def parseResponse(self, response_body): + """ + Override this with the logic for parsing the response. + + Should return the IP address of the probe. + """ + pass + + def failed(self, failure): + log.err("Failed to lookup via %s" % url) + log.exception(failure) + return failure + + def lookup(self): + agent = client.Agent(reactor) + headers = {} + headers['User-Agent'] = [random.choice(userAgents)] + + d = agent.request("GET", self.url, http_headers.Headers(headers)) + d.addCallback(self._response) + d.addErrback(self.failed) + return d + +class UbuntuGeoIP(HTTPGeoIPLookupper): + url = "http://geoip.ubuntu.com/lookup" + + def parseResponse(self, response_body): + response = ET.fromstring(response_body) + probe_ip = response.find('Ip').text + return probe_ip + +class TorProjectGeoIP(HTTPGeoIPLookupper): + url = "https://check.torproject.org/" + + def parseResponse(self, response_body): + regexp = "Your IP address appears to be: <b>((\d+.)+(\d+))" + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class MaxMindGeoIP(HTTPGeoIPLookupper): + url = "https://www.maxmind.com/en/locate_my_ip" + + def parseResponse(self, response_body): + regexp = '<span id="my-ip-address">((\d+.)+(\d+))</span>' + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class ProbeIP(object): + strategy = None + geoIPServices = {'ubuntu': UbuntuGeoIP, + 'torproject': TorProjectGeoIP, + 'maximind': MaxMindGeoIP + } + address = None + + @defer.inlineCallbacks + def lookup(self): + try: + yield self.askTor() + defer.returnValue(self.address) + except errors.TorStateNotFound: + log.debug("Tor is not running. Skipping IP lookup via Tor.") + except: + log.msg("Unable to lookup the probe IP via Tor.") + + try: + yield self.askTraceroute() + defer.returnValue(self.address) + except errors.InsufficientPrivileges: + log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") + except: + log.msg("Unable to lookup the probe IP via traceroute") + + try: + yield self.askGeoIPService() + defer.returnValue(self.address) + except Exception, e: + print e + log.msg("Unable to lookup the probe IP via GeoIPService") + + @defer.inlineCallbacks + def askGeoIPService(self): + for service_name, service in self.geoIPServices.items(): + s = TorProjectGeoIP() + log.msg("Looking up your IP address via %s" % service_name) + try: + self.address = yield s.lookup() + self.strategy = 'geo_ip_service-' + service_name + break + except: + log.msg("Failed to lookup your IP via %s" % service_name) + + def askTraceroute(self): + """ + Perform a UDP traceroute to determine the probes IP address. + """ + checkForRoot() + raise NotImplemented + + def askTor(self): + """ + Obtain the probes IP address by asking the Tor Control port via GET INFO + address. + + XXX this lookup method is currently broken when there are cached descriptors or consensus documents + see: https://trac.torproject.org/projects/tor/ticket/8214 + """ + if config.tor_state: + d = config.tor_state.protocol.get_info("address") + @d.addCallback + def cb(result): + self.strategy = 'tor_get_info_address' + self.address = result.values()[0] + return d + else: + raise errors.TorStateNotFound diff --git a/ooni/nettest.py b/ooni/nettest.py index 67dee9d..bd6bd41 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -5,8 +5,9 @@ from twisted.internet import defer, reactor from twisted.trial.runner import filenameToModule from twisted.python import usage, reflect

+from ooni import geoip from ooni.tasks import Measurement -from ooni.utils import log, checkForRoot, geodata +from ooni.utils import log, checkForRoot from ooni import config from ooni import otime

@@ -35,7 +36,7 @@ class NetTestLoader(object): config.privacy.includecountry or \ config.privacy.includecity): log.msg("We will include some geo data in the report") - client_geodata = geodata.IPToLocation(config.probe_ip.address) + client_geodata = geoip.IPToLocation(config.probe_ip.address)

if config.privacy.includeip: client_geodata['ip'] = config.probe_ip.address diff --git a/ooni/reporter.py b/ooni/reporter.py index 20193c9..fc73b67 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -30,7 +30,7 @@ from ooni.errors import InvalidOONIBCollectorAddress from ooni.errors import ReportNotCreated, ReportAlreadyClosed

from ooni import otime -from ooni.utils import geodata, pushFilenameStack +from ooni.utils import pushFilenameStack from ooni.utils.net import BodyReceiver, StringProducer, userAgents

from ooni import config diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py deleted file mode 100644 index d9883ba..0000000 --- a/ooni/utils/geodata.py +++ /dev/null @@ -1,40 +0,0 @@ -import re -import os - -from twisted.web.client import Agent -from twisted.internet import reactor, defer, protocol - -from ooni.utils import log, net -from ooni import config - -try: - import pygeoip -except ImportError: - log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") - -class GeoIPDataFilesNotFound(Exception): - pass - -def IPToLocation(ipaddr): - city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') - country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') - asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') - - location = {'city': None, 'countrycode': None, 'asn': None} - try: - city_dat = pygeoip.GeoIP(city_file) - location['city'] = city_dat.record_by_addr(ipaddr)['city'] - - country_dat = pygeoip.GeoIP(country_file) - location['countrycode'] = country_dat.country_code_by_addr(ipaddr) - - asn_dat = pygeoip.GeoIP(asn_file) - location['asn'] = asn_dat.org_by_addr(ipaddr) - - except IOError: - log.err("Could not find GeoIP data files. Go into data/ " - "and run make geoip") - raise GeoIPDataFilesNotFound - - return location - diff --git a/tests/test_nettest.py b/tests/test_nettest.py index 976757f..2d070c3 100644 --- a/tests/test_nettest.py +++ b/tests/test_nettest.py @@ -241,7 +241,7 @@ class TestNetTest(unittest.TestCase): # try: # net_test = NetTestLoader(StringIO(net_test_root_required), # dummyArgs) - # except NotRootError: + # except errors.InsufficientPrivileges: # pass

#def test_create_report_succeed(self):