commit e056fe9b886c3dfaf9546cd94656eec8117f70f8 Author: Arturo Filastò arturo@filasto.net Date: Wed Nov 7 17:14:59 2012 +0100

Implement GeoIP lookup support * From configuration options it is possible to choice what level of privacy the prober is willing to accept. --- ooni/config.py | 5 +++++ ooni/reporter.py | 34 ++++++++++++++++++---------------- ooni/utils/geodata.py | 27 +++++++++++++++++++++++++++ ooni/utils/log.py | 6 +++--- ooniprobe.conf | 16 +++++++++++----- 5 files changed, 64 insertions(+), 24 deletions(-)

diff --git a/ooni/config.py b/ooni/config.py index 97f396c..5880d69 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -29,6 +29,11 @@ basic = Storage() for k, v in configuration['basic'].items(): basic[k] = v

+# Process the privacy configuration options +privacy = Storage() +for k, v in configuration['privacy'].items(): + privacy[k] = v + # Process the advanced configuration options advanced = Storage() for k, v in configuration['advanced'].items(): diff --git a/ooni/reporter.py b/ooni/reporter.py index f3fbca6..6fab5f9 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -186,33 +186,35 @@ class ReporterFactory(OReporter): self._writeln("###########################################")

client_geodata = {} - log.msg("Running geo IP lookup via check.torproject.org")

- client_ip = yield geodata.myIP() - if config.includeip: + if config.privacy.includeip or \ + config.privacy.includeasn or \ + config.privacy.includecountry or \ + config.privacy.includecity: + log.msg("Running geo IP lookup via check.torproject.org") + client_ip = yield geodata.myIP() + client_location = geodata.IPToLocation(client_ip) + else: + client_ip = "127.0.0.1" + + if config.privacy.includeip: client_geodata['ip'] = client_ip else: - client_geodata['ip'] = None + client_geodata['ip'] = "127.0.0.1"

client_geodata['asn'] = None client_geodata['city'] = None client_geodata['countrycode'] = None

- try: - import txtorcon - client_location = txtorcon.util.NetLocation(client_geodata['ip']) - if config.includeasn: - client_geodata['asn'] = client_location.asn + if config.privacy.includeasn: + client_geodata['asn'] = client_location['asn']

- if config.includecity: - client_geodata['city'] = client_location.city + if config.privacy.includecity: + client_geodata['city'] = client_location['city']

- if config.includecountry: - client_geodata['countrycode'] = client_location.countrycode + if config.privacy.includecountry: + client_geodata['countrycode'] = client_location['countrycode']

- except ImportError: - log.err("txtorcon is not installed. Geolocation lookup is not"\ - "supported")

test_details = {'start_time': repr(date.now()), 'probe_asn': client_geodata['asn'], diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index 5254044..bd61dfd 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -1,7 +1,9 @@ import re import pygeoip +import os

from ooni import config +from ooni.utils import log

from twisted.web.client import Agent from twisted.internet import reactor, defer, protocol @@ -38,3 +40,28 @@ def myIP():

defer.returnValue(myip)

+class GeoIPDataFilesNotFound(Exception): + pass + +def IPToLocation(ipaddr): + city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') + country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') + asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') + + location = {'city': None, 'countrycode': None, 'asn': None} + try: + city_dat = pygeoip.GeoIP(city_file) + location['city'] = city_dat.record_by_addr(ipaddr)['city'] + + country_dat = pygeoip.GeoIP(country_file) + location['countrycode'] = country_dat.country_code_by_addr(ipaddr) + + asn_dat = pygeoip.GeoIP(asn_file) + location['asn'] = asn_dat.org_by_addr(ipaddr) + except IOError: + log.err("Could not find GeoIP data files. Go into data/ " + "and run make geoip") + raise GeoIPDataFilesNotFound + + return location + diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 57e5d61..0cb62ac 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -11,13 +11,13 @@ from twisted.python import log as txlog from twisted.python.logfile import DailyLogFile

from ooni.utils import otime -from ooni import oconfig +from ooni import config

def start(logfile=None): daily_logfile = None

if not logfile: - logfile = oconfig.basic.logfile + logfile = config.basic.logfile

log_folder = os.path.dirname(logfile) log_filename = os.path.basename(logfile) @@ -29,7 +29,7 @@ def start(logfile=None): logging.basicConfig() python_logging = txlog.PythonLoggingObserver()

- if oconfig.advanced.debug: + if config.advanced.debug: python_logging.logger.setLevel(logging.DEBUG) else: python_logging.logger.setLevel(logging.INFO) diff --git a/ooniprobe.conf b/ooniprobe.conf index 278dc4c..2039951 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -5,15 +5,21 @@ basic: # Where OONIProbe should be writing it's log file logfile: /tmp/ooniprobe.log + +privacy: # Should we include the IP address of the probe in the report? - includeip: true + includeip: false # Should we include the ASN of the probe in the report? - includeasn: true + includeasn: false # Should we include the ASN of the probe in the report? - includecountry: true + includecountry: false # Should we include the ASN of the probe in the report? - includecity: true + includecity: false + advanced: - geoip_data_dir: /home/x/code/networking/ooni-probe/data + # XXX change this to point to the directory where you have stored the GeoIP + # database file. This should be the directory in which OONI is installed + # /path/to/ooni-probe/data/ + geoip_data_dir: /home/x/code/networking/ooni-probe/data/ debug: true