[tor-commits] [ooni-probe/master] Implement GeoIP lookup support
art at torproject.org
art at torproject.org
Wed Nov 7 16:55:33 UTC 2012
commit e056fe9b886c3dfaf9546cd94656eec8117f70f8
Author: Arturo Filastò <arturo at filasto.net>
Date: Wed Nov 7 17:14:59 2012 +0100
Implement GeoIP lookup support
* From configuration options it is possible to choice what level of privacy the
prober is willing to accept.
---
ooni/config.py | 5 +++++
ooni/reporter.py | 34 ++++++++++++++++++----------------
ooni/utils/geodata.py | 27 +++++++++++++++++++++++++++
ooni/utils/log.py | 6 +++---
ooniprobe.conf | 16 +++++++++++-----
5 files changed, 64 insertions(+), 24 deletions(-)
diff --git a/ooni/config.py b/ooni/config.py
index 97f396c..5880d69 100644
--- a/ooni/config.py
+++ b/ooni/config.py
@@ -29,6 +29,11 @@ basic = Storage()
for k, v in configuration['basic'].items():
basic[k] = v
+# Process the privacy configuration options
+privacy = Storage()
+for k, v in configuration['privacy'].items():
+ privacy[k] = v
+
# Process the advanced configuration options
advanced = Storage()
for k, v in configuration['advanced'].items():
diff --git a/ooni/reporter.py b/ooni/reporter.py
index f3fbca6..6fab5f9 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -186,33 +186,35 @@ class ReporterFactory(OReporter):
self._writeln("###########################################")
client_geodata = {}
- log.msg("Running geo IP lookup via check.torproject.org")
- client_ip = yield geodata.myIP()
- if config.includeip:
+ if config.privacy.includeip or \
+ config.privacy.includeasn or \
+ config.privacy.includecountry or \
+ config.privacy.includecity:
+ log.msg("Running geo IP lookup via check.torproject.org")
+ client_ip = yield geodata.myIP()
+ client_location = geodata.IPToLocation(client_ip)
+ else:
+ client_ip = "127.0.0.1"
+
+ if config.privacy.includeip:
client_geodata['ip'] = client_ip
else:
- client_geodata['ip'] = None
+ client_geodata['ip'] = "127.0.0.1"
client_geodata['asn'] = None
client_geodata['city'] = None
client_geodata['countrycode'] = None
- try:
- import txtorcon
- client_location = txtorcon.util.NetLocation(client_geodata['ip'])
- if config.includeasn:
- client_geodata['asn'] = client_location.asn
+ if config.privacy.includeasn:
+ client_geodata['asn'] = client_location['asn']
- if config.includecity:
- client_geodata['city'] = client_location.city
+ if config.privacy.includecity:
+ client_geodata['city'] = client_location['city']
- if config.includecountry:
- client_geodata['countrycode'] = client_location.countrycode
+ if config.privacy.includecountry:
+ client_geodata['countrycode'] = client_location['countrycode']
- except ImportError:
- log.err("txtorcon is not installed. Geolocation lookup is not"\
- "supported")
test_details = {'start_time': repr(date.now()),
'probe_asn': client_geodata['asn'],
diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py
index 5254044..bd61dfd 100644
--- a/ooni/utils/geodata.py
+++ b/ooni/utils/geodata.py
@@ -1,7 +1,9 @@
import re
import pygeoip
+import os
from ooni import config
+from ooni.utils import log
from twisted.web.client import Agent
from twisted.internet import reactor, defer, protocol
@@ -38,3 +40,28 @@ def myIP():
defer.returnValue(myip)
+class GeoIPDataFilesNotFound(Exception):
+ pass
+
+def IPToLocation(ipaddr):
+ city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
+ country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
+ asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
+
+ location = {'city': None, 'countrycode': None, 'asn': None}
+ try:
+ city_dat = pygeoip.GeoIP(city_file)
+ location['city'] = city_dat.record_by_addr(ipaddr)['city']
+
+ country_dat = pygeoip.GeoIP(country_file)
+ location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
+
+ asn_dat = pygeoip.GeoIP(asn_file)
+ location['asn'] = asn_dat.org_by_addr(ipaddr)
+ except IOError:
+ log.err("Could not find GeoIP data files. Go into data/ "
+ "and run make geoip")
+ raise GeoIPDataFilesNotFound
+
+ return location
+
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 57e5d61..0cb62ac 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -11,13 +11,13 @@ from twisted.python import log as txlog
from twisted.python.logfile import DailyLogFile
from ooni.utils import otime
-from ooni import oconfig
+from ooni import config
def start(logfile=None):
daily_logfile = None
if not logfile:
- logfile = oconfig.basic.logfile
+ logfile = config.basic.logfile
log_folder = os.path.dirname(logfile)
log_filename = os.path.basename(logfile)
@@ -29,7 +29,7 @@ def start(logfile=None):
logging.basicConfig()
python_logging = txlog.PythonLoggingObserver()
- if oconfig.advanced.debug:
+ if config.advanced.debug:
python_logging.logger.setLevel(logging.DEBUG)
else:
python_logging.logger.setLevel(logging.INFO)
diff --git a/ooniprobe.conf b/ooniprobe.conf
index 278dc4c..2039951 100644
--- a/ooniprobe.conf
+++ b/ooniprobe.conf
@@ -5,15 +5,21 @@
basic:
# Where OONIProbe should be writing it's log file
logfile: /tmp/ooniprobe.log
+
+privacy:
# Should we include the IP address of the probe in the report?
- includeip: true
+ includeip: false
# Should we include the ASN of the probe in the report?
- includeasn: true
+ includeasn: false
# Should we include the ASN of the probe in the report?
- includecountry: true
+ includecountry: false
# Should we include the ASN of the probe in the report?
- includecity: true
+ includecity: false
+
advanced:
- geoip_data_dir: /home/x/code/networking/ooni-probe/data
+ # XXX change this to point to the directory where you have stored the GeoIP
+ # database file. This should be the directory in which OONI is installed
+ # /path/to/ooni-probe/data/
+ geoip_data_dir: /home/x/code/networking/ooni-probe/data/
debug: true
More information about the tor-commits
mailing list