[tor-commits] [ooni-probe/master] Refactoring of geoip related functionality to have it less coupled
art at torproject.org
art at torproject.org
Wed Apr 30 17:30:23 UTC 2014
commit d46b8791d3ba1e6520a725e94e8ed51763841334
Author: Arturo Filastò <art at fuffa.org>
Date: Sat Mar 8 13:47:25 2014 +0100
Refactoring of geoip related functionality to have it less coupled
---
ooni/director.py | 3 +--
ooni/geoip.py | 41 +++++++++++++++++++++++++++++++++++------
ooni/nettest.py | 38 ++++----------------------------------
ooni/settings.py | 6 +++---
ooni/utils/log.py | 6 +++++-
5 files changed, 48 insertions(+), 46 deletions(-)
diff --git a/ooni/director.py b/ooni/director.py
index e4fc261..27556d6 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -127,7 +127,6 @@ class Director(object):
log.msg("Connecting to Tor Control Port...")
yield self.getTorState()
- config.probe_ip = geoip.ProbeIP()
yield config.probe_ip.lookup()
@property
@@ -222,7 +221,7 @@ class Director(object):
"""
if config.privacy.includepcap:
if not config.reports.pcap:
- config.reports.pcap = config.generatePcapFilename(net_test_loader.testDetails)
+ config.reports.pcap = config.generate_pcap_filename(net_test_loader.testDetails)
self.startSniffing()
report = Report(reporters, self.reportEntryManager)
diff --git a/ooni/geoip.py b/ooni/geoip.py
index 2792126..53a1f70 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -5,11 +5,9 @@ import random
from twisted.web import client, http_headers
client._HTTP11ClientFactory.noisy = False
-from ooni.utils.net import userAgents, BodyReceiver
from twisted.internet import reactor, defer, protocol
from ooni.utils import log, net, checkForRoot
-from ooni.settings import config
from ooni import errors
try:
@@ -26,6 +24,8 @@ class GeoIPDataFilesNotFound(Exception):
pass
def IPToLocation(ipaddr):
+ from ooni.settings import config
+
city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
@@ -67,6 +67,8 @@ class HTTPGeoIPLookupper(object):
self.agent = self._agent(reactor)
def _response(self, response):
+ from ooni.utils.net import BodyReceiver
+
content_length = response.headers.getRawHeaders('content-length')
finished = defer.Deferred()
@@ -88,6 +90,8 @@ class HTTPGeoIPLookupper(object):
return failure
def lookup(self):
+ from ooni.utils.net import userAgents
+
headers = {}
headers['User-Agent'] = [random.choice(userAgents)]
@@ -117,16 +121,37 @@ class ProbeIP(object):
address = None
def __init__(self):
- self.tor_state = config.tor_state
- self.geoIPServices = {'ubuntu': UbuntuGeoIP,
+ self.geoIPServices = {
+ 'ubuntu': UbuntuGeoIP,
'torproject': TorProjectGeoIP
}
+ self.geodata = {
+ 'asn': 'AS0',
+ 'city': None,
+ 'countrycode': 'ZZ',
+ 'ip': '127.0.0.1'
+ }
+
+ def resolveGeodata(self):
+ from ooni.settings import config
+
+ self.geodata = IPToLocation(self.address)
+ self.geodata['ip'] = self.address
+ if not config.privacy.includeasn:
+ self.geodata['asn'] = 'AS0'
+ if not config.privacy.includecity:
+ self.geodata['city'] = None
+ if not config.privacy.includecountry:
+ self.geodata['countrycode'] = 'ZZ'
+ if not config.privacy.includeip:
+ self.geodata['ip'] = '127.0.0.1'
@defer.inlineCallbacks
def lookup(self):
try:
yield self.askTor()
log.msg("Found your IP via Tor %s" % self.address)
+ self.resolveGeodata()
defer.returnValue(self.address)
except errors.TorStateNotFound:
log.debug("Tor is not running. Skipping IP lookup via Tor.")
@@ -136,6 +161,7 @@ class ProbeIP(object):
try:
yield self.askTraceroute()
log.msg("Found your IP via Traceroute %s" % self.address)
+ self.resolveGeodata()
defer.returnValue(self.address)
except errors.InsufficientPrivileges:
log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
@@ -145,6 +171,7 @@ class ProbeIP(object):
try:
yield self.askGeoIPService()
log.msg("Found your IP via a GeoIP service: %s" % self.address)
+ self.resolveGeodata()
defer.returnValue(self.address)
except Exception, e:
log.msg("Unable to lookup the probe IP via GeoIPService")
@@ -183,8 +210,10 @@ class ProbeIP(object):
XXX this lookup method is currently broken when there are cached descriptors or consensus documents
see: https://trac.torproject.org/projects/tor/ticket/8214
"""
- if self.tor_state:
- d = self.tor_state.protocol.get_info("address")
+ from ooni.settings import config
+
+ if config.tor_state:
+ d = config.tor_state.protocol.get_info("address")
@d.addCallback
def cb(result):
self.strategy = 'tor_get_info_address'
diff --git a/ooni/nettest.py b/ooni/nettest.py
index ce7831e..2ea0522 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -238,45 +238,15 @@ class NetTestLoader(object):
def testDetails(self):
from ooni import __version__ as software_version
- client_geodata = {
- 'city': None,
- 'countrycode': 'ZZ',
- 'asn': 'AS0',
- 'ip': '127.0.0.1'
- }
-
- if config.probe_ip and config.probe_ip.address and (config.privacy.includeip or \
- config.privacy.includeasn or \
- config.privacy.includecountry or \
- config.privacy.includecity):
- log.msg("We will include some geo data in the report")
- client_geodata = geoip.IPToLocation(config.probe_ip.address)
-
- if config.privacy.includeip:
- client_geodata['ip'] = config.probe_ip.address
- else:
- client_geodata['ip'] = "127.0.0.1"
-
- # Here we unset all the client geodata if the option to not include then
- # has been specified
- if not config.privacy.includeasn:
- client_geodata['asn'] = 'AS0'
-
- if not config.privacy.includecity:
- client_geodata['city'] = None
-
- if not config.privacy.includecountry:
- client_geodata['countrycode'] = None
-
input_file_hashes = []
for input_file in self.inputFiles:
input_file_hashes.append(input_file['hash'])
test_details = {'start_time': time.time(),
- 'probe_asn': client_geodata['asn'],
- 'probe_cc': client_geodata['countrycode'],
- 'probe_ip': client_geodata['ip'],
- 'probe_city': client_geodata['city'],
+ 'probe_asn': config.probe_ip.geodata['asn'],
+ 'probe_cc': config.probe_ip.geodata['countrycode'],
+ 'probe_ip': config.probe_ip.geodata['ip'],
+ 'probe_city': config.probe_ip.geodata['city'],
'test_name': self.testName,
'test_version': self.testVersion,
'software_name': 'ooniprobe',
diff --git a/ooni/settings.py b/ooni/settings.py
index c1071b9..0fed604 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -8,7 +8,7 @@ from os.path import abspath, expanduser
from twisted.internet import reactor, threads, defer
-from ooni import otime
+from ooni import otime, geoip
from ooni.utils import Storage
class OConfig(object):
@@ -19,7 +19,7 @@ class OConfig(object):
self.scapyFactory = None
self.tor_state = None
# This is used to store the probes IP address obtained via Tor
- self.probe_ip = None
+ self.probe_ip = geoip.ProbeIP()
# This is used to keep track of the state of the sniffer
self.sniffer_running = None
self.logging = True
@@ -91,7 +91,7 @@ class OConfig(object):
pass
self.set_paths()
- def generatePcapFilename(self, testDetails):
+ def generate_pcap_filename(self, testDetails):
test_name, start_time = testDetails['test_name'], testDetails['start_time']
start_time = otime.epochToTimestamp(start_time)
return "report-%s-%s.%s" % (test_name, start_time, "pcap")
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 379b5c7..773673b 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -9,7 +9,6 @@ from twisted.python.failure import Failure
from twisted.python.logfile import DailyLogFile
from ooni import otime
-from ooni.settings import config
## Get rid of the annoying "No route found for
## IPv6 destination warnings":
@@ -26,6 +25,8 @@ class LogWithNoPrefix(txlog.FileLogObserver):
class OONILogger(object):
def start(self, logfile=None, application_name="ooniprobe"):
+ from ooni.settings import config
+
daily_logfile = None
if not logfile:
@@ -58,14 +59,17 @@ def stop():
oonilogger.stop()
def msg(msg, *arg, **kw):
+ from ooni.settings import config
if config.logging:
print "%s" % msg
def debug(msg, *arg, **kw):
+ from ooni.settings import config
if config.advanced.debug and config.logging:
print "[D] %s" % msg
def err(msg, *arg, **kw):
+ from ooni.settings import config
if config.logging:
print "[!] %s" % msg
More information about the tor-commits
mailing list