commit d46b8791d3ba1e6520a725e94e8ed51763841334 Author: Arturo Filastò art@fuffa.org Date: Sat Mar 8 13:47:25 2014 +0100

Refactoring of geoip related functionality to have it less coupled --- ooni/director.py | 3 +-- ooni/geoip.py | 41 +++++++++++++++++++++++++++++++++++------ ooni/nettest.py | 38 ++++---------------------------------- ooni/settings.py | 6 +++--- ooni/utils/log.py | 6 +++++- 5 files changed, 48 insertions(+), 46 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py index e4fc261..27556d6 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -127,7 +127,6 @@ class Director(object): log.msg("Connecting to Tor Control Port...") yield self.getTorState()

- config.probe_ip = geoip.ProbeIP() yield config.probe_ip.lookup()

@property @@ -222,7 +221,7 @@ class Director(object): """ if config.privacy.includepcap: if not config.reports.pcap: - config.reports.pcap = config.generatePcapFilename(net_test_loader.testDetails) + config.reports.pcap = config.generate_pcap_filename(net_test_loader.testDetails) self.startSniffing()

report = Report(reporters, self.reportEntryManager) diff --git a/ooni/geoip.py b/ooni/geoip.py index 2792126..53a1f70 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -5,11 +5,9 @@ import random from twisted.web import client, http_headers client._HTTP11ClientFactory.noisy = False

-from ooni.utils.net import userAgents, BodyReceiver from twisted.internet import reactor, defer, protocol

from ooni.utils import log, net, checkForRoot -from ooni.settings import config from ooni import errors

try: @@ -26,6 +24,8 @@ class GeoIPDataFilesNotFound(Exception): pass

def IPToLocation(ipaddr): + from ooni.settings import config + city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') @@ -67,6 +67,8 @@ class HTTPGeoIPLookupper(object): self.agent = self._agent(reactor)

def _response(self, response): + from ooni.utils.net import BodyReceiver + content_length = response.headers.getRawHeaders('content-length')

finished = defer.Deferred() @@ -88,6 +90,8 @@ class HTTPGeoIPLookupper(object): return failure

def lookup(self): + from ooni.utils.net import userAgents + headers = {} headers['User-Agent'] = [random.choice(userAgents)]

@@ -117,16 +121,37 @@ class ProbeIP(object): address = None

def __init__(self): - self.tor_state = config.tor_state - self.geoIPServices = {'ubuntu': UbuntuGeoIP, + self.geoIPServices = { + 'ubuntu': UbuntuGeoIP, 'torproject': TorProjectGeoIP } + self.geodata = { + 'asn': 'AS0', + 'city': None, + 'countrycode': 'ZZ', + 'ip': '127.0.0.1' + } + + def resolveGeodata(self): + from ooni.settings import config + + self.geodata = IPToLocation(self.address) + self.geodata['ip'] = self.address + if not config.privacy.includeasn: + self.geodata['asn'] = 'AS0' + if not config.privacy.includecity: + self.geodata['city'] = None + if not config.privacy.includecountry: + self.geodata['countrycode'] = 'ZZ' + if not config.privacy.includeip: + self.geodata['ip'] = '127.0.0.1'

@defer.inlineCallbacks def lookup(self): try: yield self.askTor() log.msg("Found your IP via Tor %s" % self.address) + self.resolveGeodata() defer.returnValue(self.address) except errors.TorStateNotFound: log.debug("Tor is not running. Skipping IP lookup via Tor.") @@ -136,6 +161,7 @@ class ProbeIP(object): try: yield self.askTraceroute() log.msg("Found your IP via Traceroute %s" % self.address) + self.resolveGeodata() defer.returnValue(self.address) except errors.InsufficientPrivileges: log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") @@ -145,6 +171,7 @@ class ProbeIP(object): try: yield self.askGeoIPService() log.msg("Found your IP via a GeoIP service: %s" % self.address) + self.resolveGeodata() defer.returnValue(self.address) except Exception, e: log.msg("Unable to lookup the probe IP via GeoIPService") @@ -183,8 +210,10 @@ class ProbeIP(object): XXX this lookup method is currently broken when there are cached descriptors or consensus documents see: https://trac.torproject.org/projects/tor/ticket/8214 """ - if self.tor_state: - d = self.tor_state.protocol.get_info("address") + from ooni.settings import config + + if config.tor_state: + d = config.tor_state.protocol.get_info("address") @d.addCallback def cb(result): self.strategy = 'tor_get_info_address' diff --git a/ooni/nettest.py b/ooni/nettest.py index ce7831e..2ea0522 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -238,45 +238,15 @@ class NetTestLoader(object): def testDetails(self): from ooni import __version__ as software_version

- client_geodata = { - 'city': None, - 'countrycode': 'ZZ', - 'asn': 'AS0', - 'ip': '127.0.0.1' - } - - if config.probe_ip and config.probe_ip.address and (config.privacy.includeip or \ - config.privacy.includeasn or \ - config.privacy.includecountry or \ - config.privacy.includecity): - log.msg("We will include some geo data in the report") - client_geodata = geoip.IPToLocation(config.probe_ip.address) - - if config.privacy.includeip: - client_geodata['ip'] = config.probe_ip.address - else: - client_geodata['ip'] = "127.0.0.1" - - # Here we unset all the client geodata if the option to not include then - # has been specified - if not config.privacy.includeasn: - client_geodata['asn'] = 'AS0' - - if not config.privacy.includecity: - client_geodata['city'] = None - - if not config.privacy.includecountry: - client_geodata['countrycode'] = None - input_file_hashes = [] for input_file in self.inputFiles: input_file_hashes.append(input_file['hash'])

test_details = {'start_time': time.time(), - 'probe_asn': client_geodata['asn'], - 'probe_cc': client_geodata['countrycode'], - 'probe_ip': client_geodata['ip'], - 'probe_city': client_geodata['city'], + 'probe_asn': config.probe_ip.geodata['asn'], + 'probe_cc': config.probe_ip.geodata['countrycode'], + 'probe_ip': config.probe_ip.geodata['ip'], + 'probe_city': config.probe_ip.geodata['city'], 'test_name': self.testName, 'test_version': self.testVersion, 'software_name': 'ooniprobe', diff --git a/ooni/settings.py b/ooni/settings.py index c1071b9..0fed604 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -8,7 +8,7 @@ from os.path import abspath, expanduser

from twisted.internet import reactor, threads, defer

-from ooni import otime +from ooni import otime, geoip from ooni.utils import Storage

class OConfig(object): @@ -19,7 +19,7 @@ class OConfig(object): self.scapyFactory = None self.tor_state = None # This is used to store the probes IP address obtained via Tor - self.probe_ip = None + self.probe_ip = geoip.ProbeIP() # This is used to keep track of the state of the sniffer self.sniffer_running = None self.logging = True @@ -91,7 +91,7 @@ class OConfig(object): pass self.set_paths()

- def generatePcapFilename(self, testDetails): + def generate_pcap_filename(self, testDetails): test_name, start_time = testDetails['test_name'], testDetails['start_time'] start_time = otime.epochToTimestamp(start_time) return "report-%s-%s.%s" % (test_name, start_time, "pcap") diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 379b5c7..773673b 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -9,7 +9,6 @@ from twisted.python.failure import Failure from twisted.python.logfile import DailyLogFile

from ooni import otime -from ooni.settings import config

## Get rid of the annoying "No route found for ## IPv6 destination warnings": @@ -26,6 +25,8 @@ class LogWithNoPrefix(txlog.FileLogObserver):

class OONILogger(object): def start(self, logfile=None, application_name="ooniprobe"): + from ooni.settings import config + daily_logfile = None

if not logfile: @@ -58,14 +59,17 @@ def stop(): oonilogger.stop()

def msg(msg, *arg, **kw): + from ooni.settings import config if config.logging: print "%s" % msg

def debug(msg, *arg, **kw): + from ooni.settings import config if config.advanced.debug and config.logging: print "[D] %s" % msg

def err(msg, *arg, **kw): + from ooni.settings import config if config.logging: print "[!] %s" % msg