commit 1247a80ec046e9c3b132ca6b23f602ca852e6b1a Author: Arturo Filastò art@fuffa.org Date: Wed Feb 27 18:12:13 2013 +0100

Make the starting of tests more robust. --- ooni/nettest.py | 11 +++++++---- ooni/oonicli.py | 18 +++++++++++++----- ooni/utils/geodata.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 9 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py index 7108cd1..0260ca2 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -39,7 +39,11 @@ class NetTestLoader(object): config.privacy.includecountry or \ config.privacy.includecity): log.msg("We will include some geo data in the report") - client_geodata = geoip.IPToLocation(config.probe_ip.address) + try: + client_geodata = geodata.IPToLocation(config.probe_ip) + except e.GeoIPDataFilesNotFound: + log.err("Unable to find the geoip data files") + client_geodata = {'city': None, 'countrycode': None, 'asn': None}

if config.privacy.includeip: client_geodata['ip'] = config.probe_ip.address @@ -52,8 +56,7 @@ class NetTestLoader(object): client_geodata['asn'] = 'AS0' elif 'asn' in client_geodata: # XXX this regexp should probably go inside of geodata - client_geodata['asn'] = \ - re.search('AS\d+', client_geodata['asn']).group(0) + client_geodata['asn'] = client_geodata['asn'] log.msg("Your AS number is: %s" % client_geodata['asn']) else: client_geodata['asn'] = None @@ -307,7 +310,7 @@ class NetTest(object): self.state.taskDone()

if len(self.report.reporters) == 0: - raise AllReportersFailed + raise e.AllReportersFailed

return report_results

diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 73d7709..93bf8bf 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -35,8 +35,8 @@ class Options(usage.Options): optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], - ["collector", "c", None, - "Address of the collector of test results. (example: http://127.0.0.1:8888)"], + ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion', + "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"], ["logfile", "l", None, "log file name"], ["pcapfile", "O", None, "pcap file name"], ["parallelism", "p", "10", "input parallelism"], @@ -130,15 +130,22 @@ def runWithDirector(): director = Director() d = director.start()

+ def director_startup_failed(failure): + log.err("Failed to start the director") + log.exception(failure) + reactor.stop() + # Wait until director has started up (including bootstrapping Tor) before adding tess def post_director_start(_): for net_test_loader in test_list: - yaml_reporter = YAMLReporter(net_test_loader.testDetails) + test_details = net_test_loader.testDetails + + yaml_reporter = YAMLReporter(test_details) reporters = [yaml_reporter]

if global_options['collector']: try: - oonib_reporter = OONIBReporter(net_test_loader.testDetails, + oonib_reporter = OONIBReporter(test_details, global_options['collector']) reporters.append(oonib_reporter) except InvalidOONIBCollectorAddress: @@ -152,7 +159,7 @@ def runWithDirector(): with open('collector') as f: reporter_url = random.choice(f.readlines()) reporter_url = reporter_url.split('#')[0].strip() - oonib_reporter = OONIBReporter(net_test_loader.testDetails, reporter_url) + oonib_reporter = OONIBReporter(test_details, reporter_url) reporters.append(oonib_reporter)

log.debug("adding callback for startNetTest") @@ -160,4 +167,5 @@ def runWithDirector(): d.addCallback(shutdown)

d.addCallback(post_director_start) + d.addErrback(director_startup_failed) reactor.run() diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py new file mode 100644 index 0000000..56ce05c --- /dev/null +++ b/ooni/utils/geodata.py @@ -0,0 +1,39 @@ +import re +import os + +from twisted.web.client import Agent +from twisted.internet import reactor, defer, protocol + +from ooni.utils import log, net +from ooni import config +from ooni.errors import GeoIPDataFilesNotFound + +try: + import pygeoip +except ImportError: + log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") + +def IPToLocation(ipaddr): + city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') + country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') + asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') + + location = {'city': None, 'countrycode': None, 'asn': None} + try: + city_dat = pygeoip.GeoIP(city_file) + location['city'] = city_dat.record_by_addr(ipaddr)['city'] + + country_dat = pygeoip.GeoIP(country_file) + location['countrycode'] = country_dat.country_code_by_addr(ipaddr) + + asn_dat = pygeoip.GeoIP(asn_file) + asn = asn_dat.org_by_addr(ipaddr) + location['asn'] = re.search('AS\d+', asn).group(0) + + except IOError: + log.err("Could not find GeoIP data files. Go into data/ " + "and run make geoip") + raise GeoIPDataFilesNotFound + + return location +