commit 5abfe2a902b60062f08feb22ef5a47268e51c8b1 Author: Arturo Filastò art@fuffa.org Date: Wed Feb 27 18:12:13 2013 +0100

Make the starting of tests more robust. --- ooni/nettest.py | 11 +++++++---- ooni/oonicli.py | 18 +++++++++++++----- ooni/utils/geodata.py | 7 +++---- 3 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py index 0b3e69c..1fe19f1 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -38,7 +38,11 @@ class NetTestLoader(object): config.privacy.includecountry or \ config.privacy.includecity): log.msg("We will include some geo data in the report") - client_geodata = geodata.IPToLocation(config.probe_ip) + try: + client_geodata = geodata.IPToLocation(config.probe_ip) + except e.GeoIPDataFilesNotFound: + log.err("Unable to find the geoip data files") + client_geodata = {'city': None, 'countrycode': None, 'asn': None}

if config.privacy.includeip: client_geodata['ip'] = config.probe_ip @@ -51,8 +55,7 @@ class NetTestLoader(object): client_geodata['asn'] = 'AS0' elif 'asn' in client_geodata: # XXX this regexp should probably go inside of geodata - client_geodata['asn'] = \ - re.search('AS\d+', client_geodata['asn']).group(0) + client_geodata['asn'] = client_geodata['asn'] log.msg("Your AS number is: %s" % client_geodata['asn']) else: client_geodata['asn'] = None @@ -306,7 +309,7 @@ class NetTest(object): self.state.taskDone()

if len(self.report.reporters) == 0: - raise AllReportersFailed + raise e.AllReportersFailed

return report_results

diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 73d7709..93bf8bf 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -35,8 +35,8 @@ class Options(usage.Options): optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], - ["collector", "c", None, - "Address of the collector of test results. (example: http://127.0.0.1:8888)"], + ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion', + "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"], ["logfile", "l", None, "log file name"], ["pcapfile", "O", None, "pcap file name"], ["parallelism", "p", "10", "input parallelism"], @@ -130,15 +130,22 @@ def runWithDirector(): director = Director() d = director.start()

+ def director_startup_failed(failure): + log.err("Failed to start the director") + log.exception(failure) + reactor.stop() + # Wait until director has started up (including bootstrapping Tor) before adding tess def post_director_start(_): for net_test_loader in test_list: - yaml_reporter = YAMLReporter(net_test_loader.testDetails) + test_details = net_test_loader.testDetails + + yaml_reporter = YAMLReporter(test_details) reporters = [yaml_reporter]

if global_options['collector']: try: - oonib_reporter = OONIBReporter(net_test_loader.testDetails, + oonib_reporter = OONIBReporter(test_details, global_options['collector']) reporters.append(oonib_reporter) except InvalidOONIBCollectorAddress: @@ -152,7 +159,7 @@ def runWithDirector(): with open('collector') as f: reporter_url = random.choice(f.readlines()) reporter_url = reporter_url.split('#')[0].strip() - oonib_reporter = OONIBReporter(net_test_loader.testDetails, reporter_url) + oonib_reporter = OONIBReporter(test_details, reporter_url) reporters.append(oonib_reporter)

log.debug("adding callback for startNetTest") @@ -160,4 +167,5 @@ def runWithDirector(): d.addCallback(shutdown)

d.addCallback(post_director_start) + d.addErrback(director_startup_failed) reactor.run() diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index d9883ba..56ce05c 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -6,15 +6,13 @@ from twisted.internet import reactor, defer, protocol

from ooni.utils import log, net from ooni import config +from ooni.errors import GeoIPDataFilesNotFound

try: import pygeoip except ImportError: log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements")

-class GeoIPDataFilesNotFound(Exception): - pass - def IPToLocation(ipaddr): city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') @@ -29,7 +27,8 @@ def IPToLocation(ipaddr): location['countrycode'] = country_dat.country_code_by_addr(ipaddr)

asn_dat = pygeoip.GeoIP(asn_file) - location['asn'] = asn_dat.org_by_addr(ipaddr) + asn = asn_dat.org_by_addr(ipaddr) + location['asn'] = re.search('AS\d+', asn).group(0)

except IOError: log.err("Could not find GeoIP data files. Go into data/ "