[tor-commits] [ooni-probe/develop] Make the starting of tests more robust.

[isis at torproject.org]

commit 5abfe2a902b60062f08feb22ef5a47268e51c8b1
Author: Arturo Filastò <art at fuffa.org>
Date:   Wed Feb 27 18:12:13 2013 +0100

    Make the starting of tests more robust.
---
 ooni/nettest.py       |   11 +++++++----
 ooni/oonicli.py       |   18 +++++++++++++-----
 ooni/utils/geodata.py |    7 +++----
 3 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py
index 0b3e69c..1fe19f1 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -38,7 +38,11 @@ class NetTestLoader(object):
                 config.privacy.includecountry or \
                 config.privacy.includecity):
             log.msg("We will include some geo data in the report")
-            client_geodata = geodata.IPToLocation(config.probe_ip)
+            try:
+                client_geodata = geodata.IPToLocation(config.probe_ip)
+            except e.GeoIPDataFilesNotFound:
+                log.err("Unable to find the geoip data files")
+                client_geodata = {'city': None, 'countrycode': None, 'asn': None}
 
         if config.privacy.includeip:
             client_geodata['ip'] = config.probe_ip
@@ -51,8 +55,7 @@ class NetTestLoader(object):
             client_geodata['asn'] = 'AS0'
         elif 'asn' in client_geodata:
             # XXX this regexp should probably go inside of geodata
-            client_geodata['asn'] = \
-                    re.search('AS\d+', client_geodata['asn']).group(0)
+            client_geodata['asn'] = client_geodata['asn']
             log.msg("Your AS number is: %s" % client_geodata['asn'])
         else:
             client_geodata['asn'] = None
@@ -306,7 +309,7 @@ class NetTest(object):
         self.state.taskDone()
 
         if len(self.report.reporters) == 0:
-            raise AllReportersFailed
+            raise e.AllReportersFailed
 
         return report_results
 
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 73d7709..93bf8bf 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -35,8 +35,8 @@ class Options(usage.Options):
     optParameters = [["reportfile", "o", None, "report file name"],
                      ["testdeck", "i", None,
                          "Specify as input a test deck: a yaml file containig the tests to run an their arguments"],
-                     ["collector", "c", None,
-                         "Address of the collector of test results. (example: http://127.0.0.1:8888)"],
+                     ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion',
+                         "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"],
                      ["logfile", "l", None, "log file name"],
                      ["pcapfile", "O", None, "pcap file name"],
                      ["parallelism", "p", "10", "input parallelism"],
@@ -130,15 +130,22 @@ def runWithDirector():
     director = Director()
     d = director.start()
 
+    def director_startup_failed(failure):
+        log.err("Failed to start the director")
+        log.exception(failure)
+        reactor.stop()
+
     # Wait until director has started up (including bootstrapping Tor) before adding tess
     def post_director_start(_):
         for net_test_loader in test_list:
-            yaml_reporter = YAMLReporter(net_test_loader.testDetails)
+            test_details = net_test_loader.testDetails
+
+            yaml_reporter = YAMLReporter(test_details)
             reporters = [yaml_reporter]
 
             if global_options['collector']:
                 try:
-                    oonib_reporter = OONIBReporter(net_test_loader.testDetails,
+                    oonib_reporter = OONIBReporter(test_details,
                             global_options['collector'])
                     reporters.append(oonib_reporter)
                 except InvalidOONIBCollectorAddress:
@@ -152,7 +159,7 @@ def runWithDirector():
                 with open('collector') as f:
                     reporter_url = random.choice(f.readlines())
                     reporter_url = reporter_url.split('#')[0].strip()
-                    oonib_reporter = OONIBReporter(net_test_loader.testDetails, reporter_url)
+                    oonib_reporter = OONIBReporter(test_details, reporter_url)
                     reporters.append(oonib_reporter)
 
             log.debug("adding callback for startNetTest")
@@ -160,4 +167,5 @@ def runWithDirector():
         d.addCallback(shutdown)
 
     d.addCallback(post_director_start)
+    d.addErrback(director_startup_failed)
     reactor.run()
diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py
index d9883ba..56ce05c 100644
--- a/ooni/utils/geodata.py
+++ b/ooni/utils/geodata.py
@@ -6,15 +6,13 @@ from twisted.internet import reactor, defer, protocol
 
 from ooni.utils import log, net
 from ooni import config
+from ooni.errors import GeoIPDataFilesNotFound
 
 try:
     import pygeoip
 except ImportError:
     log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements")
 
-class GeoIPDataFilesNotFound(Exception):
-    pass
-
 def IPToLocation(ipaddr):
     city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
     country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
@@ -29,7 +27,8 @@ def IPToLocation(ipaddr):
         location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
 
         asn_dat = pygeoip.GeoIP(asn_file)
-        location['asn'] = asn_dat.org_by_addr(ipaddr)
+        asn = asn_dat.org_by_addr(ipaddr)
+        location['asn'] = re.search('AS\d+', asn).group(0)
 
     except IOError:
         log.err("Could not find GeoIP data files. Go into data/ "