commit 0fd667f224acb19dbb925ca17aaec20b4e351519 Author: aagbsn aagbsn@extc.org Date: Fri Jun 28 01:00:08 2013 +0200

Make oonicli download inputs before starting tests. --- ooni/oonicli.py | 98 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 59 insertions(+), 39 deletions(-)

diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 218ec2e..2a141f6 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -14,6 +14,7 @@ from ooni import errors

from ooni.settings import config from ooni.director import Director +from ooni.deck import TestDeck from ooni.reporter import YAMLReporter, OONIBReporter from ooni.nettest import NetTestLoader, MissingRequiredOption

@@ -109,42 +110,43 @@ def runWithDirector(): config.read_config_file()

log.start(global_options['logfile']) - # contains (test_cases, options, cmd_line_options) - test_list = [] - if global_options['no-collector']: - log.msg("Not reporting using a collector") - global_options['collector'] = None - - if global_options['testdeck']: - test_deck = yaml.safe_load(open(global_options['testdeck'])) - for test in test_deck: - test_list.append(NetTestLoader(test['options']['subargs'], - test_file=test['options']['test_file'])) - else: - log.debug("No test deck detected") - test_list.append(NetTestLoader(global_options['subargs'], - test_file=global_options['test_file'])) - - # check each test's usageOptions - for net_test_loader in test_list: - try: - net_test_loader.checkOptions() - except MissingRequiredOption, option_name: - log.err('Missing required option: "%s"' % option_name) - print net_test_loader.usageOptions().getUsage() - sys.exit(2) - except usage.UsageError, e: - log.err(e) - print net_test_loader.usageOptions().getUsage() - sys.exit(2)

director = Director() d = director.start()

+ test_deck = TestDeck() + if global_options['no-collector']: + log.msg("Not reporting using a collector") + collector = global_options['collector'] = None + + try: + if global_options['testdeck']: + test_deck.loadDeck(global_options['testdeck']) + else: + log.debug("No test deck detected") + net_test_loader = NetTestLoader(global_options['subargs'], + test_file=global_options['test_file']) + test_deck.insert(net_test_loader) + except MissingRequiredOption, option_name: + log.err('Missing required option: "%s"' % option_name) + print net_test_loader.usageOptions().getUsage() + sys.exit(2) + except usage.UsageError, e: + log.err(e) + print net_test_loader.usageOptions().getUsage() + sys.exit(2) + + def fetch_nettest_inputs(result): + try: + test_deck.fetchAndVerifyDeckInputs() + except errors.UnableToLoadDeckInput, e: + return defer.failure.Failure(result) + def director_startup_failed(failure): log.err("Failed to start the director") r = failure.trap(errors.TorNotRunning, - errors.InvalidOONIBCollectorAddress) + errors.InvalidOONIBCollectorAddress, + errors.UnableToLoadDeckInput) if r == errors.TorNotRunning: log.err("Tor does not appear to be running") log.err("Reporting with the collector %s is not possible" % @@ -154,25 +156,42 @@ def runWithDirector(): log.err("Invalid format for oonib collector address.") log.msg("Should be in the format http://<collector_address>:<port>") log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") + elif r == errors.UnableToLoadDeckInput: + log.err('Missing required input files: %s' % failure) reactor.stop()

- # Wait until director has started up (including bootstrapping Tor) before adding tess + # Wait until director has started up (including bootstrapping Tor) + # before adding tests def post_director_start(_): - for net_test_loader in test_list: - collector = global_options['collector'] - test_details = net_test_loader.testDetails - - yaml_reporter = YAMLReporter(test_details) - reporters = [yaml_reporter] + for net_test_loader in test_deck.netTestLoaders: + # TestDecks can specify different collectors + # for each net test, so that each NetTest + # may be paired with a test_helper and its collector + # However, a user can override this behavior by + # specifying a collector from the command-line (-c). + # If a collector is not specified in the deck, or the + # deck is a singleton, the default collector set in + # ooniprobe.conf will be used + + collector = None + if not global_options['no-collector']: + if global_options['collector']: + collector = global_options['collector'] + elif net_test_loader.options['collector']: + collector = net_test_loader.options['collector']

if collector and collector.startswith('httpo:') \ and (not (config.tor_state or config.tor.socks_port)): raise errors.TorNotRunning - elif collector: + + test_details = net_test_loader.testDetails + yaml_reporter = YAMLReporter(test_details) + reporters = [yaml_reporter] + + if collector: log.msg("Reporting using collector: %s" % collector) try: - oonib_reporter = OONIBReporter(test_details, - collector) + oonib_reporter = OONIBReporter(test_details, collector) reporters.append(oonib_reporter) except errors.InvalidOONIBCollectorAddress, e: raise e @@ -184,6 +203,7 @@ def runWithDirector(): director.allTestsDone.addBoth(shutdown)

def start(): + d.addCallback(fetch_nettest_inputs) d.addCallback(post_director_start) d.addErrback(director_startup_failed)