commit 263568f28bcac776da7f9936a0e80958c4027f46 Author: Arturo Filastò <art@fuffa.org> Date: Sun Nov 25 02:43:10 2012 +0100 Implement test deck system for automating the specification of command line arguments for tests * Change before_i_commit script to use test deck system --- before_i_commit.sh | 13 ++------- before_i_commit.testdeck | 40 ++++++++++++++++++++++++++++ ooni/oonicli.py | 65 +++++++++++++++++++++++++++++----------------- ooni/reporter.py | 2 - ooni/runner.py | 2 - 5 files changed, 84 insertions(+), 38 deletions(-) diff --git a/before_i_commit.sh b/before_i_commit.sh index 5147149..8b8180f 100755 --- a/before_i_commit.sh +++ b/before_i_commit.sh @@ -14,15 +14,7 @@ rm before_i_commit.log find . -type f -name "*.py[co]" -delete -./bin/ooniprobe -l before_i_commit.log -o http_url_lists.yamloo nettests/core/http_url_list.py -f test_inputs/url_lists_file.txt - -./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -T test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt - -./bin/ooniprobe -l before_i_commit.log -o captive_portal_test.yamloo nettests/core/captiveportal.py - -./bin/ooniprobe -l before_i_commit.log -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt - -./bin/ooniprobe -l before_i_commit.log -o http_keyword_filtering.yamloo nettests/core/http_keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt +./bin/ooniprobe -i before_i_commit.testdeck echo "Below you should not see anything" echo "---------------------------------" @@ -32,5 +24,6 @@ echo "If you do, it means something is wrong." echo "Read through the log file and fix it." echo "If you are having some problems fixing some things that have to do with" echo "the core of OONI, let's first discuss it on IRC, or open a ticket" -less *yamloo +read +cat *yamloo | less rm -f *yamloo diff --git a/before_i_commit.testdeck b/before_i_commit.testdeck new file mode 100644 index 0000000..e0b30ea --- /dev/null +++ b/before_i_commit.testdeck @@ -0,0 +1,40 @@ +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: captive_portal_test.yamloo + subargs: [] + test: nettests/core/captiveportal.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: dns_tamper_test.yamloo + subargs: [-T, test_inputs/dns_tamper_test_resolvers.txt, -f, test_inputs/dns_tamper_file.txt] + test: nettests/core/dnstamper.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: http_host.yamloo + subargs: [-b, 'http://ooni.nu/test', -f, test_inputs/http_host_file.txt] + test: nettests/core/http_host.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: http_keyword_filtering.yamloo + subargs: [-b, 'http://ooni.nu/test/', -f, test_inputs/keyword_filtering_file.txt] + test: nettests/core/http_keyword_filtering.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: http_url_lists.yamloo + subargs: [-f, test_inputs/url_lists_file.txt] + test: nettests/core/http_url_list.py diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 41a31f6..5c582b2 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -13,6 +13,7 @@ import sys import os import random import time +import yaml from twisted.internet import defer, reactor from twisted.application import app @@ -27,7 +28,7 @@ from ooni.utils import net from ooni.utils import checkForRoot, NotRootError from ooni.utils import log -class Options(usage.Options, app.ReactorSelectionMixin): +class Options(usage.Options): synopsis = """%s [options] [path to test].py """ % (os.path.basename(sys.argv[0]),) @@ -35,12 +36,12 @@ class Options(usage.Options, app.ReactorSelectionMixin): " network tests. These are loaded from modules, packages and" " files listed on the command line") - optFlags = [["help", "h"], - ['debug-stacktraces', 'B', - 'Report deferred creation and callback stack traces'],] + optFlags = [["help", "h"]] optParameters = [["reportfile", "o", None, "report file name"], - ["collector", "c", None, + ["testdeck", "i", None, + "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], + ["collector", "c", None, "Address of the collector of test results. (example: http://127.0.0.1:8888)"], ["logfile", "l", None, "log file name"], ["pcapfile", "p", None, "pcap file name"]] @@ -68,6 +69,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): sys.settrace(spewer) def parseArgs(self, *args): + if self['testdeck']: + return try: self['test'] = args[0] self['subargs'] = args[1:] @@ -79,22 +82,9 @@ def testsEnded(*arg, **kw): You can place here all the post shutdown tasks. """ log.debug("testsEnded: Finished running all tests") + reactor.stop() -def run(): - """ - Call me to begin testing from a file. - """ - cmd_line_options = Options() - if len(sys.argv) == 1: - cmd_line_options.getUsage() - try: - cmd_line_options.parseOptions() - except usage.UsageError, ue: - raise SystemExit, "%s: %s" % (sys.argv[0], ue) - - if cmd_line_options['debug-stacktraces']: - defer.setDebugging(True) - +def runTest(cmd_line_options): config.cmd_line_options = cmd_line_options config.generateReportFilenames() @@ -120,10 +110,37 @@ def run(): print "Starting sniffer" net.capturePackets(config.reports.pcap) - log.start(cmd_line_options['logfile']) + return runner.runTestCases(test_cases, options, cmd_line_options) - tests_d = runner.runTestCases(test_cases, options, cmd_line_options) - tests_d.addBoth(testsEnded) +def run(): + """ + Call me to begin testing from a file. + """ + cmd_line_options = Options() + if len(sys.argv) == 1: + cmd_line_options.getUsage() + try: + cmd_line_options.parseOptions() + except usage.UsageError, ue: + raise SystemExit, "%s: %s" % (sys.argv[0], ue) - reactor.run() + deck_dl = [] + log.start(cmd_line_options['logfile']) + if cmd_line_options['testdeck']: + test_deck = yaml.safe_load(open(cmd_line_options['testdeck'])) + for test in test_deck: + del cmd_line_options + cmd_line_options = test['options'] + d1 = runTest(cmd_line_options) + deck_dl.append(d1) + else: + log.msg("No test deck detected") + del cmd_line_options['testdeck'] + d1 = runTest(cmd_line_options) + deck_dl.append(d1) + + d2 = defer.DeferredList(deck_dl) + d2.addCallback(testsEnded) + + reactor.run() diff --git a/ooni/reporter.py b/ooni/reporter.py index 99b7f7a..26c6e43 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -197,10 +197,8 @@ class OReporter(object): log.debug("allDone: Finished running all tests") try: log.debug("Stopping the reactor") - reactor.stop() except: log.debug("Unable to stop the reactor") - pass return None class YAMLReporter(OReporter): diff --git a/ooni/runner.py b/ooni/runner.py index 5e8536c..bc271a6 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -262,7 +262,6 @@ def runTestCases(test_cases, options, cmd_line_options): yield oreporter.createReport(options) except reporter.OONIBReportCreationFailed: log.err("Error in creating new report") - reactor.stop() raise except Exception, e: log.exception(e) @@ -280,6 +279,5 @@ def runTestCases(test_cases, options, cmd_line_options): except Exception: log.exception("Problem in running test") - reactor.stop() oreporter.allDone()