commit eb46b078184500efd2024d403170047e64b272de Author: Arturo Filastò <art@fuffa.org> Date: Sun Nov 25 01:48:51 2012 +0100 Refactoring on runner, reporter and oonicli * Include the command line arguments in the test report --- ooni/config.py | 32 ++++++++++++++++++-------------- ooni/oonicli.py | 35 +++++++++++++++-------------------- ooni/reporter.py | 24 +++++++++++++++++++----- ooni/runner.py | 40 +++++++++++++++++----------------------- ooniprobe.conf | 2 +- 5 files changed, 70 insertions(+), 63 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index ef44ee4..5168c34 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -11,6 +11,9 @@ from twisted.internet import reactor, threads from ooni.utils import otime from ooni.utils import Storage +reports = Storage() +basic = None +cmd_line_options = None def get_root_path(): this_directory = os.path.dirname(__file__) @@ -18,19 +21,6 @@ def get_root_path(): root = os.path.abspath(root) return root -def oreport_filenames(file_name): - """ - returns the filenames for the pcap file and the yamloo report - - returns - yamloo_filename, pcap_filename - """ - test_name = '.'.join(file_name.split(".")[:-1]) - base_filename = "%s_%s_"+otime.timestamp()+".%s" - yamloo_filename = base_filename % (test_name, "report", "yamloo") - pcap_filename = base_filename % (test_name, "packets", "pcap") - return yamloo_filename, pcap_filename - def loadConfigFile(): """ This is a helper function that makes sure that the configuration attributes @@ -62,7 +52,21 @@ def loadConfigFile(): advanced[k] = v return basic, privacy, advanced -basic = None +class TestFilenameNotSet(Exception): + pass + +def generateReportFilenames(): + try: + test_file_name = os.path.basename(cmd_line_options['test']) + except IndexError: + raise TestFilenameNotSet + + test_name = '.'.join(test_file_name.split(".")[:-1]) + base_filename = "%s_%s_"+otime.timestamp()+".%s" + print "Setting yamloo to %s" % base_filename + reports.yamloo = base_filename % (test_name, "report", "yamloo") + reports.pcap = base_filename % (test_name, "packets", "pcap") + if not basic: # Here we make sure that we instance the config file attributes only once basic, privacy, advanced = loadConfigFile() diff --git a/ooni/oonicli.py b/ooni/oonicli.py index c8385ed..41a31f6 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -70,7 +70,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): try: self['test'] = args[0] - self['subArgs'] = args[1:] + self['subargs'] = args[1:] except: raise usage.UsageError("No test filename specified!") @@ -95,24 +95,19 @@ def run(): if cmd_line_options['debug-stacktraces']: defer.setDebugging(True) - test_file_name = os.path.basename(cmd_line_options['test']) - - yamloo_filename, pcap_filename = config.oreport_filenames(test_file_name) + config.cmd_line_options = cmd_line_options + config.generateReportFilenames() if cmd_line_options['reportfile']: - yamloo_filename = cmd_line_options['reportfile'] - pcap_filename = yamloo_filename+".pcap" - - if os.path.exists(yamloo_filename): - print "Report already exists with filename %s" % yamloo_filename - print "Renaming it to %s" % yamloo_filename+'.old' - os.rename(yamloo_filename, yamloo_filename+'.old') - if os.path.exists(pcap_filename): - print "Report PCAP already exists with filename %s" % pcap_filename - print "Renaming it to %s" % pcap_filename+'.old' - os.rename(pcap_filename, pcap_filename+'.old') - - classes = runner.findTestClassesFromConfig(cmd_line_options) + config.reports.yamloo = cmd_line_options['reportfile'] + config.reports.pcap = config.reports.yamloo+".pcap" + + if os.path.exists(config.reports.pcap): + print "Report PCAP already exists with filename %s" % config.reports.pcap + print "Renaming it to %s" % config.reports.pcap+'.old' + os.rename(config.reports.pcap, config.reports.pcap+'.old') + + classes = runner.findTestClassesFromFile(cmd_line_options['test']) test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options) if config.privacy.includepcap: try: @@ -121,13 +116,13 @@ def run(): print "[!] Includepcap options requires root priviledges to run" print " you should run ooniprobe as root or disable the options in ooniprobe.conf" sys.exit(1) + print "Starting sniffer" - net.capturePackets(pcap_filename) + net.capturePackets(config.reports.pcap) log.start(cmd_line_options['logfile']) - tests_d = runner.runTestCases(test_cases, options, - cmd_line_options, yamloo_filename) + tests_d = runner.runTestCases(test_cases, options, cmd_line_options) tests_d.addBoth(testsEnded) reactor.run() diff --git a/ooni/reporter.py b/ooni/reporter.py index 1e2924d..99b7f7a 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -10,6 +10,7 @@ import itertools import logging import sys +import os import time import yaml import json @@ -155,6 +156,9 @@ def getTestDetails(options): defer.returnValue(test_details) class OReporter(object): + def __init__(self, cmd_line_options): + self.cmd_line_options = dict(cmd_line_options) + def createReport(self, options): """ Override this with your own logic to implement tests. @@ -172,7 +176,6 @@ class OReporter(object): def testDone(self, test, test_name): log.msg("Finished running %s" % test_name) - log.debug("Writing report") test_report = dict(test.report) if isinstance(test.input, Packet): @@ -204,8 +207,15 @@ class YAMLReporter(OReporter): """ These are useful functions for reporting to YAML format. """ - def __init__(self, stream): - self._stream = stream + def __init__(self, cmd_line_options): + if os.path.exists(config.reports.yamloo): + log.msg("Report already exists with filename %s" % config.reports.yamloo) + log.msg("Renaming it to %s" % config.reports.yamloo+'.old') + os.rename(config.reports.yamloo, config.reports.yamloo+'.old') + + log.debug("Creating %s" % config.reports.yamloo) + self._stream = open(config.reports.yamloo, 'w+') + OReporter.__init__(self, cmd_line_options) def _writeln(self, line): self._write("%s\n" % line) @@ -233,6 +243,7 @@ class YAMLReporter(OReporter): self._writeln("###########################################") test_details = yield getTestDetails(options) + test_details['options'] = self.cmd_line_options self.writeReportEntry(test_details) @@ -250,7 +261,9 @@ class OONIBTestDetailsLookupFailed(Exception): pass class OONIBReporter(OReporter): - def __init__(self, backend_url): + def __init__(self, cmd_line_options): + self.backend_url = cmd_line_options['collector'] + from ooni.utils.txagentwithsocks import Agent from twisted.internet import reactor try: @@ -259,7 +272,7 @@ class OONIBReporter(OReporter): except Exception, e: log.exception(e) - self.backend_url = backend_url + OReporter.__init__(self, cmd_line_options) @defer.inlineCallbacks def writeReportEntry(self, entry): @@ -306,6 +319,7 @@ class OONIBReporter(OReporter): software_version = '0.0.1' test_details = yield getTestDetails(options) + test_details['options'] = self.cmd_line_options content = '---\n' content += safe_dump(test_details) diff --git a/ooni/runner.py b/ooni/runner.py index 92aa406..5e8536c 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -23,11 +23,11 @@ from twisted.internet import reactor, threads from ooni.inputunit import InputUnitFactory from ooni.nettest import NetTestCase, NoPostProcessor -from ooni import reporter +from ooni import reporter, config from ooni.utils import log, checkForRoot, NotRootError -def processTest(obj, cmd_line_options): +def processTest(obj): """ Process the parameters and :class:`twisted.python.usage.Options` of a :class:`ooni.nettest.Nettest`. @@ -57,7 +57,7 @@ def processTest(obj, cmd_line_options): options = obj.usageOptions() - options.parseOptions(cmd_line_options['subArgs']) + options.parseOptions(config.cmd_line_options['subargs']) obj.localOptions = options if obj.inputFile: @@ -73,7 +73,7 @@ def processTest(obj, cmd_line_options): log.err("There was an error in running %s!" % test_name) log.err("%s" % e) options.opt_help() - raise usage.UsageError("Error in parsing command line args for %s" % test_name) + raise usage.UsageError("Error in parsing command line args for %s" % test_name) if obj.requiresRoot: try: @@ -90,28 +90,23 @@ def isTestCase(obj): except TypeError: return False -def findTestClassesFromConfig(cmd_line_options): +def findTestClassesFromFile(filename): """ Takes as input the command line config parameters and returns the test case classes. - If it detects that a certain test class is using the old OONIProbe format, - then it will adapt it to the new testing system. - :param cmd_line_options: - A configured and instantiated :class:`twisted.python.usage.Options` - class. + :param filename: + the absolute path to the file containing the ooniprobe test classes + :return: A list of class objects found in a file or module given on the commandline. """ - - filename = cmd_line_options['test'] classes = [] - module = filenameToModule(filename) for name, val in inspect.getmembers(module): if isTestCase(val): - classes.append(processTest(val, cmd_line_options)) + classes.append(processTest(val)) return classes def makeTestCases(klass, tests, method_prefix): @@ -227,8 +222,10 @@ def runTestCasesWithInputUnit(test_cases, input_unit, oreporter): return defer.DeferredList(dl) @defer.inlineCallbacks -def runTestCases(test_cases, options, - cmd_line_options, yamloo_filename): +def runTestCases(test_cases, options, cmd_line_options): + log.debug("Running %s" % test_cases) + log.debug("Options %s" % options) + log.debug("cmd_line_options %s" % dict(cmd_line_options)) try: assert len(options) != 0, "Length of options is zero!" except AssertionError, ae: @@ -247,15 +244,12 @@ def runTestCases(test_cases, options, log.msg("options[0] = %s" % first) test_inputs = [None] - log.debug("Creating %s" % yamloo_filename) - if cmd_line_options['collector']: - log.debug("Using remote collector %s" % cmd_line_options['collector']) - oreporter = reporter.OONIBReporter(cmd_line_options['collector']) + log.debug("Using remote collector") + oreporter = reporter.OONIBReporter(cmd_line_options) else: - reportFile = open(yamloo_filename, 'w+') - log.debug("Reporting to file %s" % reportFile) - oreporter = reporter.YAMLReporter(reportFile) + log.debug("Reporting to file %s" % config.reports.yamloo) + oreporter = reporter.YAMLReporter(cmd_line_options) try: input_unit_factory = InputUnitFactory(test_inputs) diff --git a/ooniprobe.conf b/ooniprobe.conf index ad761e1..e9f208f 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -22,7 +22,7 @@ advanced: # /path/to/ooni-probe/data/ #geoip_data_dir: /usr/share/GeoIP/ geoip_data_dir: /home/x/code/networking/ooni-probe/data/ - debug: false + debug: true threadpool_size: 10 tor_socksport: 9050 # For auto detection