[tor-commits] [ooni-probe/master] Refactoring on runner, reporter and oonicli
art at torproject.org
art at torproject.org
Sun Nov 25 01:44:02 UTC 2012
commit eb46b078184500efd2024d403170047e64b272de
Author: Arturo Filastò <art at fuffa.org>
Date: Sun Nov 25 01:48:51 2012 +0100
Refactoring on runner, reporter and oonicli
* Include the command line arguments in the test report
---
ooni/config.py | 32 ++++++++++++++++++--------------
ooni/oonicli.py | 35 +++++++++++++++--------------------
ooni/reporter.py | 24 +++++++++++++++++++-----
ooni/runner.py | 40 +++++++++++++++++-----------------------
ooniprobe.conf | 2 +-
5 files changed, 70 insertions(+), 63 deletions(-)
diff --git a/ooni/config.py b/ooni/config.py
index ef44ee4..5168c34 100644
--- a/ooni/config.py
+++ b/ooni/config.py
@@ -11,6 +11,9 @@ from twisted.internet import reactor, threads
from ooni.utils import otime
from ooni.utils import Storage
+reports = Storage()
+basic = None
+cmd_line_options = None
def get_root_path():
this_directory = os.path.dirname(__file__)
@@ -18,19 +21,6 @@ def get_root_path():
root = os.path.abspath(root)
return root
-def oreport_filenames(file_name):
- """
- returns the filenames for the pcap file and the yamloo report
-
- returns
- yamloo_filename, pcap_filename
- """
- test_name = '.'.join(file_name.split(".")[:-1])
- base_filename = "%s_%s_"+otime.timestamp()+".%s"
- yamloo_filename = base_filename % (test_name, "report", "yamloo")
- pcap_filename = base_filename % (test_name, "packets", "pcap")
- return yamloo_filename, pcap_filename
-
def loadConfigFile():
"""
This is a helper function that makes sure that the configuration attributes
@@ -62,7 +52,21 @@ def loadConfigFile():
advanced[k] = v
return basic, privacy, advanced
-basic = None
+class TestFilenameNotSet(Exception):
+ pass
+
+def generateReportFilenames():
+ try:
+ test_file_name = os.path.basename(cmd_line_options['test'])
+ except IndexError:
+ raise TestFilenameNotSet
+
+ test_name = '.'.join(test_file_name.split(".")[:-1])
+ base_filename = "%s_%s_"+otime.timestamp()+".%s"
+ print "Setting yamloo to %s" % base_filename
+ reports.yamloo = base_filename % (test_name, "report", "yamloo")
+ reports.pcap = base_filename % (test_name, "packets", "pcap")
+
if not basic:
# Here we make sure that we instance the config file attributes only once
basic, privacy, advanced = loadConfigFile()
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index c8385ed..41a31f6 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -70,7 +70,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
def parseArgs(self, *args):
try:
self['test'] = args[0]
- self['subArgs'] = args[1:]
+ self['subargs'] = args[1:]
except:
raise usage.UsageError("No test filename specified!")
@@ -95,24 +95,19 @@ def run():
if cmd_line_options['debug-stacktraces']:
defer.setDebugging(True)
- test_file_name = os.path.basename(cmd_line_options['test'])
-
- yamloo_filename, pcap_filename = config.oreport_filenames(test_file_name)
+ config.cmd_line_options = cmd_line_options
+ config.generateReportFilenames()
if cmd_line_options['reportfile']:
- yamloo_filename = cmd_line_options['reportfile']
- pcap_filename = yamloo_filename+".pcap"
-
- if os.path.exists(yamloo_filename):
- print "Report already exists with filename %s" % yamloo_filename
- print "Renaming it to %s" % yamloo_filename+'.old'
- os.rename(yamloo_filename, yamloo_filename+'.old')
- if os.path.exists(pcap_filename):
- print "Report PCAP already exists with filename %s" % pcap_filename
- print "Renaming it to %s" % pcap_filename+'.old'
- os.rename(pcap_filename, pcap_filename+'.old')
-
- classes = runner.findTestClassesFromConfig(cmd_line_options)
+ config.reports.yamloo = cmd_line_options['reportfile']
+ config.reports.pcap = config.reports.yamloo+".pcap"
+
+ if os.path.exists(config.reports.pcap):
+ print "Report PCAP already exists with filename %s" % config.reports.pcap
+ print "Renaming it to %s" % config.reports.pcap+'.old'
+ os.rename(config.reports.pcap, config.reports.pcap+'.old')
+
+ classes = runner.findTestClassesFromFile(cmd_line_options['test'])
test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options)
if config.privacy.includepcap:
try:
@@ -121,13 +116,13 @@ def run():
print "[!] Includepcap options requires root priviledges to run"
print " you should run ooniprobe as root or disable the options in ooniprobe.conf"
sys.exit(1)
+
print "Starting sniffer"
- net.capturePackets(pcap_filename)
+ net.capturePackets(config.reports.pcap)
log.start(cmd_line_options['logfile'])
- tests_d = runner.runTestCases(test_cases, options,
- cmd_line_options, yamloo_filename)
+ tests_d = runner.runTestCases(test_cases, options, cmd_line_options)
tests_d.addBoth(testsEnded)
reactor.run()
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 1e2924d..99b7f7a 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -10,6 +10,7 @@
import itertools
import logging
import sys
+import os
import time
import yaml
import json
@@ -155,6 +156,9 @@ def getTestDetails(options):
defer.returnValue(test_details)
class OReporter(object):
+ def __init__(self, cmd_line_options):
+ self.cmd_line_options = dict(cmd_line_options)
+
def createReport(self, options):
"""
Override this with your own logic to implement tests.
@@ -172,7 +176,6 @@ class OReporter(object):
def testDone(self, test, test_name):
log.msg("Finished running %s" % test_name)
- log.debug("Writing report")
test_report = dict(test.report)
if isinstance(test.input, Packet):
@@ -204,8 +207,15 @@ class YAMLReporter(OReporter):
"""
These are useful functions for reporting to YAML format.
"""
- def __init__(self, stream):
- self._stream = stream
+ def __init__(self, cmd_line_options):
+ if os.path.exists(config.reports.yamloo):
+ log.msg("Report already exists with filename %s" % config.reports.yamloo)
+ log.msg("Renaming it to %s" % config.reports.yamloo+'.old')
+ os.rename(config.reports.yamloo, config.reports.yamloo+'.old')
+
+ log.debug("Creating %s" % config.reports.yamloo)
+ self._stream = open(config.reports.yamloo, 'w+')
+ OReporter.__init__(self, cmd_line_options)
def _writeln(self, line):
self._write("%s\n" % line)
@@ -233,6 +243,7 @@ class YAMLReporter(OReporter):
self._writeln("###########################################")
test_details = yield getTestDetails(options)
+ test_details['options'] = self.cmd_line_options
self.writeReportEntry(test_details)
@@ -250,7 +261,9 @@ class OONIBTestDetailsLookupFailed(Exception):
pass
class OONIBReporter(OReporter):
- def __init__(self, backend_url):
+ def __init__(self, cmd_line_options):
+ self.backend_url = cmd_line_options['collector']
+
from ooni.utils.txagentwithsocks import Agent
from twisted.internet import reactor
try:
@@ -259,7 +272,7 @@ class OONIBReporter(OReporter):
except Exception, e:
log.exception(e)
- self.backend_url = backend_url
+ OReporter.__init__(self, cmd_line_options)
@defer.inlineCallbacks
def writeReportEntry(self, entry):
@@ -306,6 +319,7 @@ class OONIBReporter(OReporter):
software_version = '0.0.1'
test_details = yield getTestDetails(options)
+ test_details['options'] = self.cmd_line_options
content = '---\n'
content += safe_dump(test_details)
diff --git a/ooni/runner.py b/ooni/runner.py
index 92aa406..5e8536c 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -23,11 +23,11 @@ from twisted.internet import reactor, threads
from ooni.inputunit import InputUnitFactory
from ooni.nettest import NetTestCase, NoPostProcessor
-from ooni import reporter
+from ooni import reporter, config
from ooni.utils import log, checkForRoot, NotRootError
-def processTest(obj, cmd_line_options):
+def processTest(obj):
"""
Process the parameters and :class:`twisted.python.usage.Options` of a
:class:`ooni.nettest.Nettest`.
@@ -57,7 +57,7 @@ def processTest(obj, cmd_line_options):
options = obj.usageOptions()
- options.parseOptions(cmd_line_options['subArgs'])
+ options.parseOptions(config.cmd_line_options['subargs'])
obj.localOptions = options
if obj.inputFile:
@@ -73,7 +73,7 @@ def processTest(obj, cmd_line_options):
log.err("There was an error in running %s!" % test_name)
log.err("%s" % e)
options.opt_help()
- raise usage.UsageError("Error in parsing command line args for %s" % test_name)
+ raise usage.UsageError("Error in parsing command line args for %s" % test_name)
if obj.requiresRoot:
try:
@@ -90,28 +90,23 @@ def isTestCase(obj):
except TypeError:
return False
-def findTestClassesFromConfig(cmd_line_options):
+def findTestClassesFromFile(filename):
"""
Takes as input the command line config parameters and returns the test
case classes.
- If it detects that a certain test class is using the old OONIProbe format,
- then it will adapt it to the new testing system.
- :param cmd_line_options:
- A configured and instantiated :class:`twisted.python.usage.Options`
- class.
+ :param filename:
+ the absolute path to the file containing the ooniprobe test classes
+
:return:
A list of class objects found in a file or module given on the
commandline.
"""
-
- filename = cmd_line_options['test']
classes = []
-
module = filenameToModule(filename)
for name, val in inspect.getmembers(module):
if isTestCase(val):
- classes.append(processTest(val, cmd_line_options))
+ classes.append(processTest(val))
return classes
def makeTestCases(klass, tests, method_prefix):
@@ -227,8 +222,10 @@ def runTestCasesWithInputUnit(test_cases, input_unit, oreporter):
return defer.DeferredList(dl)
@defer.inlineCallbacks
-def runTestCases(test_cases, options,
- cmd_line_options, yamloo_filename):
+def runTestCases(test_cases, options, cmd_line_options):
+ log.debug("Running %s" % test_cases)
+ log.debug("Options %s" % options)
+ log.debug("cmd_line_options %s" % dict(cmd_line_options))
try:
assert len(options) != 0, "Length of options is zero!"
except AssertionError, ae:
@@ -247,15 +244,12 @@ def runTestCases(test_cases, options,
log.msg("options[0] = %s" % first)
test_inputs = [None]
- log.debug("Creating %s" % yamloo_filename)
-
if cmd_line_options['collector']:
- log.debug("Using remote collector %s" % cmd_line_options['collector'])
- oreporter = reporter.OONIBReporter(cmd_line_options['collector'])
+ log.debug("Using remote collector")
+ oreporter = reporter.OONIBReporter(cmd_line_options)
else:
- reportFile = open(yamloo_filename, 'w+')
- log.debug("Reporting to file %s" % reportFile)
- oreporter = reporter.YAMLReporter(reportFile)
+ log.debug("Reporting to file %s" % config.reports.yamloo)
+ oreporter = reporter.YAMLReporter(cmd_line_options)
try:
input_unit_factory = InputUnitFactory(test_inputs)
diff --git a/ooniprobe.conf b/ooniprobe.conf
index ad761e1..e9f208f 100644
--- a/ooniprobe.conf
+++ b/ooniprobe.conf
@@ -22,7 +22,7 @@ advanced:
# /path/to/ooni-probe/data/
#geoip_data_dir: /usr/share/GeoIP/
geoip_data_dir: /home/x/code/networking/ooni-probe/data/
- debug: false
+ debug: true
threadpool_size: 10
tor_socksport: 9050
# For auto detection
More information about the tor-commits
mailing list