commit 37221cd7bfe6fad3750ae2c701d22a7bf03b9793 Author: Arturo Filastò arturo@filasto.net Date: Wed Nov 7 17:06:40 2012 +0100
Implement config file support * You are able to specify basic and advanced options in YAML format --- .gitignore | 2 ++ ooni/__init__.py | 4 ++-- ooni/config.py | 36 ++++++++++++++++++++++++++++++++++++ ooni/oconfig.py | 8 -------- ooni/oonicli.py | 14 +++++++------- ooni/reporter.py | 26 +++++++++++++++++++------- ooni/runner.py | 20 ++++++++++---------- ooni/utils/geodata.py | 4 ++++ ooniprobe.conf | 19 +++++++++++++++++++ 9 files changed, 99 insertions(+), 34 deletions(-)
diff --git a/.gitignore b/.gitignore index 4a6b565..d1015ab 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ oonib/data/* fluff/ pcaps/ report*.yaml +docs/build/* +data/*.dat diff --git a/ooni/__init__.py b/ooni/__init__.py index 0c9f297..16b20ca 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,4 +1,4 @@ -from . import oconfig +from . import config from . import inputunit from . import kit from . import lib @@ -9,6 +9,6 @@ from . import runner from . import templates from . import utils
-__all__ = ['oconfig', 'inputunit', 'kit', +__all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', 'runner', 'templates', 'utils'] diff --git a/ooni/config.py b/ooni/config.py new file mode 100644 index 0000000..97f396c --- /dev/null +++ b/ooni/config.py @@ -0,0 +1,36 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo "hellais" Filastò art@fuffa.org +# :licence: see LICENSE + +import os +import yaml + +from ooni.utils import Storage + +def get_root_path(): + this_directory = os.path.dirname(__file__) + root = os.path.join(this_directory, '..') + root = os.path.abspath(root) + return root + +config_file = os.path.join(get_root_path(), 'ooniprobe.conf') +try: + f = open(config_file) +except IOError: + raise Exception("Unable to open config file. "\ + "Create a config file called ooniprobe.conf") + +config_file_contents = '\n'.join(f.readlines()) +configuration = yaml.safe_load(config_file_contents) + +# Process the basic configuration options +basic = Storage() +for k, v in configuration['basic'].items(): + basic[k] = v + +# Process the advanced configuration options +advanced = Storage() +for k, v in configuration['advanced'].items(): + advanced[k] = v + diff --git a/ooni/oconfig.py b/ooni/oconfig.py deleted file mode 100644 index 202b569..0000000 --- a/ooni/oconfig.py +++ /dev/null @@ -1,8 +0,0 @@ -from ooni.utils import Storage - -# XXX move this to an actual configuration file -basic = Storage() -basic.logfile = '/tmp/ooniprobe.log' -advanced = Storage() - -advanced.debug = True diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 6def1df..5ee6deb 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -87,20 +87,20 @@ class Options(usage.Options, app.ReactorSelectionMixin): def run(): if len(sys.argv) == 1: sys.argv.append("--help") - config = Options() + cmd_line_options = Options() try: - config.parseOptions() + cmd_line_options.parseOptions() except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue)
- if config['debug-stacktraces']: + if cmd_line_options['debug-stacktraces']: defer.setDebugging(True)
- classes = runner.findTestClassesFromConfig(config) - casesList, options = runner.loadTestsAndOptions(classes, config) + classes = runner.findTestClassesFromConfig(cmd_line_options) + casesList, options = runner.loadTestsAndOptions(classes, cmd_line_options)
for idx, cases in enumerate(casesList): - orunner = runner.ORunner(cases, options[idx], config) - log.start(config['logfile']) + orunner = runner.ORunner(cases, options[idx], cmd_line_options) + log.start(cmd_line_options['logfile']) orunner.run()
diff --git a/ooni/reporter.py b/ooni/reporter.py index a02e5a9..f3fbca6 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -18,6 +18,7 @@ from twisted.internet import defer
from ooni.templates.httpt import BodyReceiver, StringProducer from ooni.utils import date, log, geodata +from ooni import config
try: ## Get rid of the annoying "No route found for @@ -187,17 +188,28 @@ class ReporterFactory(OReporter): client_geodata = {} log.msg("Running geo IP lookup via check.torproject.org")
- client_geodata['ip'] = yield geodata.myIP() - client_geodata['asn'] = 'unknown' - client_geodata['city'] = 'unknown' - client_geodata['countrycode'] = 'unknown' + client_ip = yield geodata.myIP() + if config.includeip: + client_geodata['ip'] = client_ip + else: + client_geodata['ip'] = None + + client_geodata['asn'] = None + client_geodata['city'] = None + client_geodata['countrycode'] = None
try: import txtorcon client_location = txtorcon.util.NetLocation(client_geodata['ip']) - client_geodata['asn'] = client_location.asn - client_geodata['city'] = client_location.city - client_geodata['countrycode'] = client_location.countrycode + if config.includeasn: + client_geodata['asn'] = client_location.asn + + if config.includecity: + client_geodata['city'] = client_location.city + + if config.includecountry: + client_geodata['countrycode'] = client_location.countrycode + except ImportError: log.err("txtorcon is not installed. Geolocation lookup is not"\ "supported") diff --git a/ooni/runner.py b/ooni/runner.py index 102ff29..797a9d4 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -23,7 +23,7 @@ from ooni.nettest import InputTestSuite from ooni.reporter import ReporterFactory from ooni.utils import log, date
-def processTest(obj, config): +def processTest(obj, cmd_line_options): """ Process the parameters and :class:`twisted.python.usage.Options` of a :class:`ooni.nettest.Nettest`. @@ -31,7 +31,7 @@ def processTest(obj, config): :param obj: An uninstantiated old test, which should be a subclass of :class:`ooni.plugoo.tests.OONITest`. - :param config: + :param cmd_line_options: A configured and instantiated :class:`twisted.python.usage.Options` class. """ @@ -63,7 +63,7 @@ def processTest(obj, config):
options = Options()
- options.parseOptions(config['subArgs']) + options.parseOptions(cmd_line_options['subArgs']) obj.localOptions = options
if input_file: @@ -76,14 +76,14 @@ def processTest(obj, config):
return obj
-def findTestClassesFromConfig(config): +def findTestClassesFromConfig(cmd_line_options): """ Takes as input the command line config parameters and returns the test case classes. If it detects that a certain test class is using the old OONIProbe format, then it will adapt it to the new testing system.
- :param config: + :param cmd_line_options: A configured and instantiated :class:`twisted.python.usage.Options` class. :return: @@ -91,14 +91,14 @@ def findTestClassesFromConfig(config): commandline. """
- filename = config['test'] + filename = cmd_line_options['test'] classes = []
module = filenameToModule(filename) for name, val in inspect.getmembers(module): if isTestCase(val): log.debug("Detected TestCase %s" % val) - classes.append(processTest(val, config)) + classes.append(processTest(val, cmd_line_options)) return classes
def makeTestCases(klass, tests, method_prefix): @@ -112,7 +112,7 @@ def makeTestCases(klass, tests, method_prefix): cases.append(klass(method_prefix+test)) return cases
-def loadTestsAndOptions(classes, config): +def loadTestsAndOptions(classes, cmd_line_options): """ Takes a list of test classes and returns their testcases and options. Legacy tests will be adapted. @@ -144,7 +144,7 @@ class ORunner(object): them in input units. I also create all the report instances required to run the tests. """ - def __init__(self, cases, options=None, config=None): + def __init__(self, cases, options=None, cmd_line_options=None): self.baseSuite = InputTestSuite self.cases = cases self.options = options @@ -168,7 +168,7 @@ class ORunner(object): self.inputs = [None]
try: - reportFile = open(config['reportfile'], 'a+') + reportFile = open(cmd_line_options['reportfile'], 'a+') except TypeError: filename = 'report_'+date.timestamp()+'.yaml' reportFile = open(filename, 'a+') diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index 9f782e8..5254044 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -1,4 +1,8 @@ import re +import pygeoip + +from ooni import config + from twisted.web.client import Agent from twisted.internet import reactor, defer, protocol
diff --git a/ooniprobe.conf b/ooniprobe.conf new file mode 100644 index 0000000..278dc4c --- /dev/null +++ b/ooniprobe.conf @@ -0,0 +1,19 @@ +# This is the configuration file for OONIProbe +# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html +# Keep in mind that indentation matters. + +basic: + # Where OONIProbe should be writing it's log file + logfile: /tmp/ooniprobe.log + # Should we include the IP address of the probe in the report? + includeip: true + # Should we include the ASN of the probe in the report? + includeasn: true + # Should we include the ASN of the probe in the report? + includecountry: true + # Should we include the ASN of the probe in the report? + includecity: true +advanced: + geoip_data_dir: /home/x/code/networking/ooni-probe/data + debug: true +