[tor-commits] [ooni-probe/master] Implement config file support
art at torproject.org
art at torproject.org
Wed Nov 7 16:08:02 UTC 2012
commit 37221cd7bfe6fad3750ae2c701d22a7bf03b9793
Author: Arturo Filastò <arturo at filasto.net>
Date: Wed Nov 7 17:06:40 2012 +0100
Implement config file support
* You are able to specify basic and advanced options in YAML format
---
.gitignore | 2 ++
ooni/__init__.py | 4 ++--
ooni/config.py | 36 ++++++++++++++++++++++++++++++++++++
ooni/oconfig.py | 8 --------
ooni/oonicli.py | 14 +++++++-------
ooni/reporter.py | 26 +++++++++++++++++++-------
ooni/runner.py | 20 ++++++++++----------
ooni/utils/geodata.py | 4 ++++
ooniprobe.conf | 19 +++++++++++++++++++
9 files changed, 99 insertions(+), 34 deletions(-)
diff --git a/.gitignore b/.gitignore
index 4a6b565..d1015ab 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,5 @@ oonib/data/*
fluff/
pcaps/
report*.yaml
+docs/build/*
+data/*.dat
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 0c9f297..16b20ca 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,4 +1,4 @@
-from . import oconfig
+from . import config
from . import inputunit
from . import kit
from . import lib
@@ -9,6 +9,6 @@ from . import runner
from . import templates
from . import utils
-__all__ = ['oconfig', 'inputunit', 'kit',
+__all__ = ['config', 'inputunit', 'kit',
'lib', 'nettest', 'oonicli', 'reporter',
'runner', 'templates', 'utils']
diff --git a/ooni/config.py b/ooni/config.py
new file mode 100644
index 0000000..97f396c
--- /dev/null
+++ b/ooni/config.py
@@ -0,0 +1,36 @@
+# -*- encoding: utf-8 -*-
+#
+# :authors: Arturo "hellais" Filastò <art at fuffa.org>
+# :licence: see LICENSE
+
+import os
+import yaml
+
+from ooni.utils import Storage
+
+def get_root_path():
+ this_directory = os.path.dirname(__file__)
+ root = os.path.join(this_directory, '..')
+ root = os.path.abspath(root)
+ return root
+
+config_file = os.path.join(get_root_path(), 'ooniprobe.conf')
+try:
+ f = open(config_file)
+except IOError:
+ raise Exception("Unable to open config file. "\
+ "Create a config file called ooniprobe.conf")
+
+config_file_contents = '\n'.join(f.readlines())
+configuration = yaml.safe_load(config_file_contents)
+
+# Process the basic configuration options
+basic = Storage()
+for k, v in configuration['basic'].items():
+ basic[k] = v
+
+# Process the advanced configuration options
+advanced = Storage()
+for k, v in configuration['advanced'].items():
+ advanced[k] = v
+
diff --git a/ooni/oconfig.py b/ooni/oconfig.py
deleted file mode 100644
index 202b569..0000000
--- a/ooni/oconfig.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from ooni.utils import Storage
-
-# XXX move this to an actual configuration file
-basic = Storage()
-basic.logfile = '/tmp/ooniprobe.log'
-advanced = Storage()
-
-advanced.debug = True
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 6def1df..5ee6deb 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -87,20 +87,20 @@ class Options(usage.Options, app.ReactorSelectionMixin):
def run():
if len(sys.argv) == 1:
sys.argv.append("--help")
- config = Options()
+ cmd_line_options = Options()
try:
- config.parseOptions()
+ cmd_line_options.parseOptions()
except usage.error, ue:
raise SystemExit, "%s: %s" % (sys.argv[0], ue)
- if config['debug-stacktraces']:
+ if cmd_line_options['debug-stacktraces']:
defer.setDebugging(True)
- classes = runner.findTestClassesFromConfig(config)
- casesList, options = runner.loadTestsAndOptions(classes, config)
+ classes = runner.findTestClassesFromConfig(cmd_line_options)
+ casesList, options = runner.loadTestsAndOptions(classes, cmd_line_options)
for idx, cases in enumerate(casesList):
- orunner = runner.ORunner(cases, options[idx], config)
- log.start(config['logfile'])
+ orunner = runner.ORunner(cases, options[idx], cmd_line_options)
+ log.start(cmd_line_options['logfile'])
orunner.run()
diff --git a/ooni/reporter.py b/ooni/reporter.py
index a02e5a9..f3fbca6 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -18,6 +18,7 @@ from twisted.internet import defer
from ooni.templates.httpt import BodyReceiver, StringProducer
from ooni.utils import date, log, geodata
+from ooni import config
try:
## Get rid of the annoying "No route found for
@@ -187,17 +188,28 @@ class ReporterFactory(OReporter):
client_geodata = {}
log.msg("Running geo IP lookup via check.torproject.org")
- client_geodata['ip'] = yield geodata.myIP()
- client_geodata['asn'] = 'unknown'
- client_geodata['city'] = 'unknown'
- client_geodata['countrycode'] = 'unknown'
+ client_ip = yield geodata.myIP()
+ if config.includeip:
+ client_geodata['ip'] = client_ip
+ else:
+ client_geodata['ip'] = None
+
+ client_geodata['asn'] = None
+ client_geodata['city'] = None
+ client_geodata['countrycode'] = None
try:
import txtorcon
client_location = txtorcon.util.NetLocation(client_geodata['ip'])
- client_geodata['asn'] = client_location.asn
- client_geodata['city'] = client_location.city
- client_geodata['countrycode'] = client_location.countrycode
+ if config.includeasn:
+ client_geodata['asn'] = client_location.asn
+
+ if config.includecity:
+ client_geodata['city'] = client_location.city
+
+ if config.includecountry:
+ client_geodata['countrycode'] = client_location.countrycode
+
except ImportError:
log.err("txtorcon is not installed. Geolocation lookup is not"\
"supported")
diff --git a/ooni/runner.py b/ooni/runner.py
index 102ff29..797a9d4 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -23,7 +23,7 @@ from ooni.nettest import InputTestSuite
from ooni.reporter import ReporterFactory
from ooni.utils import log, date
-def processTest(obj, config):
+def processTest(obj, cmd_line_options):
"""
Process the parameters and :class:`twisted.python.usage.Options` of a
:class:`ooni.nettest.Nettest`.
@@ -31,7 +31,7 @@ def processTest(obj, config):
:param obj:
An uninstantiated old test, which should be a subclass of
:class:`ooni.plugoo.tests.OONITest`.
- :param config:
+ :param cmd_line_options:
A configured and instantiated :class:`twisted.python.usage.Options`
class.
"""
@@ -63,7 +63,7 @@ def processTest(obj, config):
options = Options()
- options.parseOptions(config['subArgs'])
+ options.parseOptions(cmd_line_options['subArgs'])
obj.localOptions = options
if input_file:
@@ -76,14 +76,14 @@ def processTest(obj, config):
return obj
-def findTestClassesFromConfig(config):
+def findTestClassesFromConfig(cmd_line_options):
"""
Takes as input the command line config parameters and returns the test
case classes.
If it detects that a certain test class is using the old OONIProbe format,
then it will adapt it to the new testing system.
- :param config:
+ :param cmd_line_options:
A configured and instantiated :class:`twisted.python.usage.Options`
class.
:return:
@@ -91,14 +91,14 @@ def findTestClassesFromConfig(config):
commandline.
"""
- filename = config['test']
+ filename = cmd_line_options['test']
classes = []
module = filenameToModule(filename)
for name, val in inspect.getmembers(module):
if isTestCase(val):
log.debug("Detected TestCase %s" % val)
- classes.append(processTest(val, config))
+ classes.append(processTest(val, cmd_line_options))
return classes
def makeTestCases(klass, tests, method_prefix):
@@ -112,7 +112,7 @@ def makeTestCases(klass, tests, method_prefix):
cases.append(klass(method_prefix+test))
return cases
-def loadTestsAndOptions(classes, config):
+def loadTestsAndOptions(classes, cmd_line_options):
"""
Takes a list of test classes and returns their testcases and options.
Legacy tests will be adapted.
@@ -144,7 +144,7 @@ class ORunner(object):
them in input units. I also create all the report instances required to run
the tests.
"""
- def __init__(self, cases, options=None, config=None):
+ def __init__(self, cases, options=None, cmd_line_options=None):
self.baseSuite = InputTestSuite
self.cases = cases
self.options = options
@@ -168,7 +168,7 @@ class ORunner(object):
self.inputs = [None]
try:
- reportFile = open(config['reportfile'], 'a+')
+ reportFile = open(cmd_line_options['reportfile'], 'a+')
except TypeError:
filename = 'report_'+date.timestamp()+'.yaml'
reportFile = open(filename, 'a+')
diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py
index 9f782e8..5254044 100644
--- a/ooni/utils/geodata.py
+++ b/ooni/utils/geodata.py
@@ -1,4 +1,8 @@
import re
+import pygeoip
+
+from ooni import config
+
from twisted.web.client import Agent
from twisted.internet import reactor, defer, protocol
diff --git a/ooniprobe.conf b/ooniprobe.conf
new file mode 100644
index 0000000..278dc4c
--- /dev/null
+++ b/ooniprobe.conf
@@ -0,0 +1,19 @@
+# This is the configuration file for OONIProbe
+# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
+# Keep in mind that indentation matters.
+
+basic:
+ # Where OONIProbe should be writing it's log file
+ logfile: /tmp/ooniprobe.log
+ # Should we include the IP address of the probe in the report?
+ includeip: true
+ # Should we include the ASN of the probe in the report?
+ includeasn: true
+ # Should we include the ASN of the probe in the report?
+ includecountry: true
+ # Should we include the ASN of the probe in the report?
+ includecity: true
+advanced:
+ geoip_data_dir: /home/x/code/networking/ooni-probe/data
+ debug: true
+
More information about the tor-commits
mailing list