[tor-commits] [ooni-probe/master] Improve configuration management
art at torproject.org
art at torproject.org
Wed Jun 19 12:32:45 UTC 2013
commit c7f65ae7df71c469f9cf5db126f12bf1c11c49da
Author: Arturo Filastò <art at fuffa.org>
Date: Tue Apr 23 16:59:24 2013 +0200
Improve configuration management
* Install ooniprobe settings in the users home directory (~/.ooni)
* Define the paths for ooniprobe data directories
---
ooni/__init__.py | 2 -
ooni/api/spec.py | 2 +-
ooni/config.py | 98 -------------------------------------------
ooni/director.py | 15 ++++---
ooni/geoip.py | 3 +-
ooni/managers.py | 18 +++++---
ooni/nettest.py | 2 +-
ooni/oonicli.py | 11 ++++-
ooni/oonid.py | 2 +-
ooni/reporter.py | 2 +-
ooni/settings.py | 92 ++++++++++++++++++++++++++++++++++++++++
ooni/tasks.py | 10 ++---
ooni/templates/httpt.py | 2 +-
ooni/templates/scapyt.py | 2 +-
ooni/tests/mocks.py | 2 +-
ooni/tests/test_managers.py | 8 ++++
ooni/tests/test_nettest.py | 5 ++-
ooni/utils/geodata.py | 2 +-
ooni/utils/log.py | 2 +-
ooni/utils/txscapy.py | 2 +-
20 files changed, 149 insertions(+), 133 deletions(-)
diff --git a/ooni/__init__.py b/ooni/__init__.py
index cc4bf03..815e16e 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,5 @@
# -*- encoding: utf-8 -*-
-from . import config
-from . import kit
from . import nettest
from . import oonicli
from . import reporter
diff --git a/ooni/api/spec.py b/ooni/api/spec.py
index af238f4..5b538b2 100644
--- a/ooni/api/spec.py
+++ b/ooni/api/spec.py
@@ -5,7 +5,7 @@ import types
from cyclone import web, escape
-from ooni import config
+from ooni.settings import config
class InvalidInputFilename(Exception):
pass
diff --git a/ooni/config.py b/ooni/config.py
deleted file mode 100644
index 5aeb49d..0000000
--- a/ooni/config.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import os
-import yaml
-
-from twisted.internet import reactor, threads, defer
-
-from ooni import otime
-from ooni.utils import Storage
-
-class TestFilenameNotSet(Exception):
- pass
-
-def get_root_path():
- this_directory = os.path.dirname(__file__)
- root = os.path.join(this_directory, '..')
- root = os.path.abspath(root)
- return root
-
-def createConfigFile():
- """
- XXX implement me
- """
- sample_config_file = os.path.join(get_root_path(), 'ooniprobe.conf.sample')
-
-def generatePcapFilename():
- if cmd_line_options['pcapfile']:
- reports.pcap = cmd_line_options['pcapfile']
- else:
- if cmd_line_options['test']:
- test_filename = os.path.basename(cmd_line_options['test'])
- else:
- test_filename = os.path.basename(cmd_line_options['testdeck'])
-
- test_name = '.'.join(test_filename.split(".")[:-1])
- frm_str = "report_%s_"+otime.timestamp()+".%s"
- reports.pcap = frm_str % (test_name, "pcap")
-
-class ConfigurationSetting(Storage):
- def __init__(self, key):
- config_file = os.path.join(get_root_path(), 'ooniprobe.conf')
- try:
- f = open(config_file)
- except IOError:
- createConfigFile()
- raise Exception("Unable to open config file. "\
- "Copy ooniprobe.conf.sample to ooniprobe.conf")
-
- config_file_contents = '\n'.join(f.readlines())
- configuration = yaml.safe_load(config_file_contents)
-
- try:
- for k, v in configuration[key].items():
- self[k] = v
- except AttributeError:
- pass
-
-basic = ConfigurationSetting('basic')
-advanced = ConfigurationSetting('advanced')
-privacy = ConfigurationSetting('privacy')
-tor = ConfigurationSetting('tor')
-
-data_directory = os.path.join(get_root_path(), 'data')
-nettest_directory = os.path.join(get_root_path(), 'nettests')
-inputs_directory = os.path.join(get_root_path(), 'inputs')
-
-reports = Storage()
-state = Storage()
-scapyFactory = None
-stateDict = None
-
-# XXX refactor this to use a database
-resume_lock = defer.DeferredLock()
-
-cmd_line_options = None
-resume_filename = None
-
-# XXX-Twisted this is used to check if we have started the reactor or not. It
-# is necessary because if the tests are already concluded because we have
-# resumed a test session then it will call reactor.run() even though there is
-# no condition that will ever stop it.
-# There should be a more twisted way of doing this.
-start_reactor = True
-tor_state = None
-tor_control = None
-config_file = None
-sample_config_file = None
-# This is used to store the probes IP address obtained via Tor
-probe_ip = None
-# This is used to keep track of the state of the sniffer
-sniffer_running = None
-
-logging = True
-
-if not resume_filename:
- resume_filename = os.path.join(get_root_path(), 'ooniprobe.resume')
- try:
- with open(resume_filename) as f: pass
- except IOError as e:
- with open(resume_filename, 'w+') as f: pass
diff --git a/ooni/director.py b/ooni/director.py
index 5f23668..1ef878c 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -3,13 +3,13 @@ import sys
import os
import re
-from ooni import config
from ooni import geoip
from ooni.managers import ReportEntryManager, MeasurementManager
from ooni.reporter import Report
from ooni.utils import log, checkForRoot
from ooni.utils.net import randomFreePort
-from ooni.nettest import NetTest
+from ooni.nettest import NetTest, getNetTestInformation
+from ooni.settings import config
from ooni import errors
from txtorcon import TorConfig
@@ -85,6 +85,10 @@ class Director(object):
self.torControlProtocol = None
+ # This deferred is fired once all the measurements and their reporting
+ # tasks are completed.
+ self.allTestsDone = defer.Deferred()
+
def getNetTests(self):
nettests = {}
def is_nettest(filename):
@@ -108,16 +112,12 @@ class Director(object):
return nettests
- # This deferred is fired once all the measurements and their reporting
- # tasks are completed.
- self.allTestsDone = defer.Deferred()
-
@defer.inlineCallbacks
def start(self):
if config.privacy.includepcap:
log.msg("Starting")
if not config.reports.pcap:
- config.reports.pcap = config.generatePcapFilename()
+ config.generate_pcap_filename()
self.startSniffing()
if config.advanced.start_tor:
@@ -324,7 +324,6 @@ class Director(object):
log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
d = launch_tor(tor_config, reactor,
- tor_binary=config.advanced.tor_binary,
progress_updates=updates)
d.addCallback(setup_complete)
d.addErrback(setup_failed)
diff --git a/ooni/geoip.py b/ooni/geoip.py
index 1e6f6e9..c1987fc 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -7,7 +7,8 @@ from ooni.utils.net import userAgents, BodyReceiver
from twisted.internet import reactor, defer, protocol
from ooni.utils import log, net, checkForRoot
-from ooni import config, errors
+from ooni.settings import config
+from ooni import errors
try:
from pygeoip import GeoIP
diff --git a/ooni/managers.py b/ooni/managers.py
index 9f8366f..ff7c2f2 100644
--- a/ooni/managers.py
+++ b/ooni/managers.py
@@ -2,7 +2,7 @@ import itertools
from twisted.internet import defer
from ooni.utils import log
-from ooni import config
+from ooni.settings import config
def makeIterable(item):
"""
@@ -141,8 +141,12 @@ class MeasurementManager(TaskManager):
NetTest on the contrary is aware of the typology of measurements that it is
dispatching as they are logically grouped by test file.
"""
- retries = config.advanced.measurement_retries
- concurrency = config.advanced.measurement_concurrency
+ def __init__(self):
+ if config.advanced.measurement_retries:
+ self.retries = config.advanced.measurement_retries
+ if config.advanced.measurement_concurrency:
+ self.concurrency = config.advanced.measurement_concurrency
+ super(MeasurementManager, self).__init__()
def succeeded(self, result, measurement):
log.debug("Successfully performed measurement %s" % measurement)
@@ -152,8 +156,12 @@ class MeasurementManager(TaskManager):
pass
class ReportEntryManager(TaskManager):
- retries = config.advanced.reporting_retries
- concurrency = config.advanced.reporting_concurrency
+ def __init__(self):
+ if config.advanced.reporting_retries:
+ self.retries = config.advanced.reporting_retries
+ if config.advanced.reporting_concurrency:
+ self.concurrency = config.advanced.reporting_concurrency
+ super(ReportEntryManager, self).__init__()
def succeeded(self, result, task):
log.debug("Successfully performed report %s" % task)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index d9bc94d..0d0e889 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -8,8 +8,8 @@ from twisted.python import usage, reflect
from ooni import geoip
from ooni.tasks import Measurement
from ooni.utils import log, checkForRoot, geodata
-from ooni import config
from ooni import otime
+from ooni.settings import config
from ooni import errors as e
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index b5e8e27..40453b1 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -12,7 +12,7 @@ from twisted.python.util import spewer
from ooni import errors
-from ooni import config
+from ooni.settings import config
from ooni.director import Director
from ooni.reporter import YAMLReporter, OONIBReporter
from ooni.nettest import NetTestLoader, MissingRequiredOption
@@ -40,6 +40,10 @@ class Options(usage.Options):
["logfile", "l", None, "log file name"],
["pcapfile", "O", None, "pcap file name"],
["parallelism", "p", "10", "input parallelism"],
+ ["configfile", "f", None,
+ "Specify a path to the ooniprobe configuration file"],
+ ["datadir", "d", None,
+ "Specify a path to the ooniprobe data directory"]
]
compData = usage.Completions(
@@ -100,8 +104,11 @@ def runWithDirector():
test!
"""
global_options = parseOptions()
- log.start(global_options['logfile'])
+ config.global_options = global_options
+ config.set_paths()
+ config.read_config_file()
+ log.start(global_options['logfile'])
# contains (test_cases, options, cmd_line_options)
test_list = []
if global_options['no-collector']:
diff --git a/ooni/oonid.py b/ooni/oonid.py
index dde768e..cc71d47 100644
--- a/ooni/oonid.py
+++ b/ooni/oonid.py
@@ -4,7 +4,7 @@ import random
from twisted.application import service, internet
from twisted.web import static, server
-from ooni import config
+from ooni.settings import config
from ooni.api.spec import oonidApplication
from ooni.director import Director
from ooni.reporter import YAMLReporter, OONIBReporter
diff --git a/ooni/reporter.py b/ooni/reporter.py
index b04b46b..109eccf 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -32,7 +32,7 @@ from ooni import otime
from ooni.utils import geodata, pushFilenameStack
from ooni.utils.net import BodyReceiver, StringProducer, userAgents
-from ooni import config
+from ooni.settings import config
from ooni.tasks import ReportEntry, TaskTimedOut
diff --git a/ooni/settings.py b/ooni/settings.py
new file mode 100644
index 0000000..acb7502
--- /dev/null
+++ b/ooni/settings.py
@@ -0,0 +1,92 @@
+import os
+import yaml
+from shutil import copyfile
+from os.path import abspath, expanduser
+
+from twisted.internet import reactor, threads, defer
+
+from ooni import otime
+from ooni.utils import Storage
+
+class OConfig(object):
+ def __init__(self):
+ self.global_options = {}
+ self.reports = Storage()
+ self.scapyFactory = None
+ self.tor_state = None
+ # This is used to store the probes IP address obtained via Tor
+ self.probe_ip = None
+ # This is used to keep track of the state of the sniffer
+ self.sniffer_running = None
+ self.logging = True
+ self.basic = Storage()
+ self.advanced = Storage()
+ self.tor = Storage()
+ self.privacy = Storage()
+ self.set_paths()
+ self.initialize_ooni_home()
+
+ def set_paths(self):
+ if self.global_options.get('datadir'):
+ self.data_directory = abspath(expanduser(self.global_options['datadir']))
+ else:
+ self.data_directory = '/usr/share/ooni/'
+ self.nettest_directory = os.path.join(self.data_directory, 'nettests')
+
+ self.ooni_home = os.path.join(expanduser('~'), '.ooni')
+ self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
+
+ if self.global_options.get('configfile'):
+ config_file = global_options['configfile']
+ else:
+ config_file = os.path.join('~', '.ooni', 'ooniprobe.conf')
+ self.config_file = expanduser(config_file)
+
+ def initialize_ooni_home(self):
+ if not os.path.isdir(self.ooni_home):
+ print "Ooni home directory does not exist."
+ print "Creating it in '%s'." % self.ooni_home
+ os.mkdir(self.ooni_home)
+ os.mkdir(self.inputs_directory)
+
+ def _create_config_file(self):
+ sample_config_file = os.path.join(self.data_directory,
+ 'ooniprobe.conf.sample')
+ target_config_file = os.path.join(self.ooni_home,
+ 'ooniprobe.conf')
+ print "Creating it for you in '%s'." % target_config_file
+ copyfile(sample_config_file, target_config_file)
+
+ def read_config_file(self):
+ try:
+ with open(self.config_file) as f: pass
+ except IOError:
+ print "Configuration file does not exist."
+ self._create_config_file()
+ self.read_config_file()
+
+ with open(self.config_file) as f:
+ config_file_contents = '\n'.join(f.readlines())
+ configuration = yaml.safe_load(config_file_contents)
+
+ for setting in ['basic', 'advanced', 'privacy', 'tor']:
+ try:
+ for k, v in configuration[setting].items():
+ getattr(self, setting)[k] = v
+ except AttributeError:
+ pass
+
+ def generate_pcap_filename():
+ if self.global_options.get('pcapfile'):
+ self.reports.pcap = self.global_options['pcapfile']
+ else:
+ if self.global_options.get('test'):
+ test_filename = os.path.basename(self.global_options['test'])
+ else:
+ test_filename = os.path.basename(self.global_options['testdeck'])
+
+ test_name = '.'.join(test_filename.split(".")[:-1])
+ frm_str = "report_%s_"+otime.timestamp()+".%s"
+ self.reports.pcap = frm_str % (test_name, "pcap")
+
+config = OConfig()
diff --git a/ooni/tasks.py b/ooni/tasks.py
index 829d11e..f686a9c 100644
--- a/ooni/tasks.py
+++ b/ooni/tasks.py
@@ -1,6 +1,6 @@
import time
-from ooni import config
+from ooni.settings import config
from twisted.internet import defer, reactor
class BaseTask(object):
@@ -91,8 +91,6 @@ class TaskWithTimeout(BaseTask):
return BaseTask.start(self)
class Measurement(TaskWithTimeout):
- timeout = config.advanced.measurement_timeout
-
def __init__(self, test_class, test_method, test_input):
"""
test_class:
@@ -117,6 +115,8 @@ class Measurement(TaskWithTimeout):
self.netTestMethod = getattr(self.testInstance, test_method)
+ if config.advanced.measurement_timeout:
+ self.timeout = config.advanced.measurement_timeout
TaskWithTimeout.__init__(self)
def succeeded(self, result):
@@ -130,12 +130,12 @@ class Measurement(TaskWithTimeout):
return d
class ReportEntry(TaskWithTimeout):
- timeout = config.advanced.reporting_timeout
-
def __init__(self, reporter, measurement):
self.reporter = reporter
self.measurement = measurement
+ if config.advanced.reporting_timeout:
+ self.timeout = config.advanced.reporting_timeout
TaskWithTimeout.__init__(self)
def run(self):
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index e8891c7..0bca5df 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -13,7 +13,7 @@ from twisted.web._newclient import Request, Response, ResponseNeverReceived
from ooni.nettest import NetTestCase
from ooni.utils import log
-from ooni import config
+from ooni.settings import config
from ooni.utils.net import BodyReceiver, StringProducer, userAgents
diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py
index d5d6564..fdc5a24 100644
--- a/ooni/templates/scapyt.py
+++ b/ooni/templates/scapyt.py
@@ -8,7 +8,7 @@ from scapy.all import send, sr, IP, TCP, config
from ooni.reporter import createPacketReport
from ooni.nettest import NetTestCase
from ooni.utils import log
-from ooni import config
+from ooni.settings import config
from ooni.utils.txscapy import ScapySender, getDefaultIface, ScapyFactory
from ooni.utils.txscapy import hasRawSocketPermission
diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py
index 4c4a015..f849344 100644
--- a/ooni/tests/mocks.py
+++ b/ooni/tests/mocks.py
@@ -1,7 +1,7 @@
from twisted.python import failure
from twisted.internet import defer
-from ooni import config
+from ooni.settings import config
from ooni.tasks import BaseTask, TaskWithTimeout
from ooni.nettest import NetTest
from ooni.managers import TaskManager
diff --git a/ooni/tests/test_managers.py b/ooni/tests/test_managers.py
index e2af7b3..c290155 100644
--- a/ooni/tests/test_managers.py
+++ b/ooni/tests/test_managers.py
@@ -1,3 +1,5 @@
+import os
+
from twisted.trial import unittest
from twisted.python import failure
from twisted.internet import defer, task
@@ -11,6 +13,8 @@ from ooni.tests.mocks import MockTimeoutOnceTask, MockFailTaskWithTimeout
from ooni.tests.mocks import MockTaskManager, mockFailure, MockDirector
from ooni.tests.mocks import MockNetTest, MockMeasurement, MockSuccessMeasurement
from ooni.tests.mocks import MockFailMeasurement, MockFailOnceMeasurement
+from ooni.settings import config
+
class TestTaskManager(unittest.TestCase):
timeout = 1
@@ -22,6 +26,10 @@ class TestTaskManager(unittest.TestCase):
self.measurementManager.start()
self.clock = task.Clock()
+ data_dir = os.path.dirname(os.path.abspath(__file__))
+ data_dir = os.path.join(data_dir, '..', '..', 'data')
+ config.global_options['datadir'] = data_dir
+ config.set_paths()
def schedule_successful_tasks(self, task_type, number=1):
all_done = []
diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py
index 3b59cc4..77b8a1c 100644
--- a/ooni/tests/test_nettest.py
+++ b/ooni/tests/test_nettest.py
@@ -11,7 +11,6 @@ from ooni.nettest import NetTestLoader, FailureToLoadNetTest
from ooni.tasks import BaseTask
from ooni.director import Director
-
from ooni.managers import TaskManager
from ooni.tests.mocks import MockMeasurement, MockMeasurementFailOnce
@@ -100,6 +99,9 @@ class TestNetTest(unittest.TestCase):
for i in range(10):
f.write("%s\n" % i)
+ from ooni.settings import config
+ config.read_config_file()
+
def assertCallable(self, thing):
self.assertIn('__call__', dir(thing))
@@ -226,7 +228,6 @@ class TestNetTest(unittest.TestCase):
@d.addCallback
def complete(result):
- print "IN here y0"
self.assertEqual(result, None)
self.assertEqual(director.successfulMeasurements, 20)
diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py
index 2acfdb0..c8a9a3a 100644
--- a/ooni/utils/geodata.py
+++ b/ooni/utils/geodata.py
@@ -5,8 +5,8 @@ from twisted.web.client import Agent
from twisted.internet import reactor, defer, protocol
from ooni.utils import log, net
-from ooni import config
from ooni.errors import GeoIPDataFilesNotFound
+from ooni.settings import config
try:
import pygeoip
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 141116e..067d6a6 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -9,7 +9,7 @@ from twisted.python.failure import Failure
from twisted.python.logfile import DailyLogFile
from ooni import otime
-from ooni import config
+from ooni.settings import config
## Get rid of the annoying "No route found for
## IPv6 destination warnings":
diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py
index 80dd1c2..e02de60 100644
--- a/ooni/utils/txscapy.py
+++ b/ooni/utils/txscapy.py
@@ -12,7 +12,7 @@ from zope.interface import implements
from scapy.config import conf
from ooni.utils import log
-from ooni import config
+from ooni.settings import config
class LibraryNotInstalledError(Exception):
pass
More information about the tor-commits
mailing list