commit c7f65ae7df71c469f9cf5db126f12bf1c11c49da Author: Arturo Filastò art@fuffa.org Date: Tue Apr 23 16:59:24 2013 +0200

Improve configuration management

* Install ooniprobe settings in the users home directory (~/.ooni) * Define the paths for ooniprobe data directories --- ooni/__init__.py | 2 - ooni/api/spec.py | 2 +- ooni/config.py | 98 ------------------------------------------- ooni/director.py | 15 ++++--- ooni/geoip.py | 3 +- ooni/managers.py | 18 +++++--- ooni/nettest.py | 2 +- ooni/oonicli.py | 11 ++++- ooni/oonid.py | 2 +- ooni/reporter.py | 2 +- ooni/settings.py | 92 ++++++++++++++++++++++++++++++++++++++++ ooni/tasks.py | 10 ++--- ooni/templates/httpt.py | 2 +- ooni/templates/scapyt.py | 2 +- ooni/tests/mocks.py | 2 +- ooni/tests/test_managers.py | 8 ++++ ooni/tests/test_nettest.py | 5 ++- ooni/utils/geodata.py | 2 +- ooni/utils/log.py | 2 +- ooni/utils/txscapy.py | 2 +- 20 files changed, 149 insertions(+), 133 deletions(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py index cc4bf03..815e16e 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,5 @@ # -*- encoding: utf-8 -*-

-from . import config -from . import kit from . import nettest from . import oonicli from . import reporter diff --git a/ooni/api/spec.py b/ooni/api/spec.py index af238f4..5b538b2 100644 --- a/ooni/api/spec.py +++ b/ooni/api/spec.py @@ -5,7 +5,7 @@ import types

from cyclone import web, escape

-from ooni import config +from ooni.settings import config

class InvalidInputFilename(Exception): pass diff --git a/ooni/config.py b/ooni/config.py deleted file mode 100644 index 5aeb49d..0000000 --- a/ooni/config.py +++ /dev/null @@ -1,98 +0,0 @@ -import os -import yaml - -from twisted.internet import reactor, threads, defer - -from ooni import otime -from ooni.utils import Storage - -class TestFilenameNotSet(Exception): - pass - -def get_root_path(): - this_directory = os.path.dirname(__file__) - root = os.path.join(this_directory, '..') - root = os.path.abspath(root) - return root - -def createConfigFile(): - """ - XXX implement me - """ - sample_config_file = os.path.join(get_root_path(), 'ooniprobe.conf.sample') - -def generatePcapFilename(): - if cmd_line_options['pcapfile']: - reports.pcap = cmd_line_options['pcapfile'] - else: - if cmd_line_options['test']: - test_filename = os.path.basename(cmd_line_options['test']) - else: - test_filename = os.path.basename(cmd_line_options['testdeck']) - - test_name = '.'.join(test_filename.split(".")[:-1]) - frm_str = "report_%s_"+otime.timestamp()+".%s" - reports.pcap = frm_str % (test_name, "pcap") - -class ConfigurationSetting(Storage): - def __init__(self, key): - config_file = os.path.join(get_root_path(), 'ooniprobe.conf') - try: - f = open(config_file) - except IOError: - createConfigFile() - raise Exception("Unable to open config file. "\ - "Copy ooniprobe.conf.sample to ooniprobe.conf") - - config_file_contents = '\n'.join(f.readlines()) - configuration = yaml.safe_load(config_file_contents) - - try: - for k, v in configuration[key].items(): - self[k] = v - except AttributeError: - pass - -basic = ConfigurationSetting('basic') -advanced = ConfigurationSetting('advanced') -privacy = ConfigurationSetting('privacy') -tor = ConfigurationSetting('tor') - -data_directory = os.path.join(get_root_path(), 'data') -nettest_directory = os.path.join(get_root_path(), 'nettests') -inputs_directory = os.path.join(get_root_path(), 'inputs') - -reports = Storage() -state = Storage() -scapyFactory = None -stateDict = None - -# XXX refactor this to use a database -resume_lock = defer.DeferredLock() - -cmd_line_options = None -resume_filename = None - -# XXX-Twisted this is used to check if we have started the reactor or not. It -# is necessary because if the tests are already concluded because we have -# resumed a test session then it will call reactor.run() even though there is -# no condition that will ever stop it. -# There should be a more twisted way of doing this. -start_reactor = True -tor_state = None -tor_control = None -config_file = None -sample_config_file = None -# This is used to store the probes IP address obtained via Tor -probe_ip = None -# This is used to keep track of the state of the sniffer -sniffer_running = None - -logging = True - -if not resume_filename: - resume_filename = os.path.join(get_root_path(), 'ooniprobe.resume') - try: - with open(resume_filename) as f: pass - except IOError as e: - with open(resume_filename, 'w+') as f: pass diff --git a/ooni/director.py b/ooni/director.py index 5f23668..1ef878c 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -3,13 +3,13 @@ import sys import os import re

-from ooni import config from ooni import geoip from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report from ooni.utils import log, checkForRoot from ooni.utils.net import randomFreePort -from ooni.nettest import NetTest +from ooni.nettest import NetTest, getNetTestInformation +from ooni.settings import config from ooni import errors

from txtorcon import TorConfig @@ -85,6 +85,10 @@ class Director(object):

self.torControlProtocol = None

+ # This deferred is fired once all the measurements and their reporting + # tasks are completed. + self.allTestsDone = defer.Deferred() + def getNetTests(self): nettests = {} def is_nettest(filename): @@ -108,16 +112,12 @@ class Director(object):

return nettests

- # This deferred is fired once all the measurements and their reporting - # tasks are completed. - self.allTestsDone = defer.Deferred() - @defer.inlineCallbacks def start(self): if config.privacy.includepcap: log.msg("Starting") if not config.reports.pcap: - config.reports.pcap = config.generatePcapFilename() + config.generate_pcap_filename() self.startSniffing()

if config.advanced.start_tor: @@ -324,7 +324,6 @@ class Director(object): log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)

d = launch_tor(tor_config, reactor, - tor_binary=config.advanced.tor_binary, progress_updates=updates) d.addCallback(setup_complete) d.addErrback(setup_failed) diff --git a/ooni/geoip.py b/ooni/geoip.py index 1e6f6e9..c1987fc 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -7,7 +7,8 @@ from ooni.utils.net import userAgents, BodyReceiver from twisted.internet import reactor, defer, protocol

from ooni.utils import log, net, checkForRoot -from ooni import config, errors +from ooni.settings import config +from ooni import errors

try: from pygeoip import GeoIP diff --git a/ooni/managers.py b/ooni/managers.py index 9f8366f..ff7c2f2 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -2,7 +2,7 @@ import itertools

from twisted.internet import defer from ooni.utils import log -from ooni import config +from ooni.settings import config

def makeIterable(item): """ @@ -141,8 +141,12 @@ class MeasurementManager(TaskManager): NetTest on the contrary is aware of the typology of measurements that it is dispatching as they are logically grouped by test file. """ - retries = config.advanced.measurement_retries - concurrency = config.advanced.measurement_concurrency + def __init__(self): + if config.advanced.measurement_retries: + self.retries = config.advanced.measurement_retries + if config.advanced.measurement_concurrency: + self.concurrency = config.advanced.measurement_concurrency + super(MeasurementManager, self).__init__()

def succeeded(self, result, measurement): log.debug("Successfully performed measurement %s" % measurement) @@ -152,8 +156,12 @@ class MeasurementManager(TaskManager): pass

class ReportEntryManager(TaskManager): - retries = config.advanced.reporting_retries - concurrency = config.advanced.reporting_concurrency + def __init__(self): + if config.advanced.reporting_retries: + self.retries = config.advanced.reporting_retries + if config.advanced.reporting_concurrency: + self.concurrency = config.advanced.reporting_concurrency + super(ReportEntryManager, self).__init__()

def succeeded(self, result, task): log.debug("Successfully performed report %s" % task) diff --git a/ooni/nettest.py b/ooni/nettest.py index d9bc94d..0d0e889 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -8,8 +8,8 @@ from twisted.python import usage, reflect from ooni import geoip from ooni.tasks import Measurement from ooni.utils import log, checkForRoot, geodata -from ooni import config from ooni import otime +from ooni.settings import config

from ooni import errors as e

diff --git a/ooni/oonicli.py b/ooni/oonicli.py index b5e8e27..40453b1 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -12,7 +12,7 @@ from twisted.python.util import spewer

from ooni import errors

-from ooni import config +from ooni.settings import config from ooni.director import Director from ooni.reporter import YAMLReporter, OONIBReporter from ooni.nettest import NetTestLoader, MissingRequiredOption @@ -40,6 +40,10 @@ class Options(usage.Options): ["logfile", "l", None, "log file name"], ["pcapfile", "O", None, "pcap file name"], ["parallelism", "p", "10", "input parallelism"], + ["configfile", "f", None, + "Specify a path to the ooniprobe configuration file"], + ["datadir", "d", None, + "Specify a path to the ooniprobe data directory"] ]

compData = usage.Completions( @@ -100,8 +104,11 @@ def runWithDirector(): test! """ global_options = parseOptions() - log.start(global_options['logfile']) + config.global_options = global_options + config.set_paths() + config.read_config_file()

+ log.start(global_options['logfile']) # contains (test_cases, options, cmd_line_options) test_list = [] if global_options['no-collector']: diff --git a/ooni/oonid.py b/ooni/oonid.py index dde768e..cc71d47 100644 --- a/ooni/oonid.py +++ b/ooni/oonid.py @@ -4,7 +4,7 @@ import random from twisted.application import service, internet from twisted.web import static, server

-from ooni import config +from ooni.settings import config from ooni.api.spec import oonidApplication from ooni.director import Director from ooni.reporter import YAMLReporter, OONIBReporter diff --git a/ooni/reporter.py b/ooni/reporter.py index b04b46b..109eccf 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -32,7 +32,7 @@ from ooni import otime from ooni.utils import geodata, pushFilenameStack from ooni.utils.net import BodyReceiver, StringProducer, userAgents

-from ooni import config +from ooni.settings import config

from ooni.tasks import ReportEntry, TaskTimedOut

diff --git a/ooni/settings.py b/ooni/settings.py new file mode 100644 index 0000000..acb7502 --- /dev/null +++ b/ooni/settings.py @@ -0,0 +1,92 @@ +import os +import yaml +from shutil import copyfile +from os.path import abspath, expanduser + +from twisted.internet import reactor, threads, defer + +from ooni import otime +from ooni.utils import Storage + +class OConfig(object): + def __init__(self): + self.global_options = {} + self.reports = Storage() + self.scapyFactory = None + self.tor_state = None + # This is used to store the probes IP address obtained via Tor + self.probe_ip = None + # This is used to keep track of the state of the sniffer + self.sniffer_running = None + self.logging = True + self.basic = Storage() + self.advanced = Storage() + self.tor = Storage() + self.privacy = Storage() + self.set_paths() + self.initialize_ooni_home() + + def set_paths(self): + if self.global_options.get('datadir'): + self.data_directory = abspath(expanduser(self.global_options['datadir'])) + else: + self.data_directory = '/usr/share/ooni/' + self.nettest_directory = os.path.join(self.data_directory, 'nettests') + + self.ooni_home = os.path.join(expanduser('~'), '.ooni') + self.inputs_directory = os.path.join(self.ooni_home, 'inputs') + + if self.global_options.get('configfile'): + config_file = global_options['configfile'] + else: + config_file = os.path.join('~', '.ooni', 'ooniprobe.conf') + self.config_file = expanduser(config_file) + + def initialize_ooni_home(self): + if not os.path.isdir(self.ooni_home): + print "Ooni home directory does not exist." + print "Creating it in '%s'." % self.ooni_home + os.mkdir(self.ooni_home) + os.mkdir(self.inputs_directory) + + def _create_config_file(self): + sample_config_file = os.path.join(self.data_directory, + 'ooniprobe.conf.sample') + target_config_file = os.path.join(self.ooni_home, + 'ooniprobe.conf') + print "Creating it for you in '%s'." % target_config_file + copyfile(sample_config_file, target_config_file) + + def read_config_file(self): + try: + with open(self.config_file) as f: pass + except IOError: + print "Configuration file does not exist." + self._create_config_file() + self.read_config_file() + + with open(self.config_file) as f: + config_file_contents = '\n'.join(f.readlines()) + configuration = yaml.safe_load(config_file_contents) + + for setting in ['basic', 'advanced', 'privacy', 'tor']: + try: + for k, v in configuration[setting].items(): + getattr(self, setting)[k] = v + except AttributeError: + pass + + def generate_pcap_filename(): + if self.global_options.get('pcapfile'): + self.reports.pcap = self.global_options['pcapfile'] + else: + if self.global_options.get('test'): + test_filename = os.path.basename(self.global_options['test']) + else: + test_filename = os.path.basename(self.global_options['testdeck']) + + test_name = '.'.join(test_filename.split(".")[:-1]) + frm_str = "report_%s_"+otime.timestamp()+".%s" + self.reports.pcap = frm_str % (test_name, "pcap") + +config = OConfig() diff --git a/ooni/tasks.py b/ooni/tasks.py index 829d11e..f686a9c 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -1,6 +1,6 @@ import time

-from ooni import config +from ooni.settings import config from twisted.internet import defer, reactor

class BaseTask(object): @@ -91,8 +91,6 @@ class TaskWithTimeout(BaseTask): return BaseTask.start(self)

class Measurement(TaskWithTimeout): - timeout = config.advanced.measurement_timeout - def __init__(self, test_class, test_method, test_input): """ test_class: @@ -117,6 +115,8 @@ class Measurement(TaskWithTimeout):

self.netTestMethod = getattr(self.testInstance, test_method)

+ if config.advanced.measurement_timeout: + self.timeout = config.advanced.measurement_timeout TaskWithTimeout.__init__(self)

def succeeded(self, result): @@ -130,12 +130,12 @@ class Measurement(TaskWithTimeout): return d

class ReportEntry(TaskWithTimeout): - timeout = config.advanced.reporting_timeout - def __init__(self, reporter, measurement): self.reporter = reporter self.measurement = measurement

+ if config.advanced.reporting_timeout: + self.timeout = config.advanced.reporting_timeout TaskWithTimeout.__init__(self)

def run(self): diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index e8891c7..0bca5df 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -13,7 +13,7 @@ from twisted.web._newclient import Request, Response, ResponseNeverReceived

from ooni.nettest import NetTestCase from ooni.utils import log -from ooni import config +from ooni.settings import config

from ooni.utils.net import BodyReceiver, StringProducer, userAgents

diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index d5d6564..fdc5a24 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -8,7 +8,7 @@ from scapy.all import send, sr, IP, TCP, config from ooni.reporter import createPacketReport from ooni.nettest import NetTestCase from ooni.utils import log -from ooni import config +from ooni.settings import config

from ooni.utils.txscapy import ScapySender, getDefaultIface, ScapyFactory from ooni.utils.txscapy import hasRawSocketPermission diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py index 4c4a015..f849344 100644 --- a/ooni/tests/mocks.py +++ b/ooni/tests/mocks.py @@ -1,7 +1,7 @@ from twisted.python import failure from twisted.internet import defer

-from ooni import config +from ooni.settings import config from ooni.tasks import BaseTask, TaskWithTimeout from ooni.nettest import NetTest from ooni.managers import TaskManager diff --git a/ooni/tests/test_managers.py b/ooni/tests/test_managers.py index e2af7b3..c290155 100644 --- a/ooni/tests/test_managers.py +++ b/ooni/tests/test_managers.py @@ -1,3 +1,5 @@ +import os + from twisted.trial import unittest from twisted.python import failure from twisted.internet import defer, task @@ -11,6 +13,8 @@ from ooni.tests.mocks import MockTimeoutOnceTask, MockFailTaskWithTimeout from ooni.tests.mocks import MockTaskManager, mockFailure, MockDirector from ooni.tests.mocks import MockNetTest, MockMeasurement, MockSuccessMeasurement from ooni.tests.mocks import MockFailMeasurement, MockFailOnceMeasurement +from ooni.settings import config +

class TestTaskManager(unittest.TestCase): timeout = 1 @@ -22,6 +26,10 @@ class TestTaskManager(unittest.TestCase): self.measurementManager.start()

self.clock = task.Clock() + data_dir = os.path.dirname(os.path.abspath(__file__)) + data_dir = os.path.join(data_dir, '..', '..', 'data') + config.global_options['datadir'] = data_dir + config.set_paths()

def schedule_successful_tasks(self, task_type, number=1): all_done = [] diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 3b59cc4..77b8a1c 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -11,7 +11,6 @@ from ooni.nettest import NetTestLoader, FailureToLoadNetTest from ooni.tasks import BaseTask

from ooni.director import Director - from ooni.managers import TaskManager

from ooni.tests.mocks import MockMeasurement, MockMeasurementFailOnce @@ -100,6 +99,9 @@ class TestNetTest(unittest.TestCase): for i in range(10): f.write("%s\n" % i)

+ from ooni.settings import config + config.read_config_file() + def assertCallable(self, thing): self.assertIn('__call__', dir(thing))

@@ -226,7 +228,6 @@ class TestNetTest(unittest.TestCase):

@d.addCallback def complete(result): - print "IN here y0" self.assertEqual(result, None) self.assertEqual(director.successfulMeasurements, 20)

diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index 2acfdb0..c8a9a3a 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -5,8 +5,8 @@ from twisted.web.client import Agent from twisted.internet import reactor, defer, protocol

from ooni.utils import log, net -from ooni import config from ooni.errors import GeoIPDataFilesNotFound +from ooni.settings import config

try: import pygeoip diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 141116e..067d6a6 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -9,7 +9,7 @@ from twisted.python.failure import Failure from twisted.python.logfile import DailyLogFile

from ooni import otime -from ooni import config +from ooni.settings import config

## Get rid of the annoying "No route found for ## IPv6 destination warnings": diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 80dd1c2..e02de60 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -12,7 +12,7 @@ from zope.interface import implements from scapy.config import conf

from ooni.utils import log -from ooni import config +from ooni.settings import config

class LibraryNotInstalledError(Exception): pass