commit 31bcae623b8a621aa941f284e67438168243c44a Author: Arturo Filastò arturo@filasto.net Date: Fri Sep 2 15:00:48 2016 +0200

Add support for parsing multiple config files

* Configuration files are parsed in a certain order and they override the default behavior. --- Vagrantfile | 5 + ooni/__init__.py | 2 +- ooni/scripts/ooniprobe_agent.py | 1 + ooni/settings.py | 125 ++++++++++++++----- ooni/tests/test_settings.py | 61 ++++++++++ ooni/ui/cli.py | 4 +- ooni/ui/web/client/index.html | 2 +- setup.py | 262 ++++++++++++++++++---------------------- 8 files changed, 285 insertions(+), 177 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile index 71f7c7a..8ff0bbd 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -81,4 +81,9 @@ Vagrant.configure("2") do |config| end end

+ config.vm.define "testing" do |testing| + testing.vm.network "forwarded_port", guest: 8842, host: 8142 + testing.vm.synced_folder ".", "/data/ooni-probe" + end + end diff --git a/ooni/__init__.py b/ooni/__init__.py index 9eecb67..d4b6f95 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*-

__author__ = "Open Observatory of Network Interference" -__version__ = "2.0.0-alpha.2" +__version__ = "2.0.0-alpha.3"

__all__ = [ 'agent', diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index f56f2df..3eb1d22 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -58,6 +58,7 @@ def start_agent(options=None): # a null log observer twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull', '--umask', '022'] + twistd_config = OoniprobeTwistdConfig() if options is not None: twistd_args.extend(options.twistd_args) diff --git a/ooni/settings.py b/ooni/settings.py index a0ef0a5..66bd212 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -3,6 +3,7 @@ import sys import yaml import errno import getpass +from pkg_resources import parse_version from ConfigParser import SafeConfigParser

from twisted.internet import defer, reactor @@ -10,6 +11,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint

from os.path import abspath, expanduser

+from ooni import __version__ as ooniprobe_version from ooni.utils import Storage, log, get_ooni_root

CONFIG_FILE_TEMPLATE = """\ @@ -19,8 +21,8 @@ CONFIG_FILE_TEMPLATE = """\

basic: # Where OONIProbe should be writing it's log file - logfile: {logfile} - loglevel: WARNING + # logfile: {logfile} + # loglevel: WARNING # The maximum amount of data to store on disk. Once the quota is reached, # we will start deleting older reports. # measurement_quota: 1G @@ -191,6 +193,37 @@ elif os.path.isfile(_SETTINGS_INI): if _ETC_PATH is not None: ETC_PATH = _ETC_PATH

+ +def _load_config_files_with_defaults(config_files, defaults): + """ + This takes care of reading the config files in reverse order (the first + item will have priority over the last element) and produce a + configuration that includes ONLY the options inside of the defaults + dictionary. + + :param config_files: a list of configuration file paths + :param defaults: the default values for the configuration file + :return: a configuration that is the result of reading the config files + and joining it with the default options. + """ + config_from_files = {} + configuration = {} + for config_file_path in reversed(config_files): + if not os.path.exists(config_file_path): + continue + with open(config_file_path) as in_file: + c = yaml.safe_load(in_file) + config_from_files.update(c) + + for category in defaults.keys(): + configuration[category] = {} + for k, v in defaults[category].items(): + try: + configuration[category][k] = config_from_files[category][k] + except KeyError: + configuration[category][k] = defaults[category][k] + return configuration + class OConfig(object): _custom_home = None

@@ -211,6 +244,10 @@ class OConfig(object): self.tor = Storage() self.privacy = Storage()

+ # In here we store the configuration files ordered by priority. + # First configuration file takes priority over the others. + self.config_files = [] + self.set_paths()

def is_initialized(self): @@ -225,6 +262,31 @@ class OConfig(object): with open(initialized_path, 'w+'): pass

@property + def last_run_version(self): + """ + :return: Version identifying the last run version of ooniprobe. + """ + last_run_version_path = os.path.join( + self.running_path, "last_run_version" + ) + if not os.path.exists(last_run_version_path): + return parse_version("0") + with open(last_run_version_path) as in_file: + last_run_version = in_file.read() + return parse_version(last_run_version) + + @property + def current_version(self): + return parse_version(ooniprobe_version) + + def set_last_run_version(self): + last_run_version_path = os.path.join( + self.running_path, "last_run_version" + ) + with open(last_run_version_path, "w") as out_file: + out_file.write(ooniprobe_version) + + @property def running_path(self): """ This is the directory used to store state application data. @@ -266,6 +328,10 @@ class OConfig(object): return VAR_LIB_PATH

@property + def user_config_file_path(self): + return os.path.join(self.running_path, 'ooniprobe.conf') + + @property def ooni_home(self): home = expanduser('~'+self.current_user) if os.getenv("HOME"): @@ -283,7 +349,8 @@ class OConfig(object):

def set_paths(self): self.nettest_directory = os.path.join(OONIPROBE_ROOT, 'nettests') - self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web','client') + self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web', + 'client')

self.inputs_directory = os.path.join(self.running_path, 'inputs') self.scheduler_directory = os.path.join(self.running_path, 'scheduler') @@ -297,12 +364,13 @@ class OConfig(object): self.measurements_directory = os.path.join(self.running_path, 'measurements')

+ self.config_files = [ + self.user_config_file_path, + '/etc/ooniprobe.conf' + ] if self.global_options.get('configfile'): config_file = self.global_options['configfile'] - self.config_file = expanduser(config_file) - else: - self.config_file = os.path.join(self.running_path, - 'ooniprobe.conf') + self.config_files.insert(0, expanduser(config_file))

if 'logfile' in self.basic: self.basic.logfile = expanduser( @@ -336,6 +404,14 @@ class OConfig(object): if exc.errno != errno.EEXIST: raise

+ # This means ooniprobe was installed for the first time or is coming + # from a 1.x series installation. We should configure the default deck. + if self.last_run_version.public == "0": + from ooni.deck.store import deck_store + DEFAULT_DECKS = ['web-full'] + for deck_id in DEFAULT_DECKS: + deck_store.enable(deck_id) + def create_config_file(self, include_ip=False, include_asn=True, include_country=True, should_upload=True, preferred_backend="onion"): @@ -354,9 +430,10 @@ class OConfig(object): should_upload = _bool_to_yaml(should_upload)

logfile = os.path.join(self.running_path, 'ooniprobe.log') - with open(self.config_file, 'w+') as out_file: + with open(self.user_config_file_path, 'w') as out_file: out_file.write( - CONFIG_FILE_TEMPLATE.format(logfile=logfile, + CONFIG_FILE_TEMPLATE.format( + logfile=logfile, include_ip=include_ip, include_asn=include_asn, include_country=include_country, @@ -366,23 +443,12 @@ class OConfig(object): self.read_config_file()

def read_config_file(self, check_incoherences=False): - configuration = {} - config_file = {} - log.debug("Reading config file from %s" % self.config_file) - if os.path.isfile(self.config_file): - with open(self.config_file) as f: - config_file_contents = '\n'.join(f.readlines()) - config_file = yaml.safe_load(config_file_contents) - - for category in defaults.keys(): - configuration[category] = {} - for k, v in defaults[category].items(): - try: - value = config_file.get(category, {})[k] - except KeyError: - value = v - configuration[category][k] = value - getattr(self, category)[k] = value + configuration = _load_config_files_with_defaults( + self.config_files, defaults) + + for category in configuration.keys(): + for key, value in configuration[category].items(): + getattr(self, category)[key] = value

self.set_paths() if check_incoherences: @@ -405,7 +471,8 @@ class OConfig(object): incoherent_pretty = ", ".join(incoherences[:-1]) + ' and ' + incoherences[-1] else: incoherent_pretty = incoherences[0] - log.err("You must set properly %s in %s." % (incoherent_pretty, self.config_file)) + log.err("You must set properly %s in %s." % (incoherent_pretty, + self.config_files[0])) raise errors.ConfigFileIncoherent

@defer.inlineCallbacks @@ -439,7 +506,3 @@ class OConfig(object): self.log_incoherences(incoherent)

config = OConfig() -if not os.path.isfile(config.config_file) \ - and os.path.isfile('/etc/ooniprobe.conf'): - config.global_options['configfile'] = '/etc/ooniprobe.conf' - config.set_paths() diff --git a/ooni/tests/test_settings.py b/ooni/tests/test_settings.py index f94ca14..30d9e19 100644 --- a/ooni/tests/test_settings.py +++ b/ooni/tests/test_settings.py @@ -1,4 +1,8 @@ +import os import random +import tempfile + +import yaml

from twisted.internet import defer, reactor from twisted.internet.protocol import Protocol, Factory @@ -10,6 +14,7 @@ from ooni import errors from ooni.utils import net from bases import ConfigTestCase

+from ooni.settings import _load_config_files_with_defaults

class TestSettings(ConfigTestCase): def setUp(self): @@ -133,3 +138,59 @@ class TestSettings(ConfigTestCase):

self.configuration['advanced']['interface'] = random.choice(get_if_list()) self.conf.check_incoherences(self.configuration) + + + def test_load_config_files(self): + defaults = { + 'cat1': { + 'key': 'value' + }, + 'cat2': { + 'key': 'value' + }, + 'cat3': { + 'key': 'value' + } + } + config_file_A = { + 'cat1': { + 'key': 'valueA' + }, + 'cat2': { + 'key': 'valueA', + 'invalid_key': 'ignored' + }, + 'invalid_category': { + 'ignored': 'ignored' + } + } + config_file_B = { + 'cat1': { + 'key': 'valueB' + } + } + temp_dir = tempfile.mkdtemp() + config_file_A_path = os.path.join(temp_dir, "configA.conf") + config_file_B_path = os.path.join(temp_dir, "configB.conf") + with open(config_file_A_path, 'w') as out_file: + yaml.safe_dump(config_file_A, out_file) + + with open(config_file_B_path, 'w') as out_file: + yaml.safe_dump(config_file_B, out_file) + + config = _load_config_files_with_defaults([config_file_B_path, + '/invalid/path/ignored.txt', + config_file_A_path], + defaults) + + self.assertEqual(config, { + 'cat1': { + 'key': 'valueB' + }, + 'cat2': { + 'key': 'valueA' + }, + 'cat3': { + 'key': 'value' + } + }) diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 3ad9605..e7a0d88 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -330,7 +330,7 @@ def createDeck(global_options, url=None): except errors.MissingRequiredOption as option_name: log.err('Missing required option: "%s"' % option_name) incomplete_net_test_loader = option_name.net_test_loader - print incomplete_net_test_loader.usageOptions().getUsage() + map(log.msg, incomplete_net_test_loader.usageOptions().getUsage().split("\n")) raise SystemExit(2)

except errors.NetTestNotFound as path: @@ -508,7 +508,7 @@ def runWithDaemonDirector(global_options): url=data['url'].encode('utf8')) # When the test has been completed, go back to waiting for a message. d.addCallback(readmsg, channel, queue_object, consumer_tag, counter+1) - except exceptions.AMQPError,v: + except exceptions.AMQPError, v: log.msg("Error") log.exception(v) finished.errback(v) diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index ad2dc50..ecb4cdb 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?f06164bd3b339e781c75"></script></body> + <script type="text/javascript" src="app.bundle.js?5432cc07ccbd2f5613be"></script></body> </html> diff --git a/setup.py b/setup.py index e5dd836..4c0bcfe 100644 --- a/setup.py +++ b/setup.py @@ -87,39 +87,54 @@ Have fun! from __future__ import print_function

import os -import shutil import tempfile -import subprocess from glob import glob

from ConfigParser import SafeConfigParser

from os.path import join as pj from setuptools import setup -from setuptools.command.install import install +from setuptools.command.install import install as InstallCommand

from ooni import __version__, __author__

-GEOIP_ASN_URL = "https://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz" -GEOIP_URL = "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat..." -TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip" - -def run_command(args, cwd=None): - try: - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - return None - stdout = p.communicate()[0].strip() - if p.returncode != 0: - return None - return stdout - -def is_lepidopter(): - if os.path.exists("/etc/default/lepidopter"): - return True - return False - -class OoniInstall(install): +CLASSIFIERS = """\ +Development Status :: 5 - Production/Stable +Environment :: Console +Framework :: Twisted +Intended Audience :: Developers +Intended Audience :: Education +Intended Audience :: End Users/Desktop +Intended Audience :: Information Technology +Intended Audience :: Science/Research +Intended Audience :: Telecommunications Industry +License :: OSI Approved :: BSD Licens +Programming Language :: Python +Programming Language :: Python :: 2 +Programming Language :: Python :: 2 :: Only +Programming Language :: Python :: 2.6 +Programming Language :: Python :: 2.7 +Operating System :: MacOS :: MacOS X +Operating System :: POSIX +Operating System :: POSIX :: BSD +Operating System :: POSIX :: BSD :: BSD/OS +Operating System :: POSIX :: BSD :: FreeBSD +Operating System :: POSIX :: BSD :: NetBSD +Operating System :: POSIX :: BSD :: OpenBSD +Operating System :: POSIX :: Linux +Operating System :: Unix +Topic :: Scientific/Engineering :: Information Analysis +Topic :: Security +Topic :: Security :: Cryptography +Topic :: Software Development :: Libraries :: Application Frameworks +Topic :: Software Development :: Libraries :: Python Modules +Topic :: Software Development :: Testing +Topic :: Software Development :: Testing :: Traffic Generation +Topic :: System :: Networking :: Monitoring +""" + + +class OoniInstall(InstallCommand): def gen_config(self, share_path): config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample") o = open(config_file, "w+") @@ -164,127 +179,90 @@ class OoniInstall(install): except OSError: pass

- def ooniresources(self): - from ooni.resources import check_for_update - from twisted.internet import task - task.react(lambda _: check_for_update()) + def pre_install(self): + prefix = os.path.abspath(self.prefix) + self.set_data_files(prefix)

- def update_lepidopter_config(self): - try: - shutil.copyfile("data/configs/lepidopter-ooniprobe.conf", - "/etc/ooniprobe/ooniprobe.conf") - shutil.copyfile("data/configs/lepidopter-oonireport.conf", - "/etc/ooniprobe/oonireport.conf") - except Exception: - print("ERR: Failed to copy configuration files to /etc/ooniprobe/") + def post_install(self): + pass

def run(self): - prefix = os.path.abspath(self.prefix) - self.set_data_files(prefix) + self.pre_install() self.do_egg_install() - self.ooniresources() - if is_lepidopter(): - self.update_lepidopter_config() - -setup_requires = ['twisted', 'pyyaml'] -install_requires = [] -dependency_links = [] -data_files = [] -packages = [ - 'ooni', - 'ooni.agent', - 'ooni.common', - 'ooni.contrib', - 'ooni.contrib.dateutil', - 'ooni.contrib.dateutil.tz', - 'ooni.deck', - 'ooni.kit', - 'ooni.nettests', - 'ooni.nettests.manipulation', - 'ooni.nettests.experimental', - 'ooni.nettests.scanning', - 'ooni.nettests.blocking', - 'ooni.nettests.third_party', - 'ooni.scripts', - 'ooni.templates', - 'ooni.tests', - 'ooni.ui', - 'ooni.ui.web', - 'ooni.utils' -] - -with open('requirements.txt') as f: - for line in f: - if line.startswith("#"): - continue - if line.startswith('https'): - dependency_links.append(line) - continue - install_requires.append(line) - -setup( - name="ooniprobe", - version=__version__, - author=__author__, - author_email="contact@openobservatory.org", - description="Network measurement tool for" - "identifying traffic manipulation and blocking.", - long_description=__doc__, - license='BSD 2 clause', - url="https://ooni.torproject.org/", - package_dir={'ooni': 'ooni'}, - data_files=data_files, - packages=packages, - include_package_data=True, - dependency_links=dependency_links, - install_requires=install_requires, - setup_requires=setup_requires, - zip_safe=False, - entry_points={ - 'console_scripts': [ - 'ooniresources = ooni.scripts.ooniresources:run', # This is deprecated - 'oonideckgen = ooni.scripts.oonideckgen:run', # This is deprecated - - 'ooniprobe = ooni.scripts.ooniprobe:run', - 'oonireport = ooni.scripts.oonireport:run', - 'ooniprobe-agent = ooni.scripts.ooniprobe_agent:run' - ] - }, - cmdclass={ - "install": OoniInstall - }, - classifiers=( - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Framework :: Twisted", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Intended Audience :: End Users/Desktop", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Intended Audience :: Telecommunications Industry", - "License :: OSI Approved :: BSD License" - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2 :: Only", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX", - "Operating System :: POSIX :: BSD", - "Operating System :: POSIX :: BSD :: BSD/OS", - "Operating System :: POSIX :: BSD :: FreeBSD", - "Operating System :: POSIX :: BSD :: NetBSD", - "Operating System :: POSIX :: BSD :: OpenBSD", - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - "Topic :: Scientific/Engineering :: Information Analysis", - "Topic :: Security", - "Topic :: Security :: Cryptography", - "Topic :: Software Development :: Libraries :: Application Frameworks", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Software Development :: Testing", - "Topic :: Software Development :: Testing :: Traffic Generation", - "Topic :: System :: Networking :: Monitoring", + self.post_install() + +def setup_package(): + setup_requires = [] + install_requires = [] + dependency_links = [] + data_files = [] + packages = [ + 'ooni', + 'ooni.agent', + 'ooni.common', + 'ooni.contrib', + 'ooni.contrib.dateutil', + 'ooni.contrib.dateutil.tz', + 'ooni.deck', + 'ooni.kit', + 'ooni.nettests', + 'ooni.nettests.manipulation', + 'ooni.nettests.experimental', + 'ooni.nettests.scanning', + 'ooni.nettests.blocking', + 'ooni.nettests.third_party', + 'ooni.scripts', + 'ooni.templates', + 'ooni.tests', + 'ooni.ui', + 'ooni.ui.web', + 'ooni.utils' + ] + + with open('requirements.txt') as f: + for line in f: + if line.startswith("#"): + continue + if line.startswith('https'): + dependency_links.append(line) + continue + install_requires.append(line) + + metadata = dict( + name="ooniprobe", + version=__version__, + author=__author__, + author_email="contact@openobservatory.org", + description="Network measurement tool for" + "identifying traffic manipulation and blocking.", + long_description=__doc__, + license='BSD 2 clause', + url="https://ooni.torproject.org/", + package_dir={'ooni': 'ooni'}, + data_files=data_files, + packages=packages, + include_package_data=True, + dependency_links=dependency_links, + install_requires=install_requires, + setup_requires=setup_requires, + zip_safe=False, + entry_points={ + 'console_scripts': [ + 'ooniresources = ooni.scripts.ooniresources:run', # This is deprecated + 'oonideckgen = ooni.scripts.oonideckgen:run', # This is deprecated + + 'ooniprobe = ooni.scripts.ooniprobe:run', + 'oonireport = ooni.scripts.oonireport:run', + 'ooniprobe-agent = ooni.scripts.ooniprobe_agent:run' + ] + }, + cmdclass={ + "install": OoniInstall + }, + classifiers=[c for c in CLASSIFIERS.split('\n') if c] ) -) + + setup(**metadata) + +if __name__ == "__main__": + setup_package()