September 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Make settings parsing more robust
by art＠torproject.org 19 Sep '16

19 Sep '16

commit b334679f58c835b7a93b6f93662b7cf7aab80240 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 2 17:35:36 2016 +0200 Make settings parsing more robust * Expand test case to cover more edge cases --- ooni/settings.py | 10 +++++++--- ooni/tests/test_settings.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ooni/settings.py b/ooni/settings.py index 66bd212..9f24f9b 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -89,7 +89,7 @@ tor: # This is the timeout after which we consider to to not have # bootstrapped properly. #timeout: 200 - torrc: + #torrc: #HTTPProxy: host:port #HTTPProxyAuthenticator: user:password #HTTPSProxy: host:port @@ -213,14 +213,18 @@ def _load_config_files_with_defaults(config_files, defaults): continue with open(config_file_path) as in_file: c = yaml.safe_load(in_file) - config_from_files.update(c) + for category in c.keys(): + if c[category] is None: + continue + config_from_files[category] = config_from_files.get(category, {}) + config_from_files[category].update(c[category]) for category in defaults.keys(): configuration[category] = {} for k, v in defaults[category].items(): try: configuration[category][k] = config_from_files[category][k] - except KeyError: + except (KeyError, TypeError): configuration[category][k] = defaults[category][k] return configuration diff --git a/ooni/tests/test_settings.py b/ooni/tests/test_settings.py index 30d9e19..b88aa68 100644 --- a/ooni/tests/test_settings.py +++ b/ooni/tests/test_settings.py @@ -146,7 +146,8 @@ class TestSettings(ConfigTestCase): 'key': 'value' }, 'cat2': { - 'key': 'value' + 'key': 'value', + 'key2': 'value2' }, 'cat3': { 'key': 'value' @@ -162,11 +163,15 @@ class TestSettings(ConfigTestCase): }, 'invalid_category': { 'ignored': 'ignored' - } + }, + 'cat3': None } config_file_B = { 'cat1': { 'key': 'valueB' + }, + 'cat2': { + 'key2': 'value2B' } } temp_dir = tempfile.mkdtemp() @@ -188,7 +193,8 @@ class TestSettings(ConfigTestCase): 'key': 'valueB' }, 'cat2': { - 'key': 'valueA' + 'key': 'valueA', + 'key2': 'value2B' }, 'cat3': { 'key': 'value'

1 0

[ooni-probe/master] Add support for parsing multiple config files
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 31bcae623b8a621aa941f284e67438168243c44a Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 2 15:00:48 2016 +0200 Add support for parsing multiple config files * Configuration files are parsed in a certain order and they override the default behavior. --- Vagrantfile | 5 + ooni/__init__.py | 2 +- ooni/scripts/ooniprobe_agent.py | 1 + ooni/settings.py | 125 ++++++++++++++----- ooni/tests/test_settings.py | 61 ++++++++++ ooni/ui/cli.py | 4 +- ooni/ui/web/client/index.html | 2 +- setup.py | 262 ++++++++++++++++++---------------------- 8 files changed, 285 insertions(+), 177 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 71f7c7a..8ff0bbd 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -81,4 +81,9 @@ Vagrant.configure("2") do |config| end end + config.vm.define "testing" do |testing| + testing.vm.network "forwarded_port", guest: 8842, host: 8142 + testing.vm.synced_folder ".", "/data/ooni-probe" + end + end diff --git a/ooni/__init__.py b/ooni/__init__.py index 9eecb67..d4b6f95 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "2.0.0-alpha.2" +__version__ = "2.0.0-alpha.3" __all__ = [ 'agent', diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index f56f2df..3eb1d22 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -58,6 +58,7 @@ def start_agent(options=None): # a null log observer twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull', '--umask', '022'] + twistd_config = OoniprobeTwistdConfig() if options is not None: twistd_args.extend(options.twistd_args) diff --git a/ooni/settings.py b/ooni/settings.py index a0ef0a5..66bd212 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -3,6 +3,7 @@ import sys import yaml import errno import getpass +from pkg_resources import parse_version from ConfigParser import SafeConfigParser from twisted.internet import defer, reactor @@ -10,6 +11,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from os.path import abspath, expanduser +from ooni import __version__ as ooniprobe_version from ooni.utils import Storage, log, get_ooni_root CONFIG_FILE_TEMPLATE = """\ @@ -19,8 +21,8 @@ CONFIG_FILE_TEMPLATE = """\ basic: # Where OONIProbe should be writing it's log file - logfile: {logfile} - loglevel: WARNING + # logfile: {logfile} + # loglevel: WARNING # The maximum amount of data to store on disk. Once the quota is reached, # we will start deleting older reports. # measurement_quota: 1G @@ -191,6 +193,37 @@ elif os.path.isfile(_SETTINGS_INI): if _ETC_PATH is not None: ETC_PATH = _ETC_PATH + +def _load_config_files_with_defaults(config_files, defaults): + """ + This takes care of reading the config files in reverse order (the first + item will have priority over the last element) and produce a + configuration that includes ONLY the options inside of the defaults + dictionary. + + :param config_files: a list of configuration file paths + :param defaults: the default values for the configuration file + :return: a configuration that is the result of reading the config files + and joining it with the default options. + """ + config_from_files = {} + configuration = {} + for config_file_path in reversed(config_files): + if not os.path.exists(config_file_path): + continue + with open(config_file_path) as in_file: + c = yaml.safe_load(in_file) + config_from_files.update(c) + + for category in defaults.keys(): + configuration[category] = {} + for k, v in defaults[category].items(): + try: + configuration[category][k] = config_from_files[category][k] + except KeyError: + configuration[category][k] = defaults[category][k] + return configuration + class OConfig(object): _custom_home = None @@ -211,6 +244,10 @@ class OConfig(object): self.tor = Storage() self.privacy = Storage() + # In here we store the configuration files ordered by priority. + # First configuration file takes priority over the others. + self.config_files = [] + self.set_paths() def is_initialized(self): @@ -225,6 +262,31 @@ class OConfig(object): with open(initialized_path, 'w+'): pass @property + def last_run_version(self): + """ + :return: Version identifying the last run version of ooniprobe. + """ + last_run_version_path = os.path.join( + self.running_path, "last_run_version" + ) + if not os.path.exists(last_run_version_path): + return parse_version("0") + with open(last_run_version_path) as in_file: + last_run_version = in_file.read() + return parse_version(last_run_version) + + @property + def current_version(self): + return parse_version(ooniprobe_version) + + def set_last_run_version(self): + last_run_version_path = os.path.join( + self.running_path, "last_run_version" + ) + with open(last_run_version_path, "w") as out_file: + out_file.write(ooniprobe_version) + + @property def running_path(self): """ This is the directory used to store state application data. @@ -266,6 +328,10 @@ class OConfig(object): return VAR_LIB_PATH @property + def user_config_file_path(self): + return os.path.join(self.running_path, 'ooniprobe.conf') + + @property def ooni_home(self): home = expanduser('~'+self.current_user) if os.getenv("HOME"): @@ -283,7 +349,8 @@ class OConfig(object): def set_paths(self): self.nettest_directory = os.path.join(OONIPROBE_ROOT, 'nettests') - self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web','client') + self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web', + 'client') self.inputs_directory = os.path.join(self.running_path, 'inputs') self.scheduler_directory = os.path.join(self.running_path, 'scheduler') @@ -297,12 +364,13 @@ class OConfig(object): self.measurements_directory = os.path.join(self.running_path, 'measurements') + self.config_files = [ + self.user_config_file_path, + '/etc/ooniprobe.conf' + ] if self.global_options.get('configfile'): config_file = self.global_options['configfile'] - self.config_file = expanduser(config_file) - else: - self.config_file = os.path.join(self.running_path, - 'ooniprobe.conf') + self.config_files.insert(0, expanduser(config_file)) if 'logfile' in self.basic: self.basic.logfile = expanduser( @@ -336,6 +404,14 @@ class OConfig(object): if exc.errno != errno.EEXIST: raise + # This means ooniprobe was installed for the first time or is coming + # from a 1.x series installation. We should configure the default deck. + if self.last_run_version.public == "0": + from ooni.deck.store import deck_store + DEFAULT_DECKS = ['web-full'] + for deck_id in DEFAULT_DECKS: + deck_store.enable(deck_id) + def create_config_file(self, include_ip=False, include_asn=True, include_country=True, should_upload=True, preferred_backend="onion"): @@ -354,9 +430,10 @@ class OConfig(object): should_upload = _bool_to_yaml(should_upload) logfile = os.path.join(self.running_path, 'ooniprobe.log') - with open(self.config_file, 'w+') as out_file: + with open(self.user_config_file_path, 'w') as out_file: out_file.write( - CONFIG_FILE_TEMPLATE.format(logfile=logfile, + CONFIG_FILE_TEMPLATE.format( + logfile=logfile, include_ip=include_ip, include_asn=include_asn, include_country=include_country, @@ -366,23 +443,12 @@ class OConfig(object): self.read_config_file() def read_config_file(self, check_incoherences=False): - configuration = {} - config_file = {} - log.debug("Reading config file from %s" % self.config_file) - if os.path.isfile(self.config_file): - with open(self.config_file) as f: - config_file_contents = '\n'.join(f.readlines()) - config_file = yaml.safe_load(config_file_contents) - - for category in defaults.keys(): - configuration[category] = {} - for k, v in defaults[category].items(): - try: - value = config_file.get(category, {})[k] - except KeyError: - value = v - configuration[category][k] = value - getattr(self, category)[k] = value + configuration = _load_config_files_with_defaults( + self.config_files, defaults) + + for category in configuration.keys(): + for key, value in configuration[category].items(): + getattr(self, category)[key] = value self.set_paths() if check_incoherences: @@ -405,7 +471,8 @@ class OConfig(object): incoherent_pretty = ", ".join(incoherences[:-1]) + ' and ' + incoherences[-1] else: incoherent_pretty = incoherences[0] - log.err("You must set properly %s in %s." % (incoherent_pretty, self.config_file)) + log.err("You must set properly %s in %s." % (incoherent_pretty, + self.config_files[0])) raise errors.ConfigFileIncoherent @defer.inlineCallbacks @@ -439,7 +506,3 @@ class OConfig(object): self.log_incoherences(incoherent) config = OConfig() -if not os.path.isfile(config.config_file) \ - and os.path.isfile('/etc/ooniprobe.conf'): - config.global_options['configfile'] = '/etc/ooniprobe.conf' - config.set_paths() diff --git a/ooni/tests/test_settings.py b/ooni/tests/test_settings.py index f94ca14..30d9e19 100644 --- a/ooni/tests/test_settings.py +++ b/ooni/tests/test_settings.py @@ -1,4 +1,8 @@ +import os import random +import tempfile + +import yaml from twisted.internet import defer, reactor from twisted.internet.protocol import Protocol, Factory @@ -10,6 +14,7 @@ from ooni import errors from ooni.utils import net from bases import ConfigTestCase +from ooni.settings import _load_config_files_with_defaults class TestSettings(ConfigTestCase): def setUp(self): @@ -133,3 +138,59 @@ class TestSettings(ConfigTestCase): self.configuration['advanced']['interface'] = random.choice(get_if_list()) self.conf.check_incoherences(self.configuration) + + + def test_load_config_files(self): + defaults = { + 'cat1': { + 'key': 'value' + }, + 'cat2': { + 'key': 'value' + }, + 'cat3': { + 'key': 'value' + } + } + config_file_A = { + 'cat1': { + 'key': 'valueA' + }, + 'cat2': { + 'key': 'valueA', + 'invalid_key': 'ignored' + }, + 'invalid_category': { + 'ignored': 'ignored' + } + } + config_file_B = { + 'cat1': { + 'key': 'valueB' + } + } + temp_dir = tempfile.mkdtemp() + config_file_A_path = os.path.join(temp_dir, "configA.conf") + config_file_B_path = os.path.join(temp_dir, "configB.conf") + with open(config_file_A_path, 'w') as out_file: + yaml.safe_dump(config_file_A, out_file) + + with open(config_file_B_path, 'w') as out_file: + yaml.safe_dump(config_file_B, out_file) + + config = _load_config_files_with_defaults([config_file_B_path, + '/invalid/path/ignored.txt', + config_file_A_path], + defaults) + + self.assertEqual(config, { + 'cat1': { + 'key': 'valueB' + }, + 'cat2': { + 'key': 'valueA' + }, + 'cat3': { + 'key': 'value' + } + }) diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 3ad9605..e7a0d88 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -330,7 +330,7 @@ def createDeck(global_options, url=None): except errors.MissingRequiredOption as option_name: log.err('Missing required option: "%s"' % option_name) incomplete_net_test_loader = option_name.net_test_loader - print incomplete_net_test_loader.usageOptions().getUsage() + map(log.msg, incomplete_net_test_loader.usageOptions().getUsage().split("\n")) raise SystemExit(2) except errors.NetTestNotFound as path: @@ -508,7 +508,7 @@ def runWithDaemonDirector(global_options): url=data['url'].encode('utf8')) # When the test has been completed, go back to waiting for a message. d.addCallback(readmsg, channel, queue_object, consumer_tag, counter+1) - except exceptions.AMQPError,v: + except exceptions.AMQPError, v: log.msg("Error") log.exception(v) finished.errback(v) diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index ad2dc50..ecb4cdb 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?f06164bd3b339e781c75"></script></body> + <script type="text/javascript" src="app.bundle.js?5432cc07ccbd2f5613be"></script></body> </html> diff --git a/setup.py b/setup.py index e5dd836..4c0bcfe 100644 --- a/setup.py +++ b/setup.py @@ -87,39 +87,54 @@ Have fun! from __future__ import print_function import os -import shutil import tempfile -import subprocess from glob import glob from ConfigParser import SafeConfigParser from os.path import join as pj from setuptools import setup -from setuptools.command.install import install +from setuptools.command.install import install as InstallCommand from ooni import __version__, __author__ -GEOIP_ASN_URL = "https://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz" -GEOIP_URL = "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.da…" -TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip" - -def run_command(args, cwd=None): - try: - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - return None - stdout = p.communicate()[0].strip() - if p.returncode != 0: - return None - return stdout - -def is_lepidopter(): - if os.path.exists("/etc/default/lepidopter"): - return True - return False - -class OoniInstall(install): +CLASSIFIERS = """\ +Development Status :: 5 - Production/Stable +Environment :: Console +Framework :: Twisted +Intended Audience :: Developers +Intended Audience :: Education +Intended Audience :: End Users/Desktop +Intended Audience :: Information Technology +Intended Audience :: Science/Research +Intended Audience :: Telecommunications Industry +License :: OSI Approved :: BSD Licens +Programming Language :: Python +Programming Language :: Python :: 2 +Programming Language :: Python :: 2 :: Only +Programming Language :: Python :: 2.6 +Programming Language :: Python :: 2.7 +Operating System :: MacOS :: MacOS X +Operating System :: POSIX +Operating System :: POSIX :: BSD +Operating System :: POSIX :: BSD :: BSD/OS +Operating System :: POSIX :: BSD :: FreeBSD +Operating System :: POSIX :: BSD :: NetBSD +Operating System :: POSIX :: BSD :: OpenBSD +Operating System :: POSIX :: Linux +Operating System :: Unix +Topic :: Scientific/Engineering :: Information Analysis +Topic :: Security +Topic :: Security :: Cryptography +Topic :: Software Development :: Libraries :: Application Frameworks +Topic :: Software Development :: Libraries :: Python Modules +Topic :: Software Development :: Testing +Topic :: Software Development :: Testing :: Traffic Generation +Topic :: System :: Networking :: Monitoring +""" + + +class OoniInstall(InstallCommand): def gen_config(self, share_path): config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample") o = open(config_file, "w+") @@ -164,127 +179,90 @@ class OoniInstall(install): except OSError: pass - def ooniresources(self): - from ooni.resources import check_for_update - from twisted.internet import task - task.react(lambda _: check_for_update()) + def pre_install(self): + prefix = os.path.abspath(self.prefix) + self.set_data_files(prefix) - def update_lepidopter_config(self): - try: - shutil.copyfile("data/configs/lepidopter-ooniprobe.conf", - "/etc/ooniprobe/ooniprobe.conf") - shutil.copyfile("data/configs/lepidopter-oonireport.conf", - "/etc/ooniprobe/oonireport.conf") - except Exception: - print("ERR: Failed to copy configuration files to /etc/ooniprobe/") + def post_install(self): + pass def run(self): - prefix = os.path.abspath(self.prefix) - self.set_data_files(prefix) + self.pre_install() self.do_egg_install() - self.ooniresources() - if is_lepidopter(): - self.update_lepidopter_config() - -setup_requires = ['twisted', 'pyyaml'] -install_requires = [] -dependency_links = [] -data_files = [] -packages = [ - 'ooni', - 'ooni.agent', - 'ooni.common', - 'ooni.contrib', - 'ooni.contrib.dateutil', - 'ooni.contrib.dateutil.tz', - 'ooni.deck', - 'ooni.kit', - 'ooni.nettests', - 'ooni.nettests.manipulation', - 'ooni.nettests.experimental', - 'ooni.nettests.scanning', - 'ooni.nettests.blocking', - 'ooni.nettests.third_party', - 'ooni.scripts', - 'ooni.templates', - 'ooni.tests', - 'ooni.ui', - 'ooni.ui.web', - 'ooni.utils' -] - -with open('requirements.txt') as f: - for line in f: - if line.startswith("#"): - continue - if line.startswith('https'): - dependency_links.append(line) - continue - install_requires.append(line) - -setup( - name="ooniprobe", - version=__version__, - author=__author__, - author_email="contact(a)openobservatory.org", - description="Network measurement tool for" - "identifying traffic manipulation and blocking.", - long_description=__doc__, - license='BSD 2 clause', - url="https://ooni.torproject.org/", - package_dir={'ooni': 'ooni'}, - data_files=data_files, - packages=packages, - include_package_data=True, - dependency_links=dependency_links, - install_requires=install_requires, - setup_requires=setup_requires, - zip_safe=False, - entry_points={ - 'console_scripts': [ - 'ooniresources = ooni.scripts.ooniresources:run', # This is deprecated - 'oonideckgen = ooni.scripts.oonideckgen:run', # This is deprecated - - 'ooniprobe = ooni.scripts.ooniprobe:run', - 'oonireport = ooni.scripts.oonireport:run', - 'ooniprobe-agent = ooni.scripts.ooniprobe_agent:run' - ] - }, - cmdclass={ - "install": OoniInstall - }, - classifiers=( - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Framework :: Twisted", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Intended Audience :: End Users/Desktop", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Intended Audience :: Telecommunications Industry", - "License :: OSI Approved :: BSD License" - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2 :: Only", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX", - "Operating System :: POSIX :: BSD", - "Operating System :: POSIX :: BSD :: BSD/OS", - "Operating System :: POSIX :: BSD :: FreeBSD", - "Operating System :: POSIX :: BSD :: NetBSD", - "Operating System :: POSIX :: BSD :: OpenBSD", - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - "Topic :: Scientific/Engineering :: Information Analysis", - "Topic :: Security", - "Topic :: Security :: Cryptography", - "Topic :: Software Development :: Libraries :: Application Frameworks", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Software Development :: Testing", - "Topic :: Software Development :: Testing :: Traffic Generation", - "Topic :: System :: Networking :: Monitoring", + self.post_install() + +def setup_package(): + setup_requires = [] + install_requires = [] + dependency_links = [] + data_files = [] + packages = [ + 'ooni', + 'ooni.agent', + 'ooni.common', + 'ooni.contrib', + 'ooni.contrib.dateutil', + 'ooni.contrib.dateutil.tz', + 'ooni.deck', + 'ooni.kit', + 'ooni.nettests', + 'ooni.nettests.manipulation', + 'ooni.nettests.experimental', + 'ooni.nettests.scanning', + 'ooni.nettests.blocking', + 'ooni.nettests.third_party', + 'ooni.scripts', + 'ooni.templates', + 'ooni.tests', + 'ooni.ui', + 'ooni.ui.web', + 'ooni.utils' + ] + + with open('requirements.txt') as f: + for line in f: + if line.startswith("#"): + continue + if line.startswith('https'): + dependency_links.append(line) + continue + install_requires.append(line) + + metadata = dict( + name="ooniprobe", + version=__version__, + author=__author__, + author_email="contact(a)openobservatory.org", + description="Network measurement tool for" + "identifying traffic manipulation and blocking.", + long_description=__doc__, + license='BSD 2 clause', + url="https://ooni.torproject.org/", + package_dir={'ooni': 'ooni'}, + data_files=data_files, + packages=packages, + include_package_data=True, + dependency_links=dependency_links, + install_requires=install_requires, + setup_requires=setup_requires, + zip_safe=False, + entry_points={ + 'console_scripts': [ + 'ooniresources = ooni.scripts.ooniresources:run', # This is deprecated + 'oonideckgen = ooni.scripts.oonideckgen:run', # This is deprecated + + 'ooniprobe = ooni.scripts.ooniprobe:run', + 'oonireport = ooni.scripts.oonireport:run', + 'ooniprobe-agent = ooni.scripts.ooniprobe_agent:run' + ] + }, + cmdclass={ + "install": OoniInstall + }, + classifiers=[c for c in CLASSIFIERS.split('\n') if c] ) -) + + setup(**metadata) + +if __name__ == "__main__": + setup_package()

1 0

[ooni-probe/master] Configure default decks when the scheduler is run for the first time.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 4e777252050d8b0266ce726ee8e43abfb6eaeb21 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 2 17:36:14 2016 +0200 Configure default decks when the scheduler is run for the first time. * Add hook for running an operation when the task is run for the first time. --- ooni/agent/scheduler.py | 28 ++++++++++++++++++++++++++-- ooni/deck/store.py | 3 +++ ooni/director.py | 2 +- ooni/settings.py | 33 --------------------------------- 4 files changed, 30 insertions(+), 36 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 54551f6..7c2212b 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -12,7 +12,7 @@ from ooni.scripts import oonireport from ooni import resources from ooni.utils import log, SHORT_DATE from ooni.utils.files import human_size_to_bytes, directory_usage -from ooni.deck.store import input_store, deck_store +from ooni.deck.store import input_store, deck_store, DEFAULT_DECKS from ooni.settings import config from ooni.contrib import croniter from ooni.geoip import probe_ip @@ -41,6 +41,10 @@ class FileSystemlockAndMutex(object): self._fs_lock.unlock() self._mutex.release() +# We use this date to indicate that the scheduled task has never run. +# Easter egg, try to see what is special about this date :)? +CANARY_DATE = datetime(1957, 8, 4) + class DidNotRun(Exception): pass @@ -76,7 +80,7 @@ class ScheduledTask(object): def last_run(self): self._last_run.restat(False) if not self._last_run.exists(): - return datetime.fromtimestamp(0) + return CANARY_DATE with self._last_run.open('r') as in_file: date_str = in_file.read() return datetime.strptime(date_str, self._time_format) @@ -89,6 +93,13 @@ class ScheduledTask(object): def task(self): raise NotImplemented + def first_run(self): + """ + This hook is called if it's the first time a particular scheduled + operation is run. + """ + pass + @defer.inlineCallbacks def run(self): yield self._last_run_lock.acquire() @@ -96,6 +107,8 @@ class ScheduledTask(object): self._last_run_lock.release() raise DidNotRun try: + if self.last_run == CANARY_DATE: + yield defer.maybeDeferred(self.first_run) yield self.task() self._update_last_run() except: @@ -240,6 +253,13 @@ class RefreshDeckList(ScheduledTask): self.scheduler = scheduler super(RefreshDeckList, self).__init__(schedule, identifier) + def first_run(self): + """ + On first run we enable the default decks. + """ + for deck_id in DEFAULT_DECKS: + deck_store.enable(deck_id) + def task(self): self.scheduler.refresh_deck_list() @@ -302,6 +322,10 @@ class SchedulerService(service.MultiService): if isinstance(scheduled_task, RunDeck): self._scheduled_tasks.remove(scheduled_task) + if not config.is_initialized(): + # Disable scheduling measurements if we are not initialized. + return + for deck_id, deck in deck_store.list_enabled(): if deck.schedule is None: continue diff --git a/ooni/deck/store.py b/ooni/deck/store.py index c938d56..bd852c7 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -11,6 +11,9 @@ from ooni.otime import timestampNowISO8601UTC from ooni.resources import check_for_update from ooni.settings import config +# These are the decks to be run by default. +DEFAULT_DECKS = ['web-full'] + class InputNotFound(Exception): pass diff --git a/ooni/director.py b/ooni/director.py index ab31bf5..cb2bdf0 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -398,7 +398,7 @@ class Director(object): self._tor_starting.callback(self._tor_state) except Exception as exc: log.err("Failed to start tor") - log.exc(exc) + log.exception(exc) self._tor_starting.errback(Failure(exc)) elif config.tor.control_port and config.tor_state is None: diff --git a/ooni/settings.py b/ooni/settings.py index 9f24f9b..a1c55fc 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -266,31 +266,6 @@ class OConfig(object): with open(initialized_path, 'w+'): pass @property - def last_run_version(self): - """ - :return: Version identifying the last run version of ooniprobe. - """ - last_run_version_path = os.path.join( - self.running_path, "last_run_version" - ) - if not os.path.exists(last_run_version_path): - return parse_version("0") - with open(last_run_version_path) as in_file: - last_run_version = in_file.read() - return parse_version(last_run_version) - - @property - def current_version(self): - return parse_version(ooniprobe_version) - - def set_last_run_version(self): - last_run_version_path = os.path.join( - self.running_path, "last_run_version" - ) - with open(last_run_version_path, "w") as out_file: - out_file.write(ooniprobe_version) - - @property def running_path(self): """ This is the directory used to store state application data. @@ -408,14 +383,6 @@ class OConfig(object): if exc.errno != errno.EEXIST: raise - # This means ooniprobe was installed for the first time or is coming - # from a 1.x series installation. We should configure the default deck. - if self.last_run_version.public == "0": - from ooni.deck.store import deck_store - DEFAULT_DECKS = ['web-full'] - for deck_id in DEFAULT_DECKS: - deck_store.enable(deck_id) - def create_config_file(self, include_ip=False, include_asn=True, include_country=True, should_upload=True, preferred_backend="onion"):

1 0

[ooni-probe/master] Better permissions handling for logfile and folder
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 73b17d70ad8ac535ddebe36ca1b1d4c6277781d1 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 2 15:17:24 2016 +0200 Better permissions handling for logfile and folder --- ooni/utils/log.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 1aba8dd..67079e0 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -1,5 +1,6 @@ import os import sys +import errno import codecs import logging @@ -132,6 +133,20 @@ class OONILogger(object): logfile = os.path.expanduser(config.basic.logfile) log_folder = os.path.dirname(logfile) + if not os.access(log_folder, os.W_OK): + # If we don't have permissions to write to the log_folder, + # write to running dir. + log_folder = config.running_path + logfile = os.path.join(log_folder, "ooniprobe.log") + + try: + os.makedirs(log_folder) + except OSError as ose: + if ose.errno == errno.EEXIST and os.path.isdir(log_folder): + pass + else: + raise + log_filename = os.path.basename(logfile) file_log_level = levels.get(config.basic.loglevel, levels['INFO'])

1 0

[ooni-probe/master] Bump version number of master
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 563e4ded184808ab68b36b3043e5a3ba7d6195f7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 29 13:42:18 2016 +0200 Bump version number of master --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 8185030..9d99323 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.6.1" +__version__ = "1.6.2.dev0" # This is the version number of resources to be downloaded # when a release is made it should be aligned to __version__ __resources_version__ = "1.6.0"

1 0

[ooni-probe/master] Delete inputs, resources and decks data directories
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 9b48ff3033e93e7386a70f8649657e70abe061cf Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jul 26 16:14:51 2016 +0200 Delete inputs, resources and decks data directories --- MANIFEST.in | 2 -- data/decks/complete.deck | 72 ---------------------------------------- data/decks/complete_no_root.deck | 60 --------------------------------- data/decks/fast.deck | 36 -------------------- data/decks/fast_no_root.deck | 24 -------------- data/decks/mlab.deck | 12 ------- data/inputs/.gitignore | 2 -- data/resources/README | 2 -- 8 files changed, 210 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index e2903fb..692c73f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,4 @@ include README.rst ChangeLog.rst requirements.txt LICENSE -recursive-include data/decks * -recursive-include data/inputs * include data/oonideckgen.1 include data/ooniprobe.1 include data/oonireport.1 diff --git a/data/decks/complete.deck b/data/decks/complete.deck deleted file mode 100644 index 7d9346c..0000000 --- a/data/decks/complete.deck +++ /dev/null @@ -1,72 +0,0 @@ -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: blocking/http_requests - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: blocking/dns_consistency - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_invalid_request_line - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_header_field_manipulation - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/traceroute - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: manipulation/http_host - testdeck: null diff --git a/data/decks/complete_no_root.deck b/data/decks/complete_no_root.deck deleted file mode 100644 index ba2c29b..0000000 --- a/data/decks/complete_no_root.deck +++ /dev/null @@ -1,60 +0,0 @@ -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: blocking/http_requests - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: blocking/dns_consistency - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_invalid_request_line - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_header_field_manipulation - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: manipulation/http_host - testdeck: null diff --git a/data/decks/fast.deck b/data/decks/fast.deck deleted file mode 100644 index ddf3c71..0000000 --- a/data/decks/fast.deck +++ /dev/null @@ -1,36 +0,0 @@ -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/traceroute - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_invalid_request_line - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_header_field_manipulation - testdeck: null diff --git a/data/decks/fast_no_root.deck b/data/decks/fast_no_root.deck deleted file mode 100644 index 69bdf2f..0000000 --- a/data/decks/fast_no_root.deck +++ /dev/null @@ -1,24 +0,0 @@ -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_invalid_request_line - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_header_field_manipulation - testdeck: null diff --git a/data/decks/mlab.deck b/data/decks/mlab.deck deleted file mode 100644 index 59dc458..0000000 --- a/data/decks/mlab.deck +++ /dev/null @@ -1,12 +0,0 @@ -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_header_field_manipulation - testdeck: null diff --git a/data/inputs/.gitignore b/data/inputs/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/data/inputs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/data/resources/README b/data/resources/README deleted file mode 100644 index fb768f4..0000000 --- a/data/resources/README +++ /dev/null @@ -1,2 +0,0 @@ -In here shall be stored all the resources downloaded via ooniresources to be -used by oonideckgen.

1 0

[ooni-probe/master] Merge branch 'master' of github.com:TheTorProject/ooni-probe
by art＠torproject.org 19 Sep '16

19 Sep '16

commit df0214e8ced0be1ab6bf965e1385cbe428ff1e8d Merge: 7eae5c1 5e86276 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 29 13:30:01 2016 +0200 Merge branch 'master' of github.com:TheTorProject/ooni-probe * 'master' of github.com:TheTorProject/ooni-probe: Add backend to required options to show error when it's not provided ooni/nettests/manipulation/traceroute.py | 5 +++-- ooni/templates/scapyt.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-)

1 0

[ooni-probe/master] Add backend to required options to show error when it's not provided
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 2b688756d9417d1c7279e8b4054fd739e5ca5c35 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Jul 24 23:46:36 2016 +0200 Add backend to required options to show error when it's not provided * Use same function to represent packets in traceroute test --- ooni/nettests/manipulation/traceroute.py | 5 +++-- ooni/templates/scapyt.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ooni/nettests/manipulation/traceroute.py b/ooni/nettests/manipulation/traceroute.py index d6f1ddf..3553bfd 100644 --- a/ooni/nettests/manipulation/traceroute.py +++ b/ooni/nettests/manipulation/traceroute.py @@ -31,6 +31,7 @@ class Traceroute(scapyt.BaseScapyTest): "set to 0, 22, 23, 53, 80, 123, 443, 8080 and 65535." requiredTestHelpers = {'backend': 'traceroute'} + requiredOptions = ['backend'] requiresRoot = True requiresTor = False @@ -63,9 +64,9 @@ class Traceroute(scapyt.BaseScapyTest): st.stopListening() st.matchResponses() for packet in st.sent_packets: - self.report['sent_packets'].append(packet) + self.report['sent_packets'].append(scapyt.representPacket(packet)) for packet in st.matched_packets.values(): - self.report['answered_packets'].extend(packet) + self.report['answered_packets'].append(scapyt.representPacket(packet)) for ttl in xrange(st.ttl_min, st.ttl_max): matchedPackets = filter( diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 1c726d8..e122525 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -7,13 +7,13 @@ from ooni.utils.net import hasRawSocketPermission from ooni.utils.txscapy import ScapySender, ScapyFactory -def _representPacket(packet): +def representPacket(packet): return { "raw_packet": { 'data': b64encode(str(packet)), 'format': 'base64' }, - "summary": repr(packet) + "summary": str(repr(packet)) } class BaseScapyTest(NetTestCase): @@ -98,8 +98,8 @@ class BaseScapyTest(NetTestCase): sent_packet.src = '127.0.0.1' received_packet.dst = '127.0.0.1' - self.report['sent_packets'].append(_representPacket(sent_packet)) - self.report['answered_packets'].append(_representPacket(received_packet)) + self.report['sent_packets'].append(representPacket(sent_packet)) + self.report['answered_packets'].append(representPacket(received_packet)) return packets def sr(self, packets, timeout=None, *arg, **kw): @@ -152,7 +152,7 @@ class BaseScapyTest(NetTestCase): scapySender.stopSending() for sent_packet in packets: - self.report['sent_packets'].append(_representPacket(sent_packet)) + self.report['sent_packets'].append(representPacket(sent_packet)) ScapyTest = BaseScapyTest

1 0

[ooni-probe/master] Merge pull request #574 from TheTorProject/hotfix/traceroute
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 5e86276327953e5f054254b6a3f3bd7e1342bbf1 Merge: da2b173 2b68875 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jul 26 02:09:06 2016 +0200 Merge pull request #574 from TheTorProject/hotfix/traceroute Add backend to required options to show error when it's not provided ooni/nettests/manipulation/traceroute.py | 5 +++-- ooni/templates/scapyt.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-)

1 0

[ooni-probe/master] Delete dead code
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 220c699d52620a4dee887ccc8438548df10960cd Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Jul 25 16:07:08 2016 +0200 Delete dead code --- ooni/nodes.conf | 53 ----------------------------------------------------- ooni/oonid.py | 15 --------------- 2 files changed, 68 deletions(-) diff --git a/ooni/nodes.conf b/ooni/nodes.conf deleted file mode 100644 index 4d14be1..0000000 --- a/ooni/nodes.conf +++ /dev/null @@ -1,53 +0,0 @@ -[control] -# Specify in the list of network nodes to -# be used as control nodes -[[node1]] -address = socks4a://user:password@192.0.2.1 - -[[node2]] -address = socks5://user:password@192.0.2.2 - -[[node3]] -address = ssh://user:password@192.0.2.3 - - -[experiment] -# Specify the list of experiment nodes to -# be used for running tests. -[[countryA]] -address = socks4a://user:password@192.0.2.1 -family = china - -[[countryA1]] -address = socks5://user:password@192.0.2.2 -family = china - -[[countryA2]] -address = ssh://user:password@192.0.2.3 -family = italy - -[operators] -# Specify the list of operator nodes. These are -# the ones capable of accepting bundles of units -# of work and scheduling work to experiment nodes. -[[Local box]] -address = localhost -capacity = 0.2 - -[[Planetlab node]] -address = ssh://user:password@192.0.2.3 -rsakey = ~/.ssh/id_rsa -other = -# Specify an float between 0 and 1 to how much -# work this operator node should support. -capacity = 0.4 - -[[M-Lab server]] -address = ssh://user:password@192.0.2.3 -capacity = 0.5 - - - - - - diff --git a/ooni/oonid.py b/ooni/oonid.py deleted file mode 100644 index c04cadc..0000000 --- a/ooni/oonid.py +++ /dev/null @@ -1,15 +0,0 @@ -from twisted.application import service, internet - -from ooni.settings import config -from ooni.api.spec import oonidApplication -from ooni.director import Director - -def getOonid(): - director = Director() - director.start() - oonidApplication.director = director - return internet.TCPServer(int(config.advanced.oonid_api_port), oonidApplication) - -application = service.Application("ooniprobe") -service = getOonid() -service.setServiceParent(application)

1 0