[tor-commits] [ooni-probe/master] Implement deck store

commit 97ed0fcbda310db18b2138f70024f08de5b523b9 Author: Arturo Filastò arturo@filasto.net Date: Fri Jul 29 22:50:45 2016 +0200

Implement deck store

* Write all runtime files to /var/lib/ooni

* Other various fixes to runtime paths

* Include the default deck and data files in distribution --- MANIFEST.in | 2 ++ data/decks/web.yaml | 22 +++++++++++++ ooni/agent/agent.py | 8 ----- ooni/agent/scheduler.py | 22 ++++++++++++- ooni/deck/deck.py | 15 +++++++-- ooni/deck/store.py | 35 +++++++++++++++++--- ooni/scripts/ooniprobe_agent.py | 17 +++++++--- ooni/settings.py | 72 ++++++++++++++++++++++++++++------------- ooni/ui/web/server.py | 8 +---- ooni/utils/log.py | 2 ++ 10 files changed, 153 insertions(+), 50 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in index 692c73f..485e834 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ +recursive-include ooni/ui/web/client * include README.rst ChangeLog.rst requirements.txt LICENSE +recursive-include data/decks * include data/oonideckgen.1 include data/ooniprobe.1 include data/oonireport.1 diff --git a/data/decks/web.yaml b/data/decks/web.yaml new file mode 100644 index 0000000..a81b8f8 --- /dev/null +++ b/data/decks/web.yaml @@ -0,0 +1,22 @@ +--- +name: Web related ooniprobe tests +description: This deck runs HTTP Header Field Manipulation, HTTP Invalid + Request and the Web Connectivity test +tasks: +- name: Runs the HTTP Header Field Manipulation test + ooni: + test_name: http_header_field_manipulation + +- name: Runs the HTTP Invalid Request Line test + ooni: + test_name: http_invalid_request_line + +- name: Runs the Web Connectivity Test + ooni: + test_name: web_connectivity + file: $citizenlab_global_urls + +- name: Runs the Web Connectivity Test + ooni: + test_name: web_connectivity + file: $citizenlab_${probe_cc}_urls diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py index ff9a2dd..c2e7e26 100644 --- a/ooni/agent/agent.py +++ b/ooni/agent/agent.py @@ -1,7 +1,6 @@ from twisted.application import service from ooni.director import Director from ooni.settings import config -from ooni.utils import log

from ooni.ui.web.web import WebUIService from ooni.agent.scheduler import SchedulerService @@ -11,9 +10,6 @@ class AgentService(service.MultiService): service.MultiService.__init__(self)

director = Director() - config.set_paths() - config.initialize_ooni_home() - config.read_config_file()

self.web_ui_service = WebUIService(director, web_ui_port) self.web_ui_service.setServiceParent(self) @@ -24,9 +20,5 @@ class AgentService(service.MultiService): def startService(self): service.MultiService.startService(self)

- log.start() - def stopService(self): service.MultiService.stopService(self) - - log.stop() diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 167a14a..43715e8 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -7,7 +7,7 @@ from twisted.python.filepath import FilePath from ooni.scripts import oonireport from ooni import resources from ooni.utils import log, SHORT_DATE -from ooni.deck.store import input_store +from ooni.deck.store import input_store, deck_store from ooni.settings import config from ooni.contrib import croniter from ooni.geoip import probe_ip @@ -116,6 +116,25 @@ class DeleteOldReports(ScheduledTask): log.debug("Deleting old report {0}".format(measurement["id"])) measurement_path.child(measurement['id']).remove()

+ +class RunDecks(ScheduledTask): + """ + This will run the decks that have been configured on the system as the + decks to run by default. + """ + schedule = '@daily' + identifier = 'run-decks' + + def __init__(self, director, schedule=None): + super(RunDecks, self).__init__(schedule) + self.director = director + + @defer.inlineCallbacks + def task(self): + for deck_id, deck in deck_store.list(): + yield deck.setup() + yield deck.run(self.director) + class SendHeartBeat(ScheduledTask): """ This task is used to send a heartbeat that the probe is still alive and @@ -193,6 +212,7 @@ class SchedulerService(service.MultiService): self.schedule(UpdateInputsAndResources()) self.schedule(UploadReports()) self.schedule(DeleteOldReports()) + self.schedule(RunDecks(self.director))

self._looping_call.start(self.interval)

diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index 868bfb8..1b2300a 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -1,5 +1,6 @@ import os from copy import deepcopy +from string import Template

import yaml from twisted.internet import defer @@ -10,7 +11,6 @@ from ooni.backend_client import BouncerClient, CollectorClient from ooni.backend_client import get_preferred_bouncer from ooni.deck.backend import lookup_collector_and_test_helpers from ooni.deck.legacy import convert_legacy_deck -from ooni.deck.store import input_store from ooni.geoip import probe_ip from ooni.nettest import NetTestLoader, nettest_to_path from ooni.measurements import generate_summary @@ -19,6 +19,7 @@ from ooni.utils import log, generate_filename

def resolve_file_path(v, prepath=None): + from ooni.deck.store import input_store if v.startswith("$"): # This raises InputNotFound and we let it carry onto the caller return input_store.get(v[1:])["filepath"] @@ -248,8 +249,13 @@ class NGDeck(object): """ This method needs to be called before you are able to run a deck. """ + from ooni.deck.store import InputNotFound for task in self._tasks: - yield task.setup() + try: + yield task.setup() + except InputNotFound: + log.msg("Skipping this task because the input cannot be found") + self._skip = True self._is_setup = True

@defer.inlineCallbacks @@ -365,7 +371,10 @@ class DeckTask(object): def _setup_ooni(self): yield probe_ip.lookup() for input_file in self.ooni['net_test_loader'].inputFiles: - file_path = resolve_file_path(input_file['filename'], self.cwd) + filename = Template(input_file['filename']).safe_substitute( + probe_cc=probe_ip.geodata['countrycode'].lower() + ) + file_path = resolve_file_path(filename, self.cwd) input_file['test_options'][input_file['key']] = file_path self.ooni['test_details'] = self.ooni['net_test_loader'].getTestDetails() self.id = generate_filename(self.ooni['test_details']) diff --git a/ooni/deck/store.py b/ooni/deck/store.py index 05c0b95..7c90204 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -5,13 +5,18 @@ from copy import deepcopy from twisted.internet import defer from twisted.python.filepath import FilePath

+from ooni.deck.deck import NGDeck from ooni.otime import timestampNowISO8601UTC from ooni.resources import check_for_update from ooni.settings import config +from ooni.utils import log

class InputNotFound(Exception): pass

+class DeckNotFound(Exception): + pass + class InputStore(object): def __init__(self): self.path = FilePath(config.inputs_directory) @@ -117,12 +122,34 @@ class InputStore(object): class DeckStore(object): def __init__(self): self.path = FilePath(config.decks_directory) + self._cache = {} + self._cache_stale = True

- def update(self): - pass + def list(self): + decks = [] + if self._cache_stale: + self._update_cache() + for deck_id, deck in self._cache.iteritems(): + decks.append((deck_id, deck)) + return decks

- def get(self): - pass + def _update_cache(self): + for deck_path in self.path.listdir(): + if not deck_path.endswith('.yaml'): + continue + deck_id = deck_path[:-1*len('.yaml')] + deck = NGDeck( + deck_path=self.path.child(deck_path).path + ) + self._cache[deck_id] = deck

+ def get(self, deck_id): + if self._cache_stale: + self._update_cache() + try: + return self._cache[deck_id] + except KeyError: + raise DeckNotFound(deck_id)

+deck_store = DeckStore() input_store = InputStore() diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index 3f8efc8..30df9b9 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -7,6 +7,7 @@ import signal from twisted.scripts import twistd from twisted.python import usage

+from ooni.utils import log from ooni.settings import config from ooni.agent.agent import AgentService

@@ -48,8 +49,15 @@ class AgentOptions(usage.Options): self.twistd_args = []

def start_agent(options=None): - os.chdir(config.ooni_home) - twistd_args = [] + config.set_paths() + config.initialize_ooni_home() + config.read_config_file() + + os.chdir(config.running_path) + + # Since we are starting the logger below ourselves we make twistd log to + # a null log observer + twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull'] twistd_config = OoniprobeTwistdConfig() if options is not None: twistd_args.extend(options.twistd_args) @@ -63,12 +71,13 @@ def start_agent(options=None): } print("Starting ooniprobe agent.") print("To view the GUI go to %s" % WEB_UI_URL) + log.start() twistd.runApp(twistd_config) return 0

def status_agent(): pidfile = os.path.join( - config.ooni_home, + config.running_path, 'twistd.pid' ) if not os.path.exists(pidfile): @@ -88,7 +97,7 @@ def status_agent(): def stop_agent(): # This function is borrowed from tahoe pidfile = os.path.join( - config.ooni_home, + config.running_path, 'twistd.pid' ) if not os.path.exists(pidfile): diff --git a/ooni/settings.py b/ooni/settings.py index 194729a..5762919 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -1,4 +1,5 @@ import os +import sys import yaml import getpass from ConfigParser import SafeConfigParser @@ -12,7 +13,6 @@ from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol from ooni.utils import Storage, log, get_ooni_root from ooni import errors

- class OConfig(object): _custom_home = None

@@ -41,18 +41,56 @@ class OConfig(object):

@property def var_lib_path(self): + if hasattr(sys, 'real_prefix'): + # We are in a virtualenv use the /usr/share in the virtualenv + return os.path.join( + os.path.abspath(sys.prefix), + 'var', 'lib', 'ooni' + ) var_lib_path = self.embedded_settings("directories", "var_lib") if var_lib_path: return os.path.abspath(var_lib_path) return "/var/lib/ooni"

@property + def running_path(self): + """ + This is the directory used to store state application data. + It defaults to /var/lib/ooni, but if that is not writeable we will + use the ooni_home. + """ + var_lib_path = self.var_lib_path + if os.access(var_lib_path, os.W_OK): + return var_lib_path + return self.ooni_home + + @property def usr_share_path(self): + if hasattr(sys, 'real_prefix'): + # We are in a virtualenv use the /usr/share in the virtualenv + return os.path.join( + os.path.abspath(sys.prefix), + 'usr', 'share', 'ooni' + ) usr_share_path = self.embedded_settings("directories", "usr_share") if usr_share_path: return os.path.abspath(usr_share_path) return "/usr/share/ooni"

+ + @property + def etc_path(self): + if hasattr(sys, 'real_prefix'): + # We are in a virtualenv use the /usr/share in the virtualenv + return os.path.join( + os.path.abspath(sys.prefix), + 'usr', 'share', 'ooni' + ) + etc_path = self.embedded_settings("directories", "etc") + if etc_path: + return os.path.abspath(etc_path) + return "/etc" + @property def data_directory_candidates(self): dirs = [ @@ -93,28 +131,15 @@ class OConfig(object):

def set_paths(self): self.nettest_directory = os.path.join(get_ooni_root(), 'nettests') + self.web_ui_directory = os.path.join(get_ooni_root(), 'web', 'client')

- if self.advanced.inputs_dir: - self.inputs_directory = self.advanced.inputs_dir - else: - self.inputs_directory = os.path.join(self.ooni_home, 'inputs') - - self.scheduler_directory = os.path.join(self.ooni_home, 'scheduler') + self.inputs_directory = os.path.join(self.running_path, 'inputs') + self.scheduler_directory = os.path.join(self.running_path, 'scheduler') + self.decks_directory = os.path.join(self.running_path, 'decks') + self.resources_directory = os.path.join(self.running_path, 'resources')

- if self.advanced.decks_dir: - self.decks_directory = self.advanced.decks_dir - else: - self.decks_directory = os.path.join(self.ooni_home, 'decks') - - self.measurements_directory = os.path.join(self.ooni_home, + self.measurements_directory = os.path.join(self.running_path, 'measurements') - self.resources_directory = os.path.join(self.ooni_home, - "resources") - if self.advanced.report_log_file: - self.report_log_file = self.advanced.report_log_file - else: - self.report_log_file = os.path.join(self.ooni_home, - 'reporting.yml')

if self.global_options.get('configfile'): config_file = self.global_options['configfile'] @@ -123,8 +148,9 @@ class OConfig(object): self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf')

if 'logfile' in self.basic: - self.basic.logfile = expanduser(self.basic.logfile.replace( - '~', '~'+self.current_user)) + self.basic.logfile = expanduser( + self.basic.logfile.replace('~', '~'+self.current_user) + )

def initialize_ooni_home(self, custom_home=None): if custom_home: @@ -147,7 +173,7 @@ class OConfig(object): ] for path in sub_directories: try: - os.mkdir(path) + os.makedirs(path) except OSError as exc: if exc.errno != 17: raise diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index e1a6398..74cb235 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -26,11 +26,6 @@ from ooni.geoip import probe_ip

config.advanced.debug = True

-def rpath(*path): - context = os.path.abspath(os.path.dirname(__file__)) - return os.path.join(context, *path) - - class WebUIError(Exception): def __init__(self, code, message): self.code = code @@ -392,5 +387,4 @@ class WebUIAPI(object): @app.route('/client/', branch=True) @xsrf_protect(check=False) def static(self, request): - path = rpath("client") - return static.File(path) + return static.File(config.web_ui_directory) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 982a353..1aba8dd 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -158,6 +158,8 @@ class OONILogger(object): self.fileObserver.stop()

oonilogger = OONILogger() +# This is a mock of a LoggerObserverFactory to be supplied to twistd. +ooniloggerNull = lambda: lambda eventDict: None

start = oonilogger.start stop = oonilogger.stop