commit 7c35e65485418d1dcb682bad1cf4eb62ef318e81 Author: Arturo Filastò arturo@filasto.net Date: Sat Jul 30 16:27:20 2016 +0200

Add support for listing enabled and disabled decks

* Fix various bugs --- ooni/agent/scheduler.py | 2 +- ooni/deck/store.py | 40 +++++++++++++++++---- ooni/director.py | 5 +-- ooni/measurements.py | 11 ++++-- ooni/settings.py | 9 +++-- ooni/ui/web/client/index.html | 2 +- ooni/ui/web/server.py | 81 +++++++++++++++++++++++++++++++++---------- 7 files changed, 117 insertions(+), 33 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 43715e8..7a77afb 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -131,7 +131,7 @@ class RunDecks(ScheduledTask):

@defer.inlineCallbacks def task(self): - for deck_id, deck in deck_store.list(): + for deck_id, deck in deck_store.list_enabled(): yield deck.setup() yield deck.run(self.director)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py index 7c90204..695f97d 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -9,7 +9,6 @@ from ooni.deck.deck import NGDeck from ooni.otime import timestampNowISO8601UTC from ooni.resources import check_for_update from ooni.settings import config -from ooni.utils import log

class InputNotFound(Exception): pass @@ -121,25 +120,54 @@ class InputStore(object):

class DeckStore(object): def __init__(self): - self.path = FilePath(config.decks_directory) + self.enabled_directory = FilePath(config.decks_enabled_directory) + self.available_directory = FilePath(config.decks_available_directory) self._cache = {} self._cache_stale = True

- def list(self): - decks = [] + def _list(self): if self._cache_stale: self._update_cache() for deck_id, deck in self._cache.iteritems(): + yield (deck_id, deck) + + def list(self): + decks = [] + for deck_id, deck in self._list(): decks.append((deck_id, deck)) return decks

+ def list_enabled(self): + decks = [] + for deck_id, deck in self._list(): + if self.is_enabled(deck_id): + continue + decks.append((deck_id, deck)) + return decks + + def is_enabled(self, deck_id): + return self.enabled_directory.child(deck_id + '.yaml').exists() + + def enable(self, deck_id): + deck_path = self.available_directory.child(deck_id + '.yaml') + if not deck_path.exists(): + raise DeckNotFound(deck_id) + deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml') + deck_enabled_path.linkTo(deck_path) + + def disable(self, deck_id): + deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml') + if not deck_enabled_path.exists(): + raise DeckNotFound(deck_id) + deck_enabled_path.remove() + def _update_cache(self): - for deck_path in self.path.listdir(): + for deck_path in self.available_directory.listdir(): if not deck_path.endswith('.yaml'): continue deck_id = deck_path[:-1*len('.yaml')] deck = NGDeck( - deck_path=self.path.child(deck_path).path + deck_path=self.available_directory.child(deck_path).path ) self._cache[deck_id] = deck

diff --git a/ooni/director.py b/ooni/director.py index 464a203..304a14a 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -9,7 +9,7 @@ from ooni.utils import log, generate_filename from ooni.nettest import NetTest, getNetTestInformation from ooni.settings import config from ooni.nettest import normalizeTestName -from ooni.deck.store import InputStore +from ooni.deck.store import input_store, deck_store from ooni.geoip import probe_ip

from ooni.agent.scheduler import run_system_tasks @@ -96,7 +96,8 @@ class Director(object): self.allTestsDone = defer.Deferred() self.sniffers = {}

- self.input_store = InputStore() + self.input_store = input_store + self.deck_store = deck_store

self._reset_director_state() self._reset_tor_state() diff --git a/ooni/measurements.py b/ooni/measurements.py index 50efd87..cdf6b14 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -5,6 +5,9 @@ import signal from twisted.python.filepath import FilePath from ooni.settings import config

+class MeasurementInProgress(Exception): + pass + class Process(): supported_tests = [ "web_connectivity", @@ -66,7 +69,7 @@ def get_measurement(measurement_id): running = False completed = True keep = False - if measurement.child("measurement.njson.progress").exists(): + if measurement.child("measurements.njson.progress").exists(): completed = False # XXX this is done quite often around the code, probably should # be moved into some utility function. @@ -97,10 +100,14 @@ def get_measurement(measurement_id): def get_summary(measurement_id): measurement_path = FilePath(config.measurements_directory) measurement = measurement_path.child(measurement_id) + + if measurement.child("measurements.njson.progress").exists(): + raise MeasurementInProgress + summary = measurement.child("summary.json") if not summary.exists(): generate_summary( - measurement.child("measurement.njson").path, + measurement.child("measurements.njson").path, summary.path )

diff --git a/ooni/settings.py b/ooni/settings.py index 3d33601..2161560 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -135,9 +135,13 @@ class OConfig(object):

self.inputs_directory = os.path.join(self.running_path, 'inputs') self.scheduler_directory = os.path.join(self.running_path, 'scheduler') - self.decks_directory = os.path.join(self.running_path, 'decks') self.resources_directory = os.path.join(self.running_path, 'resources')

+ self.decks_available_directory = os.path.join(self.running_path, + 'decks-available') + self.decks_enabled_directory = os.path.join(self.running_path, + 'decks-enabled') + self.measurements_directory = os.path.join(self.running_path, 'measurements')

@@ -166,7 +170,8 @@ class OConfig(object): # also ensure the subdirectories exist sub_directories = [ self.inputs_directory, - self.decks_directory, + self.decks_enabled_directory, + self.decks_available_directory, self.scheduler_directory, self.measurements_directory, self.resources_directory diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index ebed106..e363ba0 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?2777836bc218e75c3be5"></script></body> + <script type="text/javascript" src="app.bundle.js?9d3ccb3bc67af5ed4453"></script></body> </html> diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 74cb235..50db3ad 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -1,6 +1,5 @@ from __future__ import print_function

-import os import json import string from functools import wraps @@ -17,11 +16,12 @@ from werkzeug.exceptions import NotFound from ooni import __version__ as ooniprobe_version from ooni import errors from ooni.deck import NGDeck +from ooni.deck.store import DeckNotFound, InputNotFound from ooni.settings import config from ooni.utils import log from ooni.director import DirectorEvent -from ooni.measurements import get_summary, get_measurement -from ooni.measurements import list_measurements, MeasurementNotFound +from ooni.measurements import get_summary, get_measurement, list_measurements +from ooni.measurements import MeasurementNotFound, MeasurementInProgress from ooni.geoip import probe_ip

config.advanced.debug = True @@ -120,7 +120,6 @@ class WebUIAPI(object): self.director = director self.config = config self.measurement_path = FilePath(config.measurements_directory) - self.decks_path = FilePath(config.decks_directory)

# We use a double submit token to protect against XSRF rng = SystemRandom() @@ -207,23 +206,61 @@ class WebUIAPI(object): d.addCallback(got_status_update) return d

- @app.route('/api/deck/generate', methods=["GET"]) - @xsrf_protect(check=False) - def api_deck_generate(self, request): - return self.render_json({"generate": "deck"}, request) - - @app.route('/api/deck/string:deck_name/start', methods=["POST"]) + @app.route('/api/deck/string:deck_id/start', methods=["POST"]) @xsrf_protect(check=True) - def api_deck_start(self, request, deck_name): - return self.render_json({"start": deck_name}, request) + def api_deck_start(self, request, deck_id): + try: + deck = self.director.deck_store.get(deck_id) + except DeckNotFound: + raise WebUIError(404, "Deck not found") + + try: + self.run_deck(deck) + except: + raise WebUIError(500, "Failed to start deck") + + return self.render_json({"status": "started " + deck.name}, request)

@app.route('/api/deck', methods=["GET"]) @xsrf_protect(check=False) def api_deck_list(self, request): - for deck_id in self.decks_path.listdir(): - pass + deck_list = { + 'available': {}, + 'enabled': {} + } + for deck_id, deck in self.director.deck_store.list(): + deck_list['available'][deck_id] = { + 'name': deck.name, + 'description': deck.description + } + + for deck_id, deck in self.director.deck_store.list_enabled(): + deck_list['enabled'][deck_id] = { + 'name': deck.name, + 'description': deck.description + } + + return self.render_json(deck_list, request)

- return self.render_json({"command": "deck-list"}, request) + @app.route('/api/deck/string:deck_id/enable', methods=["POST"]) + @xsrf_protect(check=True) + def api_deck_enable(self, request, deck_id): + try: + self.director.deck_store.enable(deck_id) + except DeckNotFound: + raise WebUIError(404, "Deck not found") + + return self.render_json({"status": "enabled"}, request) + + @app.route('/api/deck/string:deck_id/disable', methods=["POST"]) + @xsrf_protect(check=True) + def api_deck_disable(self, request, deck_id): + try: + self.director.deck_store.disable(deck_id) + except DeckNotFound: + raise WebUIError(404, "Deck not found") + + return self.render_json({"status": "disabled"}, request)

@defer.inlineCallbacks def run_deck(self, deck): @@ -261,17 +298,21 @@ class WebUIAPI(object):

except errors.MissingRequiredOption, option_name: raise WebUIError( - 501, 'Missing required option: "{}"'.format(option_name) + 400, 'Missing required option: "{}"'.format(option_name) )

except usage.UsageError: raise WebUIError( - 502, 'Error in parsing options' + 400, 'Error in parsing options' )

except errors.InsufficientPrivileges: raise WebUIError( - 502, 'Insufficient priviledges' + 400, 'Insufficient priviledges' + ) + except: + raise WebUIError( + 500, 'Failed to start nettest' )

return self.render_json({"status": "started"}, request) @@ -319,6 +360,8 @@ class WebUIAPI(object): raise WebUIError(500, "invalid measurement id") except MeasurementNotFound: raise WebUIError(404, "measurement not found") + except MeasurementInProgress: + raise WebUIError(400, "measurement in progress")

if measurement['completed'] is False: raise WebUIError(400, "measurement in progress") @@ -359,7 +402,7 @@ class WebUIAPI(object): with summary.open("w+") as f: pass

- return self.render_json({"result": "ok"}, request) + return self.render_json({"status": "ok"}, request)

@app.route('/api/measurement/string:measurement_id/int:idx', methods=["GET"])