commit c937439d960ba3cfc77d173615fc029c44e417a7 Author: Arturo Filastò arturo@filasto.net Date: Wed Jan 11 14:35:03 2017 +0000

Add support for deck status --- ooni/agent/scheduler.py | 2 +- ooni/deck/deck.py | 16 +++++++++++++--- ooni/deck/store.py | 3 +-- ooni/director.py | 27 +++++++++++++++++++++++++++ ooni/measurements.py | 16 +++++++++++----- ooni/ui/cli.py | 2 +- ooni/ui/web/server.py | 38 ++++++++++++++++++-------------------- 7 files changed, 72 insertions(+), 32 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 4bf058ab..8de3b303 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -294,7 +294,7 @@ class RunDeck(ScheduledTask): def task(self): deck = deck_store.get(self.deck_id) yield deck.setup() - yield deck.run(self.director) + yield deck.run(self.director, from_schedule=True)

class RefreshDeckList(ScheduledTask): diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index 07f0799e..fbbb8c3d 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -1,6 +1,7 @@ import os import uuid import errno +import hashlib from copy import deepcopy from string import Template

@@ -82,6 +83,7 @@ class NGDeck(object): self.name = "" self.description = "" self.icon = "" + self.id = None self.schedule = None

self.metadata = {} @@ -102,10 +104,15 @@ class NGDeck(object): def open(self, deck_path, global_options=None): with open(deck_path) as fh: deck_data = yaml.safe_load(fh) + self.id = os.path.basename(deck_path[:-1*len('.yaml')]) self.deck_directory = os.path.abspath(os.path.dirname(deck_path)) self.load(deck_data, global_options)

def load(self, deck_data, global_options=None): + if self.id is None: + # This happens when you load a deck not from a filepath so we + # use the first 16 characters of the SHA256 hexdigest as an ID + self.id = hashlib.sha256(deck_data).hexdigest()[:16] if global_options is not None: self.global_options = normalize_options(global_options)

@@ -209,7 +216,8 @@ class NGDeck(object): ) generate_summary( measurement_dir.child("measurements.njson").path, - measurement_dir.child("summary.json").path + measurement_dir.child("summary.json").path, + deck_id=self.id ) measurement_dir.child("running.pid").remove()

@@ -274,18 +282,20 @@ class NGDeck(object): self._is_setup = True

@defer.inlineCallbacks - def run(self, director): + def run(self, director, from_schedule=False): assert self._is_setup, "You must call setup() before you can run a " \ "deck" if self.requires_tor: yield director.start_tor() yield self.query_bouncer() + director.deckStarted(self.id, from_schedule) for task in self._tasks: if task.skip is True: - log.msg("Skipping running {0}".format(task.id)) + log.debug("Skipping running {0}".format(task.id)) continue if task.type == "ooni": yield self._run_ooni_task(task, director) + director.deckFinished(self.id, from_schedule) self._is_setup = False

diff --git a/ooni/deck/store.py b/ooni/deck/store.py index 6e3e6de4..204d9357 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -227,11 +227,10 @@ class DeckStore(object): for deck_path in self.available_directory.listdir(): if not deck_path.endswith('.yaml'): continue - deck_id = deck_path[:-1*len('.yaml')] deck = NGDeck( deck_path=self.available_directory.child(deck_path).path ) - new_cache[deck_id] = deck + new_cache[deck.id] = deck self._cache = new_cache self._cache_stale = False

diff --git a/ooni/director.py b/ooni/director.py index 1e55bb67..f7566f5f 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -70,6 +70,7 @@ class Director(object):

def __init__(self): self.activeNetTests = [] + self.activeDecks = []

self.measurementManager = MeasurementManager() self.measurementManager.director = self @@ -290,6 +291,32 @@ class Director(object): measurement.result = failure return measurement

+ def deckStarted(self, deck_id, from_schedule): + log.debug("Starting {0} ({1})".format(deck_id, + 'scheduled' if from_schedule + else 'user-run')) + self.activeDecks.append((deck_id, from_schedule)) + + def deckFinished(self, deck_id, from_schedule): + log.debug("Finished {0} ({1})".format(deck_id, + 'scheduled' if from_schedule + else 'user-run')) + try: + self.activeDecks.remove((deck_id, from_schedule)) + except ValueError: + log.error("Completed deck {0} is not actually running".format( + deck_id)) + + + def isDeckRunning(self, deck_id, from_schedule): + """ + :param deck_id: the ID of the deck to check if it's running + :param from_schedule: True if we want to know the status of a + scheduled deck run False for user initiated runs. + :return: True if the deck is running False otherwise + """ + return (deck_id, from_schedule) in self.activeDecks + def netTestDone(self, net_test): self.notify(DirectorEvent("success", "Successfully ran test {0}".format( diff --git a/ooni/measurements.py b/ooni/measurements.py index b3b3e0fc..9197ee75 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -11,7 +11,7 @@ from ooni.settings import config class MeasurementInProgress(Exception): pass

-class Process(): +class MeasurementTypes(): supported_tests = [ "web_connectivity", "http_requests", @@ -51,14 +51,15 @@ class Process(): result['url'] = entry['input'] return result

-def generate_summary(input_file, output_file): + +def generate_summary(input_file, output_file, deck_id='none'): results = {} with open(input_file) as in_file: for idx, line in enumerate(in_file): entry = json.loads(line.strip()) result = {} - if entry['test_name'] in Process.supported_tests: - result = getattr(Process, entry['test_name'])(entry) + if entry['test_name'] in MeasurementTypes.supported_tests: + result = getattr(MeasurementTypes, entry['test_name'])(entry) result['idx'] = idx if not result.get('url', None): result['url'] = entry['input'] @@ -66,6 +67,7 @@ def generate_summary(input_file, output_file): results['test_start_time'] = entry['test_start_time'] results['country_code'] = entry['probe_cc'] results['asn'] = entry['probe_asn'] + results['deck_id'] = deck_id results['results'] = results.get('results', []) results['results'].append(result)

@@ -73,9 +75,11 @@ def generate_summary(input_file, output_file): json.dump(results, fw) return results

+ class MeasurementNotFound(Exception): pass

+ def get_measurement(measurement_id, compute_size=False): size = -1 measurement_path = FilePath(config.measurements_directory) @@ -114,7 +118,9 @@ def get_measurement(measurement_id, compute_size=False): "keep": keep, "running": running, "stale": stale, - "size": size + "size": size, + # XXX we need the deck ID in here + "deck_id": "none" }

diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 162354c1..cf889a3c 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -377,7 +377,7 @@ def runTestWithDirector(director, global_options, url=None, def post_director_start(_): try: yield deck.setup() - yield deck.run(director) + yield deck.run(director, from_schedule=False) except errors.UnableToLoadDeckInput as error: raise defer.failure.Failure(error) except errors.NoReachableTestHelpers as error: diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index efcad146..c8f191c7 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -147,7 +147,7 @@ class WebUIAPI(object): app = Klein() # Maximum number in seconds after which to return a result even if no # change happened. - _long_polling_timeout = 5 + _long_polling_timeout = 30 _reactor = reactor _enable_xsrf_protection = True

@@ -171,8 +171,8 @@ class WebUIAPI(object): # We use exponential backoff to trigger retries of the startup of # the director. self._director_startup_retries = 0 - # Maximum delay should be 6 hours - self._director_max_retry_delay = 6*60*60 + # Maximum delay should be 30 minutes + self._director_max_retry_delay = 30*60

self.status_poller = LongPoller( self._long_polling_timeout, _reactor) @@ -368,26 +368,21 @@ class WebUIAPI(object): @xsrf_protect(check=False) @requires_true(attrs=['_director_started', '_is_initialized']) def api_deck_list(self, request): - deck_list = { - 'available': {}, - 'enabled': {} - } + deck_list = {'decks': []} for deck_id, deck in self.director.deck_store.list(): - deck_list['available'][deck_id] = { + deck_list['decks'].append({ + 'id': deck_id, 'name': deck.name, + 'icon': deck.icon, + 'running': self.director.isDeckRunning( + deck_id, from_schedule=False), + 'running_scheduled': self.director.isDeckRunning( + deck_id, from_schedule=True), + 'nettests': [], #XXX 'description': deck.description, 'schedule': deck.schedule, 'enabled': self.director.deck_store.is_enabled(deck_id) - } - - for deck_id, deck in self.director.deck_store.list_enabled(): - deck_list['enabled'][deck_id] = { - 'name': deck.name, - 'description': deck.description, - 'schedule': deck.schedule, - 'enabled': True - } - + }) return self.render_json(deck_list, request)

@app.route('/api/deck/string:deck_id/run', methods=["POST"]) @@ -433,9 +428,12 @@ class WebUIAPI(object): # These are dangling deferreds try: yield deck.setup() - yield deck.run(self.director) + yield deck.run(self.director, from_schedule=False) + self.director_event_poller.notify(DirectorEvent("success", + "Started Deck " + + deck.id)) except: - self.director_event_poller.notify(DirectorEvent("error", + self.director_event_poller.notify(DirectorEvent("error", "Failed to start deck"))

@app.route('/api/nettest/string:test_name/start', methods=["POST"])