[tor-commits] [ooni-probe/master] Add support for deck status
art at torproject.org
art at torproject.org
Fri Sep 22 18:41:06 UTC 2017
commit c937439d960ba3cfc77d173615fc029c44e417a7
Author: Arturo Filastò <arturo at filasto.net>
Date: Wed Jan 11 14:35:03 2017 +0000
Add support for deck status
---
ooni/agent/scheduler.py | 2 +-
ooni/deck/deck.py | 16 +++++++++++++---
ooni/deck/store.py | 3 +--
ooni/director.py | 27 +++++++++++++++++++++++++++
ooni/measurements.py | 16 +++++++++++-----
ooni/ui/cli.py | 2 +-
ooni/ui/web/server.py | 38 ++++++++++++++++++--------------------
7 files changed, 72 insertions(+), 32 deletions(-)
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 4bf058ab..8de3b303 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -294,7 +294,7 @@ class RunDeck(ScheduledTask):
def task(self):
deck = deck_store.get(self.deck_id)
yield deck.setup()
- yield deck.run(self.director)
+ yield deck.run(self.director, from_schedule=True)
class RefreshDeckList(ScheduledTask):
diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 07f0799e..fbbb8c3d 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -1,6 +1,7 @@
import os
import uuid
import errno
+import hashlib
from copy import deepcopy
from string import Template
@@ -82,6 +83,7 @@ class NGDeck(object):
self.name = ""
self.description = ""
self.icon = ""
+ self.id = None
self.schedule = None
self.metadata = {}
@@ -102,10 +104,15 @@ class NGDeck(object):
def open(self, deck_path, global_options=None):
with open(deck_path) as fh:
deck_data = yaml.safe_load(fh)
+ self.id = os.path.basename(deck_path[:-1*len('.yaml')])
self.deck_directory = os.path.abspath(os.path.dirname(deck_path))
self.load(deck_data, global_options)
def load(self, deck_data, global_options=None):
+ if self.id is None:
+ # This happens when you load a deck not from a filepath so we
+ # use the first 16 characters of the SHA256 hexdigest as an ID
+ self.id = hashlib.sha256(deck_data).hexdigest()[:16]
if global_options is not None:
self.global_options = normalize_options(global_options)
@@ -209,7 +216,8 @@ class NGDeck(object):
)
generate_summary(
measurement_dir.child("measurements.njson").path,
- measurement_dir.child("summary.json").path
+ measurement_dir.child("summary.json").path,
+ deck_id=self.id
)
measurement_dir.child("running.pid").remove()
@@ -274,18 +282,20 @@ class NGDeck(object):
self._is_setup = True
@defer.inlineCallbacks
- def run(self, director):
+ def run(self, director, from_schedule=False):
assert self._is_setup, "You must call setup() before you can run a " \
"deck"
if self.requires_tor:
yield director.start_tor()
yield self.query_bouncer()
+ director.deckStarted(self.id, from_schedule)
for task in self._tasks:
if task.skip is True:
- log.msg("Skipping running {0}".format(task.id))
+ log.debug("Skipping running {0}".format(task.id))
continue
if task.type == "ooni":
yield self._run_ooni_task(task, director)
+ director.deckFinished(self.id, from_schedule)
self._is_setup = False
diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 6e3e6de4..204d9357 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -227,11 +227,10 @@ class DeckStore(object):
for deck_path in self.available_directory.listdir():
if not deck_path.endswith('.yaml'):
continue
- deck_id = deck_path[:-1*len('.yaml')]
deck = NGDeck(
deck_path=self.available_directory.child(deck_path).path
)
- new_cache[deck_id] = deck
+ new_cache[deck.id] = deck
self._cache = new_cache
self._cache_stale = False
diff --git a/ooni/director.py b/ooni/director.py
index 1e55bb67..f7566f5f 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -70,6 +70,7 @@ class Director(object):
def __init__(self):
self.activeNetTests = []
+ self.activeDecks = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
@@ -290,6 +291,32 @@ class Director(object):
measurement.result = failure
return measurement
+ def deckStarted(self, deck_id, from_schedule):
+ log.debug("Starting {0} ({1})".format(deck_id,
+ 'scheduled' if from_schedule
+ else 'user-run'))
+ self.activeDecks.append((deck_id, from_schedule))
+
+ def deckFinished(self, deck_id, from_schedule):
+ log.debug("Finished {0} ({1})".format(deck_id,
+ 'scheduled' if from_schedule
+ else 'user-run'))
+ try:
+ self.activeDecks.remove((deck_id, from_schedule))
+ except ValueError:
+ log.error("Completed deck {0} is not actually running".format(
+ deck_id))
+
+
+ def isDeckRunning(self, deck_id, from_schedule):
+ """
+ :param deck_id: the ID of the deck to check if it's running
+ :param from_schedule: True if we want to know the status of a
+ scheduled deck run False for user initiated runs.
+ :return: True if the deck is running False otherwise
+ """
+ return (deck_id, from_schedule) in self.activeDecks
+
def netTestDone(self, net_test):
self.notify(DirectorEvent("success",
"Successfully ran test {0}".format(
diff --git a/ooni/measurements.py b/ooni/measurements.py
index b3b3e0fc..9197ee75 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -11,7 +11,7 @@ from ooni.settings import config
class MeasurementInProgress(Exception):
pass
-class Process():
+class MeasurementTypes():
supported_tests = [
"web_connectivity",
"http_requests",
@@ -51,14 +51,15 @@ class Process():
result['url'] = entry['input']
return result
-def generate_summary(input_file, output_file):
+
+def generate_summary(input_file, output_file, deck_id='none'):
results = {}
with open(input_file) as in_file:
for idx, line in enumerate(in_file):
entry = json.loads(line.strip())
result = {}
- if entry['test_name'] in Process.supported_tests:
- result = getattr(Process, entry['test_name'])(entry)
+ if entry['test_name'] in MeasurementTypes.supported_tests:
+ result = getattr(MeasurementTypes, entry['test_name'])(entry)
result['idx'] = idx
if not result.get('url', None):
result['url'] = entry['input']
@@ -66,6 +67,7 @@ def generate_summary(input_file, output_file):
results['test_start_time'] = entry['test_start_time']
results['country_code'] = entry['probe_cc']
results['asn'] = entry['probe_asn']
+ results['deck_id'] = deck_id
results['results'] = results.get('results', [])
results['results'].append(result)
@@ -73,9 +75,11 @@ def generate_summary(input_file, output_file):
json.dump(results, fw)
return results
+
class MeasurementNotFound(Exception):
pass
+
def get_measurement(measurement_id, compute_size=False):
size = -1
measurement_path = FilePath(config.measurements_directory)
@@ -114,7 +118,9 @@ def get_measurement(measurement_id, compute_size=False):
"keep": keep,
"running": running,
"stale": stale,
- "size": size
+ "size": size,
+ # XXX we need the deck ID in here
+ "deck_id": "none"
}
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 162354c1..cf889a3c 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -377,7 +377,7 @@ def runTestWithDirector(director, global_options, url=None,
def post_director_start(_):
try:
yield deck.setup()
- yield deck.run(director)
+ yield deck.run(director, from_schedule=False)
except errors.UnableToLoadDeckInput as error:
raise defer.failure.Failure(error)
except errors.NoReachableTestHelpers as error:
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index efcad146..c8f191c7 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -147,7 +147,7 @@ class WebUIAPI(object):
app = Klein()
# Maximum number in seconds after which to return a result even if no
# change happened.
- _long_polling_timeout = 5
+ _long_polling_timeout = 30
_reactor = reactor
_enable_xsrf_protection = True
@@ -171,8 +171,8 @@ class WebUIAPI(object):
# We use exponential backoff to trigger retries of the startup of
# the director.
self._director_startup_retries = 0
- # Maximum delay should be 6 hours
- self._director_max_retry_delay = 6*60*60
+ # Maximum delay should be 30 minutes
+ self._director_max_retry_delay = 30*60
self.status_poller = LongPoller(
self._long_polling_timeout, _reactor)
@@ -368,26 +368,21 @@ class WebUIAPI(object):
@xsrf_protect(check=False)
@requires_true(attrs=['_director_started', '_is_initialized'])
def api_deck_list(self, request):
- deck_list = {
- 'available': {},
- 'enabled': {}
- }
+ deck_list = {'decks': []}
for deck_id, deck in self.director.deck_store.list():
- deck_list['available'][deck_id] = {
+ deck_list['decks'].append({
+ 'id': deck_id,
'name': deck.name,
+ 'icon': deck.icon,
+ 'running': self.director.isDeckRunning(
+ deck_id, from_schedule=False),
+ 'running_scheduled': self.director.isDeckRunning(
+ deck_id, from_schedule=True),
+ 'nettests': [], #XXX
'description': deck.description,
'schedule': deck.schedule,
'enabled': self.director.deck_store.is_enabled(deck_id)
- }
-
- for deck_id, deck in self.director.deck_store.list_enabled():
- deck_list['enabled'][deck_id] = {
- 'name': deck.name,
- 'description': deck.description,
- 'schedule': deck.schedule,
- 'enabled': True
- }
-
+ })
return self.render_json(deck_list, request)
@app.route('/api/deck/<string:deck_id>/run', methods=["POST"])
@@ -433,9 +428,12 @@ class WebUIAPI(object):
# These are dangling deferreds
try:
yield deck.setup()
- yield deck.run(self.director)
+ yield deck.run(self.director, from_schedule=False)
+ self.director_event_poller.notify(DirectorEvent("success",
+ "Started Deck "
+ + deck.id))
except:
- self.director_event_poller.notify(DirectorEvent("error",
+ self.director_event_poller.notify(DirectorEvent("error",
"Failed to start deck"))
@app.route('/api/nettest/<string:test_name>/start', methods=["POST"])
More information about the tor-commits
mailing list