[tor-commits] [ooni-probe/master] Implement deck store
art at torproject.org
art at torproject.org
Mon Sep 19 12:14:24 UTC 2016
commit 97ed0fcbda310db18b2138f70024f08de5b523b9
Author: Arturo Filastò <arturo at filasto.net>
Date: Fri Jul 29 22:50:45 2016 +0200
Implement deck store
* Write all runtime files to /var/lib/ooni
* Other various fixes to runtime paths
* Include the default deck and data files in distribution
---
MANIFEST.in | 2 ++
data/decks/web.yaml | 22 +++++++++++++
ooni/agent/agent.py | 8 -----
ooni/agent/scheduler.py | 22 ++++++++++++-
ooni/deck/deck.py | 15 +++++++--
ooni/deck/store.py | 35 +++++++++++++++++---
ooni/scripts/ooniprobe_agent.py | 17 +++++++---
ooni/settings.py | 72 ++++++++++++++++++++++++++++-------------
ooni/ui/web/server.py | 8 +----
ooni/utils/log.py | 2 ++
10 files changed, 153 insertions(+), 50 deletions(-)
diff --git a/MANIFEST.in b/MANIFEST.in
index 692c73f..485e834 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,6 @@
+recursive-include ooni/ui/web/client *
include README.rst ChangeLog.rst requirements.txt LICENSE
+recursive-include data/decks *
include data/oonideckgen.1
include data/ooniprobe.1
include data/oonireport.1
diff --git a/data/decks/web.yaml b/data/decks/web.yaml
new file mode 100644
index 0000000..a81b8f8
--- /dev/null
+++ b/data/decks/web.yaml
@@ -0,0 +1,22 @@
+---
+name: Web related ooniprobe tests
+description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
+ Request and the Web Connectivity test
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+ ooni:
+ test_name: http_header_field_manipulation
+
+- name: Runs the HTTP Invalid Request Line test
+ ooni:
+ test_name: http_invalid_request_line
+
+- name: Runs the Web Connectivity Test
+ ooni:
+ test_name: web_connectivity
+ file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+ ooni:
+ test_name: web_connectivity
+ file: $citizenlab_${probe_cc}_urls
diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index ff9a2dd..c2e7e26 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -1,7 +1,6 @@
from twisted.application import service
from ooni.director import Director
from ooni.settings import config
-from ooni.utils import log
from ooni.ui.web.web import WebUIService
from ooni.agent.scheduler import SchedulerService
@@ -11,9 +10,6 @@ class AgentService(service.MultiService):
service.MultiService.__init__(self)
director = Director()
- config.set_paths()
- config.initialize_ooni_home()
- config.read_config_file()
self.web_ui_service = WebUIService(director, web_ui_port)
self.web_ui_service.setServiceParent(self)
@@ -24,9 +20,5 @@ class AgentService(service.MultiService):
def startService(self):
service.MultiService.startService(self)
- log.start()
-
def stopService(self):
service.MultiService.stopService(self)
-
- log.stop()
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 167a14a..43715e8 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -7,7 +7,7 @@ from twisted.python.filepath import FilePath
from ooni.scripts import oonireport
from ooni import resources
from ooni.utils import log, SHORT_DATE
-from ooni.deck.store import input_store
+from ooni.deck.store import input_store, deck_store
from ooni.settings import config
from ooni.contrib import croniter
from ooni.geoip import probe_ip
@@ -116,6 +116,25 @@ class DeleteOldReports(ScheduledTask):
log.debug("Deleting old report {0}".format(measurement["id"]))
measurement_path.child(measurement['id']).remove()
+
+class RunDecks(ScheduledTask):
+ """
+ This will run the decks that have been configured on the system as the
+ decks to run by default.
+ """
+ schedule = '@daily'
+ identifier = 'run-decks'
+
+ def __init__(self, director, schedule=None):
+ super(RunDecks, self).__init__(schedule)
+ self.director = director
+
+ @defer.inlineCallbacks
+ def task(self):
+ for deck_id, deck in deck_store.list():
+ yield deck.setup()
+ yield deck.run(self.director)
+
class SendHeartBeat(ScheduledTask):
"""
This task is used to send a heartbeat that the probe is still alive and
@@ -193,6 +212,7 @@ class SchedulerService(service.MultiService):
self.schedule(UpdateInputsAndResources())
self.schedule(UploadReports())
self.schedule(DeleteOldReports())
+ self.schedule(RunDecks(self.director))
self._looping_call.start(self.interval)
diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 868bfb8..1b2300a 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -1,5 +1,6 @@
import os
from copy import deepcopy
+from string import Template
import yaml
from twisted.internet import defer
@@ -10,7 +11,6 @@ from ooni.backend_client import BouncerClient, CollectorClient
from ooni.backend_client import get_preferred_bouncer
from ooni.deck.backend import lookup_collector_and_test_helpers
from ooni.deck.legacy import convert_legacy_deck
-from ooni.deck.store import input_store
from ooni.geoip import probe_ip
from ooni.nettest import NetTestLoader, nettest_to_path
from ooni.measurements import generate_summary
@@ -19,6 +19,7 @@ from ooni.utils import log, generate_filename
def resolve_file_path(v, prepath=None):
+ from ooni.deck.store import input_store
if v.startswith("$"):
# This raises InputNotFound and we let it carry onto the caller
return input_store.get(v[1:])["filepath"]
@@ -248,8 +249,13 @@ class NGDeck(object):
"""
This method needs to be called before you are able to run a deck.
"""
+ from ooni.deck.store import InputNotFound
for task in self._tasks:
- yield task.setup()
+ try:
+ yield task.setup()
+ except InputNotFound:
+ log.msg("Skipping this task because the input cannot be found")
+ self._skip = True
self._is_setup = True
@defer.inlineCallbacks
@@ -365,7 +371,10 @@ class DeckTask(object):
def _setup_ooni(self):
yield probe_ip.lookup()
for input_file in self.ooni['net_test_loader'].inputFiles:
- file_path = resolve_file_path(input_file['filename'], self.cwd)
+ filename = Template(input_file['filename']).safe_substitute(
+ probe_cc=probe_ip.geodata['countrycode'].lower()
+ )
+ file_path = resolve_file_path(filename, self.cwd)
input_file['test_options'][input_file['key']] = file_path
self.ooni['test_details'] = self.ooni['net_test_loader'].getTestDetails()
self.id = generate_filename(self.ooni['test_details'])
diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 05c0b95..7c90204 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -5,13 +5,18 @@ from copy import deepcopy
from twisted.internet import defer
from twisted.python.filepath import FilePath
+from ooni.deck.deck import NGDeck
from ooni.otime import timestampNowISO8601UTC
from ooni.resources import check_for_update
from ooni.settings import config
+from ooni.utils import log
class InputNotFound(Exception):
pass
+class DeckNotFound(Exception):
+ pass
+
class InputStore(object):
def __init__(self):
self.path = FilePath(config.inputs_directory)
@@ -117,12 +122,34 @@ class InputStore(object):
class DeckStore(object):
def __init__(self):
self.path = FilePath(config.decks_directory)
+ self._cache = {}
+ self._cache_stale = True
- def update(self):
- pass
+ def list(self):
+ decks = []
+ if self._cache_stale:
+ self._update_cache()
+ for deck_id, deck in self._cache.iteritems():
+ decks.append((deck_id, deck))
+ return decks
- def get(self):
- pass
+ def _update_cache(self):
+ for deck_path in self.path.listdir():
+ if not deck_path.endswith('.yaml'):
+ continue
+ deck_id = deck_path[:-1*len('.yaml')]
+ deck = NGDeck(
+ deck_path=self.path.child(deck_path).path
+ )
+ self._cache[deck_id] = deck
+ def get(self, deck_id):
+ if self._cache_stale:
+ self._update_cache()
+ try:
+ return self._cache[deck_id]
+ except KeyError:
+ raise DeckNotFound(deck_id)
+deck_store = DeckStore()
input_store = InputStore()
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 3f8efc8..30df9b9 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -7,6 +7,7 @@ import signal
from twisted.scripts import twistd
from twisted.python import usage
+from ooni.utils import log
from ooni.settings import config
from ooni.agent.agent import AgentService
@@ -48,8 +49,15 @@ class AgentOptions(usage.Options):
self.twistd_args = []
def start_agent(options=None):
- os.chdir(config.ooni_home)
- twistd_args = []
+ config.set_paths()
+ config.initialize_ooni_home()
+ config.read_config_file()
+
+ os.chdir(config.running_path)
+
+ # Since we are starting the logger below ourselves we make twistd log to
+ # a null log observer
+ twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull']
twistd_config = OoniprobeTwistdConfig()
if options is not None:
twistd_args.extend(options.twistd_args)
@@ -63,12 +71,13 @@ def start_agent(options=None):
}
print("Starting ooniprobe agent.")
print("To view the GUI go to %s" % WEB_UI_URL)
+ log.start()
twistd.runApp(twistd_config)
return 0
def status_agent():
pidfile = os.path.join(
- config.ooni_home,
+ config.running_path,
'twistd.pid'
)
if not os.path.exists(pidfile):
@@ -88,7 +97,7 @@ def status_agent():
def stop_agent():
# This function is borrowed from tahoe
pidfile = os.path.join(
- config.ooni_home,
+ config.running_path,
'twistd.pid'
)
if not os.path.exists(pidfile):
diff --git a/ooni/settings.py b/ooni/settings.py
index 194729a..5762919 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -1,4 +1,5 @@
import os
+import sys
import yaml
import getpass
from ConfigParser import SafeConfigParser
@@ -12,7 +13,6 @@ from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol
from ooni.utils import Storage, log, get_ooni_root
from ooni import errors
-
class OConfig(object):
_custom_home = None
@@ -41,18 +41,56 @@ class OConfig(object):
@property
def var_lib_path(self):
+ if hasattr(sys, 'real_prefix'):
+ # We are in a virtualenv use the /usr/share in the virtualenv
+ return os.path.join(
+ os.path.abspath(sys.prefix),
+ 'var', 'lib', 'ooni'
+ )
var_lib_path = self.embedded_settings("directories", "var_lib")
if var_lib_path:
return os.path.abspath(var_lib_path)
return "/var/lib/ooni"
@property
+ def running_path(self):
+ """
+ This is the directory used to store state application data.
+ It defaults to /var/lib/ooni, but if that is not writeable we will
+ use the ooni_home.
+ """
+ var_lib_path = self.var_lib_path
+ if os.access(var_lib_path, os.W_OK):
+ return var_lib_path
+ return self.ooni_home
+
+ @property
def usr_share_path(self):
+ if hasattr(sys, 'real_prefix'):
+ # We are in a virtualenv use the /usr/share in the virtualenv
+ return os.path.join(
+ os.path.abspath(sys.prefix),
+ 'usr', 'share', 'ooni'
+ )
usr_share_path = self.embedded_settings("directories", "usr_share")
if usr_share_path:
return os.path.abspath(usr_share_path)
return "/usr/share/ooni"
+
+ @property
+ def etc_path(self):
+ if hasattr(sys, 'real_prefix'):
+ # We are in a virtualenv use the /usr/share in the virtualenv
+ return os.path.join(
+ os.path.abspath(sys.prefix),
+ 'usr', 'share', 'ooni'
+ )
+ etc_path = self.embedded_settings("directories", "etc")
+ if etc_path:
+ return os.path.abspath(etc_path)
+ return "/etc"
+
@property
def data_directory_candidates(self):
dirs = [
@@ -93,28 +131,15 @@ class OConfig(object):
def set_paths(self):
self.nettest_directory = os.path.join(get_ooni_root(), 'nettests')
+ self.web_ui_directory = os.path.join(get_ooni_root(), 'web', 'client')
- if self.advanced.inputs_dir:
- self.inputs_directory = self.advanced.inputs_dir
- else:
- self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
-
- self.scheduler_directory = os.path.join(self.ooni_home, 'scheduler')
+ self.inputs_directory = os.path.join(self.running_path, 'inputs')
+ self.scheduler_directory = os.path.join(self.running_path, 'scheduler')
+ self.decks_directory = os.path.join(self.running_path, 'decks')
+ self.resources_directory = os.path.join(self.running_path, 'resources')
- if self.advanced.decks_dir:
- self.decks_directory = self.advanced.decks_dir
- else:
- self.decks_directory = os.path.join(self.ooni_home, 'decks')
-
- self.measurements_directory = os.path.join(self.ooni_home,
+ self.measurements_directory = os.path.join(self.running_path,
'measurements')
- self.resources_directory = os.path.join(self.ooni_home,
- "resources")
- if self.advanced.report_log_file:
- self.report_log_file = self.advanced.report_log_file
- else:
- self.report_log_file = os.path.join(self.ooni_home,
- 'reporting.yml')
if self.global_options.get('configfile'):
config_file = self.global_options['configfile']
@@ -123,8 +148,9 @@ class OConfig(object):
self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf')
if 'logfile' in self.basic:
- self.basic.logfile = expanduser(self.basic.logfile.replace(
- '~', '~'+self.current_user))
+ self.basic.logfile = expanduser(
+ self.basic.logfile.replace('~', '~'+self.current_user)
+ )
def initialize_ooni_home(self, custom_home=None):
if custom_home:
@@ -147,7 +173,7 @@ class OConfig(object):
]
for path in sub_directories:
try:
- os.mkdir(path)
+ os.makedirs(path)
except OSError as exc:
if exc.errno != 17:
raise
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index e1a6398..74cb235 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -26,11 +26,6 @@ from ooni.geoip import probe_ip
config.advanced.debug = True
-def rpath(*path):
- context = os.path.abspath(os.path.dirname(__file__))
- return os.path.join(context, *path)
-
-
class WebUIError(Exception):
def __init__(self, code, message):
self.code = code
@@ -392,5 +387,4 @@ class WebUIAPI(object):
@app.route('/client/', branch=True)
@xsrf_protect(check=False)
def static(self, request):
- path = rpath("client")
- return static.File(path)
+ return static.File(config.web_ui_directory)
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 982a353..1aba8dd 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -158,6 +158,8 @@ class OONILogger(object):
self.fileObserver.stop()
oonilogger = OONILogger()
+# This is a mock of a LoggerObserverFactory to be supplied to twistd.
+ooniloggerNull = lambda: lambda eventDict: None
start = oonilogger.start
stop = oonilogger.stop
More information about the tor-commits
mailing list