[tor-commits] [ooni-probe/master] Add support for deck lifecycle in the web UI
art at torproject.org
art at torproject.org
Mon Sep 19 12:14:24 UTC 2016
commit 7663106d25d272ffc7ff2208d95cddef3b685f62
Author: Arturo Filastò <arturo at filasto.net>
Date: Thu Aug 4 19:46:26 2016 +0200
Add support for deck lifecycle in the web UI
* Fix a series of bugs found while testing
* Bump the version number up one
---
MANIFEST.in | 1 +
data/decks/web-full.yaml | 23 ++++++
data/decks/web-no-invalid.yaml | 18 +++++
data/decks/web.yaml | 23 ------
ooni/__init__.py | 2 +-
ooni/agent/agent.py | 9 ++-
ooni/agent/scheduler.py | 64 ++++++++++++++---
ooni/deck/deck.py | 9 ++-
ooni/deck/store.py | 4 +-
ooni/resources.py | 5 ++
ooni/scripts/oonideckgen.py | 20 ++----
ooni/scripts/ooniprobe.py | 3 +-
ooni/scripts/oonireport.py | 7 +-
ooni/settings.py | 154 +++++++++++++++++++----------------------
ooni/ui/cli.py | 37 ++++++++--
ooni/ui/web/client/index.html | 2 +-
ooni/ui/web/server.py | 28 +++++++-
ooni/ui/web/web.py | 5 +-
ooni/utils/__init__.py | 4 +-
setup.py | 29 +++-----
20 files changed, 271 insertions(+), 176 deletions(-)
diff --git a/MANIFEST.in b/MANIFEST.in
index 485e834..60d2ef9 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -9,3 +9,4 @@ include data/ooniprobe.conf.sample
include data/configs/lepidopter-ooniprobe.conf
include data/configs/lepidopter-oonireport.conf
include ooni/settings.ini
+include ooni/ui/consent-form.md
diff --git a/data/decks/web-full.yaml b/data/decks/web-full.yaml
new file mode 100644
index 0000000..7812505
--- /dev/null
+++ b/data/decks/web-full.yaml
@@ -0,0 +1,23 @@
+---
+name: Full Web test deck
+description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
+ Request and the Web Connectivity test
+schedule: "@daily"
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+ ooni:
+ test_name: http_header_field_manipulation
+
+- name: Runs the HTTP Invalid Request Line test
+ ooni:
+ test_name: http_invalid_request_line
+
+- name: Runs the Web Connectivity Test
+ ooni:
+ test_name: web_connectivity
+ file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+ ooni:
+ test_name: web_connectivity
+ file: $citizenlab_${probe_cc}_urls
diff --git a/data/decks/web-no-invalid.yaml b/data/decks/web-no-invalid.yaml
new file mode 100644
index 0000000..ea93488
--- /dev/null
+++ b/data/decks/web-no-invalid.yaml
@@ -0,0 +1,18 @@
+---
+name: Web test deck without HTTP Invalid Request Line
+description: This deck runs HTTP Header Field Manipulation, and the Web Connectivity test
+schedule: "@daily"
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+ ooni:
+ test_name: http_header_field_manipulation
+
+- name: Runs the Web Connectivity Test
+ ooni:
+ test_name: web_connectivity
+ file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+ ooni:
+ test_name: web_connectivity
+ file: $citizenlab_${probe_cc}_urls
diff --git a/data/decks/web.yaml b/data/decks/web.yaml
deleted file mode 100644
index c7b9bdc..0000000
--- a/data/decks/web.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
----
-name: Web related ooniprobe tests
-description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
- Request and the Web Connectivity test
-schedule: "@daily"
-tasks:
-- name: Runs the HTTP Header Field Manipulation test
- ooni:
- test_name: http_header_field_manipulation
-
-- name: Runs the HTTP Invalid Request Line test
- ooni:
- test_name: http_invalid_request_line
-
-- name: Runs the Web Connectivity Test
- ooni:
- test_name: web_connectivity
- file: $citizenlab_global_urls
-
-- name: Runs the Web Connectivity Test
- ooni:
- test_name: web_connectivity
- file: $citizenlab_${probe_cc}_urls
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 653a636..1a31608 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
# -*- encoding: utf-8 -*-
__author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0a0"
+__version__ = "2.0.0a1"
__all__ = [
'agent',
diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index c2e7e26..0311cef 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -11,12 +11,15 @@ class AgentService(service.MultiService):
director = Director()
- self.web_ui_service = WebUIService(director, web_ui_port)
- self.web_ui_service.setServiceParent(self)
-
self.scheduler_service = SchedulerService(director)
self.scheduler_service.setServiceParent(self)
+ self.web_ui_service = WebUIService(director,
+ self.scheduler_service,
+ web_ui_port)
+ self.web_ui_service.setServiceParent(self)
+
+
def startService(self):
service.MultiService.startService(self)
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 1f51bd4..74a1688 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -14,6 +14,29 @@ from ooni.contrib import croniter
from ooni.geoip import probe_ip
from ooni.measurements import list_measurements
+class FileSystemlockAndMutex(object):
+ """
+ This is a lock that is both a mutex lock and also on filesystem.
+ When you acquire it, it will first block on the mutex lock and then
+ once that is released it will attempt to acquire the lock on the
+ filesystem.
+
+ It's a way to support concurrent usage of the DeferredFilesystemLock
+ without races.
+ """
+ def __init__(self, file_path):
+ self._fs_lock = defer.DeferredFilesystemLock(file_path)
+ self._mutex = defer.DeferredLock()
+
+ @defer.inlineCallbacks
+ def acquire(self):
+ yield self._mutex.acquire()
+ yield self._fs_lock.deferUntilLocked()
+
+ def release(self):
+ self._fs_lock.unlock()
+ self._mutex.release()
+
class DidNotRun(Exception):
pass
@@ -33,7 +56,7 @@ class ScheduledTask(object):
scheduler_directory = config.scheduler_directory
self._last_run = FilePath(scheduler_directory).child(self.identifier)
- self._last_run_lock = defer.DeferredFilesystemLock(
+ self._last_run_lock = FileSystemlockAndMutex(
FilePath(scheduler_directory).child(self.identifier + ".lock").path
)
@@ -63,9 +86,9 @@ class ScheduledTask(object):
@defer.inlineCallbacks
def run(self):
- yield self._last_run_lock.deferUntilLocked()
+ yield self._last_run_lock.acquire()
if not self.should_run:
- self._last_run_lock.unlock()
+ self._last_run_lock.release()
raise DidNotRun
try:
yield self.task()
@@ -73,7 +96,7 @@ class ScheduledTask(object):
except:
raise
finally:
- self._last_run_lock.unlock()
+ self._last_run_lock.release()
class UpdateInputsAndResources(ScheduledTask):
@@ -140,6 +163,21 @@ class RunDeck(ScheduledTask):
yield deck.setup()
yield deck.run(self.director)
+
+class RefreshDeckList(ScheduledTask):
+ """
+ This task is configured to refresh the list of decks that are enabled.
+ """
+ identifier = 'refresh-deck-list'
+ schedule = '@hourly'
+
+ def __init__(self, scheduler, schedule=None, identifier=None):
+ self.scheduler = scheduler
+ super(RefreshDeckList, self).__init__(schedule, identifier)
+
+ def task(self):
+ self.scheduler.refresh_deck_list()
+
class SendHeartBeat(ScheduledTask):
"""
This task is used to send a heartbeat that the probe is still alive and
@@ -188,6 +226,18 @@ class SchedulerService(service.MultiService):
def schedule(self, task):
self._scheduled_tasks.append(task)
+ def refresh_deck_list(self):
+ # Deletes all the RunDeck tasks and reschedules only the ones that
+ # are enabled.
+ for scheduled_task in self._scheduled_tasks[:]:
+ if isinstance(scheduled_task, RunDeck):
+ self._scheduled_tasks.remove(scheduled_task)
+
+ for deck_id, deck in deck_store.list_enabled():
+ if deck.schedule is None:
+ continue
+ self.schedule(RunDeck(self.director, deck_id, deck.schedule))
+
def _task_did_not_run(self, failure, task):
failure.trap(DidNotRun)
log.debug("Did not run {0}".format(task.identifier))
@@ -214,13 +264,11 @@ class SchedulerService(service.MultiService):
def startService(self):
service.MultiService.startService(self)
+ self.refresh_deck_list()
self.schedule(UpdateInputsAndResources())
self.schedule(UploadReports())
self.schedule(DeleteOldReports())
- for deck_id, deck in deck_store.list_enabled():
- if deck.schedule is None:
- continue
- self.schedule(RunDeck(self.director, deck_id, deck.schedule))
+ self.schedule(RefreshDeckList(self))
self._looping_call.start(self.interval)
diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 1b2300a..75d6366 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -225,7 +225,12 @@ class NGDeck(object):
measurement_id = task.id
measurement_dir = self._measurement_path.child(measurement_id)
- measurement_dir.createDirectory()
+ try:
+ measurement_dir.createDirectory()
+ except OSError as ose:
+ # Ignore 'File Exists'
+ if ose.errno != 17:
+ raise
report_filename = measurement_dir.child("measurements.njson.progress").path
pid_file = measurement_dir.child("running.pid")
@@ -338,6 +343,8 @@ class DeckTask(object):
if task_data.get('no-collector', False):
collector_address = None
+ elif config.reports.upload is False:
+ collector_address = None
net_test_loader = NetTestLoader(
options_to_args(task_data),
diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 695f97d..2d24f29 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -140,7 +140,7 @@ class DeckStore(object):
def list_enabled(self):
decks = []
for deck_id, deck in self._list():
- if self.is_enabled(deck_id):
+ if not self.is_enabled(deck_id):
continue
decks.append((deck_id, deck))
return decks
@@ -153,7 +153,7 @@ class DeckStore(object):
if not deck_path.exists():
raise DeckNotFound(deck_id)
deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
- deck_enabled_path.linkTo(deck_path)
+ deck_path.linkTo(deck_enabled_path)
def disable(self, deck_id):
deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
diff --git a/ooni/resources.py b/ooni/resources.py
index aef0f13..9615c53 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -4,6 +4,11 @@ from twisted.python.filepath import FilePath
from twisted.internet import defer
from twisted.web.client import downloadPage, getPage, HTTPClientFactory
+# WARNING: this script is being run as part of the post install procedure.
+# Be sure to not import either in this module or in the imported modules
+# dependencies other than twisted. If you end up including something that is
+# not twisted, then you will need to add it to the setup_requires in setup.py.
+
from ooni.utils import log, gunzip, rename
from ooni.settings import config
diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
index 9b087f9..6c2882c 100644
--- a/ooni/scripts/oonideckgen.py
+++ b/ooni/scripts/oonideckgen.py
@@ -2,7 +2,6 @@ from __future__ import print_function
import errno
import os
-import shutil
import sys
from twisted.internet import defer, task
@@ -29,7 +28,7 @@ class Options(usage.Options):
"submitting reports"],
["bouncer", None, None, "Specify a custom bouncer to use"],
["output", "o", None,
- "Specify the directory where to write output."]
+ "Specify the path where we should be writing the deck to."]
]
def opt_version(self):
@@ -119,24 +118,13 @@ def oonideckgen(reactor):
print("%s: --country-code must be 2 characters" % sys.argv[0])
sys.exit(2)
- if not os.path.isdir(options['output']):
- print("%s: %s is not a directory" % (sys.argv[0],
- options['output']))
- sys.exit(3)
+ if os.path.isdir(options['output']):
+ options['output'] = os.path.join(options['output'], 'web-full.yaml')
options['country-code'] = options['country-code'].lower()
- output_dir = os.path.abspath(options['output'])
- output_dir = os.path.join(output_dir, "deck")
-
- if os.path.isdir(output_dir):
- print("Found previous deck deleting content of it")
- shutil.rmtree(output_dir)
-
- options['output'] = output_dir
-
try:
- os.makedirs(options['output'])
+ os.makedirs(os.path.dirname(options['output']))
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
index 430252a..d67dd80 100644
--- a/ooni/scripts/ooniprobe.py
+++ b/ooni/scripts/ooniprobe.py
@@ -13,7 +13,8 @@ def ooniprobe(reactor):
if global_options['queue']:
return runWithDaemonDirector(global_options)
elif global_options['initialize']:
- return initializeOoniprobe(global_options)
+ initializeOoniprobe(global_options)
+ return defer.succeed(None)
elif global_options['web-ui']:
from ooni.scripts.ooniprobe_agent import WEB_UI_URL
from ooni.scripts.ooniprobe_agent import status_agent, start_agent
diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py
index 13d8473..cd9b244 100644
--- a/ooni/scripts/oonireport.py
+++ b/ooni/scripts/oonireport.py
@@ -85,8 +85,7 @@ def upload(report_file, collector=None, bouncer=None, measurement_id=None):
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
else:
- log.msg("Could not find %s in reporting.yaml. Looking up "
- "collector with canonical bouncer." % report_file)
+ log.msg("Looking up collector with canonical bouncer." % report_file)
collector_client = yield lookup_collector_client(report.header,
CANONICAL_BOUNCER_ONION)
@@ -267,8 +266,8 @@ class Options(usage.Options):
def tor_check():
if not config.tor.socks_port:
- print("Currently oonireport requires that you start Tor yourself "
- "and set the socks_port inside of ooniprobe.conf")
+ log.err("Currently oonireport requires that you start Tor yourself "
+ "and set the socks_port inside of ooniprobe.conf")
sys.exit(1)
diff --git a/ooni/settings.py b/ooni/settings.py
index 8bb3340..e7174e2 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -13,7 +13,6 @@ from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol
from ooni.utils import Storage, log, get_ooni_root
from ooni import errors
-
CONFIG_FILE_TEMPLATE = """\
# This is the configuration file for OONIProbe
# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
@@ -127,36 +126,81 @@ defaults = {
"preferred_backend": "onion"
},
"tor": {
+ "socks_port": None,
+ "control_port": None,
+ "bridges": None,
+ "data_dir": None,
"timeout": 200,
"torrc": {}
}
}
+# This is the root of the ooniprobe source code tree
+OONIPROBE_ROOT = get_ooni_root()
+
+IS_VIRTUALENV = False
+if hasattr(sys, 'real_prefix'):
+ IS_VIRTUALENV = True
+
+# These are the the embedded settings
+_SETTINGS_INI = os.path.join(OONIPROBE_ROOT, 'settings.ini')
+
+USR_SHARE_PATH = '/var/lib/ooni'
+VAR_LIB_PATH = '/usr/share/ooni'
+ETC_PATH = '/etc'
+
+if IS_VIRTUALENV:
+ _PREFIX = os.path.abspath(sys.prefix)
+ VAR_LIB_PATH = os.path.join(
+ _PREFIX,
+ 'var', 'lib', 'ooni'
+ )
+ USR_SHARE_PATH = os.path.join(
+ _PREFIX,
+ 'usr', 'share', 'ooni'
+ )
+ ETC_PATH = os.path.join(
+ _PREFIX,
+ 'etc'
+ )
+elif os.path.isfile(_SETTINGS_INI):
+ settings = SafeConfigParser()
+ with open(_SETTINGS_INI) as fp:
+ settings.readfp(fp)
+
+ _USR_SHARE_PATH = settings.get('directories', 'usr_share')
+ if _USR_SHARE_PATH is not None:
+ USR_SHARE_PATH = _USR_SHARE_PATH
+
+ _VAR_LIB_PATH = settings.get('directories', 'var_lib')
+ if _VAR_LIB_PATH is not None:
+ VAR_LIB_PATH = _VAR_LIB_PATH
+
+ _ETC_PATH = settings.get('directories', 'etc')
+ if _ETC_PATH is not None:
+ ETC_PATH = _ETC_PATH
+
class OConfig(object):
_custom_home = None
def __init__(self):
self.current_user = getpass.getuser()
+
self.global_options = {}
- self.reports = Storage()
+
self.scapyFactory = None
self.tor_state = None
self.logging = True
+
+ # These are the configuration options
self.basic = Storage()
self.advanced = Storage()
+ self.reports = Storage()
self.tor = Storage()
self.privacy = Storage()
- self.set_paths()
- def embedded_settings(self, category, option):
- embedded_settings = os.path.join(get_ooni_root(), 'settings.ini')
- if os.path.isfile(embedded_settings):
- settings = SafeConfigParser()
- with open(embedded_settings) as fp:
- settings.readfp(fp)
- return settings.get(category, option)
- return None
+ self.set_paths()
def is_initialized(self):
# When this is false it means that the user has not gone
@@ -170,64 +214,23 @@ class OConfig(object):
with open(initialized_path, 'w+'): pass
@property
- def var_lib_path(self):
- if hasattr(sys, 'real_prefix'):
- # We are in a virtualenv use the /usr/share in the virtualenv
- return os.path.join(
- os.path.abspath(sys.prefix),
- 'var', 'lib', 'ooni'
- )
- var_lib_path = self.embedded_settings("directories", "var_lib")
- if var_lib_path:
- return os.path.abspath(var_lib_path)
- return "/var/lib/ooni"
-
- @property
def running_path(self):
"""
This is the directory used to store state application data.
It defaults to /var/lib/ooni, but if that is not writeable we will
use the ooni_home.
"""
- var_lib_path = self.var_lib_path
- if os.access(var_lib_path, os.W_OK):
- return var_lib_path
+ if os.access(VAR_LIB_PATH, os.W_OK):
+ return VAR_LIB_PATH
return self.ooni_home
@property
- def usr_share_path(self):
- if hasattr(sys, 'real_prefix'):
- # We are in a virtualenv use the /usr/share in the virtualenv
- return os.path.join(
- os.path.abspath(sys.prefix),
- 'usr', 'share', 'ooni'
- )
- usr_share_path = self.embedded_settings("directories", "usr_share")
- if usr_share_path:
- return os.path.abspath(usr_share_path)
- return "/usr/share/ooni"
-
-
- @property
- def etc_path(self):
- if hasattr(sys, 'real_prefix'):
- # We are in a virtualenv use the /usr/share in the virtualenv
- return os.path.join(
- os.path.abspath(sys.prefix),
- 'usr', 'share', 'ooni'
- )
- etc_path = self.embedded_settings("directories", "etc")
- if etc_path:
- return os.path.abspath(etc_path)
- return "/etc"
-
- @property
def data_directory_candidates(self):
dirs = [
self.ooni_home,
- self.var_lib_path,
- self.usr_share_path,
- os.path.join(get_ooni_root(), '..', 'data'),
+ VAR_LIB_PATH,
+ USR_SHARE_PATH,
+ os.path.join(OONIPROBE_ROOT, '..', 'data'),
'/usr/share/'
]
if os.getenv("OONI_DATA_DIR"):
@@ -241,7 +244,7 @@ class OConfig(object):
for target_dir in self.data_directory_candidates:
if os.path.isdir(target_dir):
return target_dir
- return self.var_lib_path
+ return VAR_LIB_PATH
@property
def ooni_home(self):
@@ -260,14 +263,14 @@ class OConfig(object):
return file_path
def set_paths(self):
- self.nettest_directory = os.path.join(get_ooni_root(), 'nettests')
- self.web_ui_directory = os.path.join(get_ooni_root(), 'ui', 'web', 'client')
+ self.nettest_directory = os.path.join(OONIPROBE_ROOT, 'nettests')
+ self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web','client')
self.inputs_directory = os.path.join(self.running_path, 'inputs')
self.scheduler_directory = os.path.join(self.running_path, 'scheduler')
self.resources_directory = os.path.join(self.running_path, 'resources')
- self.decks_available_directory = os.path.join(self.running_path,
+ self.decks_available_directory = os.path.join(USR_SHARE_PATH,
'decks-available')
self.decks_enabled_directory = os.path.join(self.running_path,
'decks-enabled')
@@ -317,6 +320,7 @@ class OConfig(object):
def create_config_file(self, include_ip=False, include_asn=True,
include_country=True, should_upload=True,
preferred_backend="onion"):
+ self.initialize_ooni_home()
def _bool_to_yaml(value):
if value is True:
return 'true'
@@ -342,39 +346,23 @@ class OConfig(object):
)
self.read_config_file()
- def _create_config_file(self):
- target_config_file = self.config_file
- print "Creating it for you in '%s'." % target_config_file
- sample_config_file = self.get_data_file_path('ooniprobe.conf.sample')
-
- with open(sample_config_file) as f:
- with open(target_config_file, 'w+') as w:
- for line in f:
- if line.startswith(' logfile: '):
- w.write(' logfile: %s\n' % (
- os.path.join(self.ooni_home, 'ooniprobe.log'))
- )
- else:
- w.write(line)
-
def read_config_file(self, check_incoherences=False):
- #if not os.path.isfile(self.config_file):
- # print "Configuration file does not exist."
- # self._create_config_file()
- # self.read_config_file()
-
configuration = {}
+ config_file = {}
+ log.debug("Reading config file from %s" % self.config_file)
if os.path.isfile(self.config_file):
with open(self.config_file) as f:
config_file_contents = '\n'.join(f.readlines())
- configuration = yaml.safe_load(config_file_contents)
+ config_file = yaml.safe_load(config_file_contents)
for category in defaults.keys():
+ configuration[category] = {}
for k, v in defaults[category].items():
try:
- value = configuration.get(category, {})[k]
+ value = config_file.get(category, {})[k]
except KeyError:
value = v
+ configuration[category][k] = value
getattr(self, category)[k] = value
self.set_paths()
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 8cd3358..65e4a0f 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -10,7 +10,7 @@ from twisted.python import usage
from twisted.internet import defer
from ooni import errors, __version__
-from ooni.settings import config
+from ooni.settings import config, OONIPROBE_ROOT
from ooni.utils import log
class LifetimeExceeded(Exception): pass
@@ -182,7 +182,30 @@ def director_startup_other_failures(failure):
def initializeOoniprobe(global_options):
- # XXX print here the informed consent documentation.
+ print("""
+ _ _ _
+ __ _ _ _ ___ ___| |_(_)_ _ __ _ __| |
+ / _` | '_/ -_) -_) _| | ' \/ _` (_-<_|
+ \__, |_| \___\___|\__|_|_||_\__, /__(_)
+ |___/ |___/ )
+ """)
+ print("It looks like this is the first time you are running ooniprobe")
+ print("Please take a minute to read through the informed consent documentation and "
+ "understand what are the risks associated with running ooniprobe.")
+ print("Press enter to continue...")
+ raw_input()
+ with open(os.path.join(OONIPROBE_ROOT, 'ui', 'consent-form.md')) as f:
+ consent_form_text = ''.join(f.readlines())
+ from pydoc import pager
+ pager(consent_form_text)
+
+ answer = ""
+ while answer.lower() != "yes":
+ print('Type "yes" if you are fully aware of the risks associated with using ooniprobe and you wish to proceed')
+ answer = raw_input("> ")
+
+ print("")
+ print("Now help us configure some things!")
answer = raw_input('Should we upload measurements to a collector? (Y/n) ')
should_upload = True
if answer.lower().startswith("n"):
@@ -195,15 +218,15 @@ def initializeOoniprobe(global_options):
answer = raw_input('Should we include your ASN (your network) in '
'measurements? (Y/n) ')
- include_asn = False
+ include_asn = True
if answer.lower().startswith("n"):
- include_asn = True
+ include_asn = False
answer = raw_input('Should we include your Country in '
'measurements? (Y/n) ')
- include_country = False
+ include_country = True
if answer.lower().startswith("n"):
- include_country = True
+ include_country = False
answer = raw_input('How would you like reports to be uploaded? (onion, '
'https, cloudfronted) ')
@@ -230,7 +253,7 @@ def setupGlobalOptions(logging, start_tor, check_incoherences):
log.err("You first need to agree to the informed consent and setup "
"ooniprobe to run it.")
global_options['initialize'] = True
- return
+ return global_options
config.set_paths()
config.initialize_ooni_home()
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index 6a7c149..ad2dc50 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
<app>
Loading...
</app>
- <script type="text/javascript" src="app.bundle.js?9c4ed560c98eaf61a836"></script></body>
+ <script type="text/javascript" src="app.bundle.js?f06164bd3b339e781c75"></script></body>
</html>
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index ed2193e..bdafe75 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -149,8 +149,10 @@ class WebUIAPI(object):
_reactor = reactor
_enable_xsrf_protection = True
- def __init__(self, config, director, _reactor=reactor):
+ def __init__(self, config, director, scheduler, _reactor=reactor):
self.director = director
+ self.scheduler = scheduler
+
self.config = config
self.measurement_path = FilePath(config.measurements_directory)
@@ -301,17 +303,34 @@ class WebUIAPI(object):
for deck_id, deck in self.director.deck_store.list():
deck_list['available'][deck_id] = {
'name': deck.name,
- 'description': deck.description
+ 'description': deck.description,
+ 'schedule': deck.schedule,
+ 'enabled': self.director.deck_store.is_enabled(deck_id)
}
for deck_id, deck in self.director.deck_store.list_enabled():
deck_list['enabled'][deck_id] = {
'name': deck.name,
- 'description': deck.description
+ 'description': deck.description,
+ 'schedule': deck.schedule,
+ 'enabled': True
}
return self.render_json(deck_list, request)
+ @app.route('/api/deck/<string:deck_id>/run', methods=["POST"])
+ @xsrf_protect(check=True)
+ @requires_true(attrs=['_director_started', '_is_initialized'])
+ def api_deck_run(self, request, deck_id):
+ try:
+ deck = self.director.deck_store.get(deck_id)
+ except DeckNotFound:
+ raise WebUIError(404, "Deck not found")
+
+ self.run_deck(deck)
+
+ return self.render_json({"status": "starting"}, request)
+
@app.route('/api/deck/<string:deck_id>/enable', methods=["POST"])
@xsrf_protect(check=True)
@requires_true(attrs=['_director_started', '_is_initialized'])
@@ -321,6 +340,8 @@ class WebUIAPI(object):
except DeckNotFound:
raise WebUIError(404, "Deck not found")
+ self.scheduler.refresh_deck_list()
+
return self.render_json({"status": "enabled"}, request)
@app.route('/api/deck/<string:deck_id>/disable', methods=["POST"])
@@ -331,6 +352,7 @@ class WebUIAPI(object):
self.director.deck_store.disable(deck_id)
except DeckNotFound:
raise WebUIError(404, "Deck not found")
+ self.scheduler.refresh_deck_list()
return self.render_json({"status": "disabled"}, request)
diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py
index eca75cb..10bbef1 100644
--- a/ooni/ui/web/web.py
+++ b/ooni/ui/web/web.py
@@ -6,16 +6,17 @@ from ooni.ui.web.server import WebUIAPI
from ooni.settings import config
class WebUIService(service.MultiService):
- def __init__(self, director, port_number=8842):
+ def __init__(self, director, scheduler, port_number=8842):
service.MultiService.__init__(self)
self.director = director
+ self.scheduler = scheduler
self.port_number = port_number
def startService(self):
service.MultiService.startService(self)
- web_ui_api = WebUIAPI(config, self.director)
+ web_ui_api = WebUIAPI(config, self.director, self.scheduler)
self._port = reactor.listenTCP(
self.port_number,
server.Site(web_ui_api.app.resource())
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index d672ca8..247758f 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -10,9 +10,6 @@ from zipfile import ZipFile
from twisted.python.filepath import FilePath
from twisted.python.runtime import platform
-from ooni import errors
-
-
class Storage(dict):
"""
A Storage object is like a dictionary except `obj.foo` can be used
@@ -57,6 +54,7 @@ class Storage(dict):
self[k] = v
def checkForRoot():
+ from ooni import errors
if os.getuid() != 0:
raise errors.InsufficientPrivileges
diff --git a/setup.py b/setup.py
index da48323..abbbf21 100644
--- a/setup.py
+++ b/setup.py
@@ -92,6 +92,8 @@ import os
import shutil
import tempfile
import subprocess
+from glob import glob
+
from ConfigParser import SafeConfigParser
from os.path import join as pj
@@ -138,30 +140,20 @@ class OoniInstall(install):
else:
var_path = pj(prefix, 'var', 'lib')
- for root, dirs, file_names in os.walk('data/'):
- files = []
- for file_name in file_names:
- if file_name.endswith('.pyc'):
- continue
- elif file_name.endswith('.dat') and \
- file_name.startswith('Geo'):
- continue
- elif file_name == "ooniprobe.conf.sample":
- files.append(self.gen_config(share_path))
- continue
- files.append(pj(root, file_name))
- self.distribution.data_files.append(
- [
- pj(share_path, 'ooni', root.replace('data/', '')),
- files
- ]
+ self.distribution.data_files.append(
+ (
+ pj(share_path, 'ooni', 'decks-available'),
+ glob('data/decks/*')
)
+ )
settings = SafeConfigParser()
settings.add_section("directories")
settings.set("directories", "usr_share",
os.path.join(share_path, "ooni"))
settings.set("directories", "var_lib",
os.path.join(var_path, "ooni"))
+ settings.set("directories", "etc",
+ os.path.join(var_path, "ooni"))
with open("ooni/settings.ini", "w+") as fp:
settings.write(fp)
@@ -196,7 +188,7 @@ class OoniInstall(install):
if is_lepidopter():
self.update_lepidopter_config()
-
+setup_requires = ['twisted']
install_requires = []
dependency_links = []
data_files = []
@@ -248,6 +240,7 @@ setup(
include_package_data=True,
dependency_links=dependency_links,
install_requires=install_requires,
+ setup_requires=setup_requires,
zip_safe=False,
entry_points={
'console_scripts': [
More information about the tor-commits
mailing list