September 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Don't raise an exception if a deck is already enabled
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 886e19ec52e96a52c4a1d34d8126337eea18d66c Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Aug 31 19:42:58 2016 +0200 Don't raise an exception if a deck is already enabled * Disable debug level logging in web server --- ooni/deck/store.py | 6 +++++- ooni/ui/web/server.py | 2 -- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ooni/deck/store.py b/ooni/deck/store.py index 295f817..c938d56 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.… [View More]

1 0

[ooni-probe/master] Bind by default to localhost. Add support for disabling the web UI.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit e020064d31ee93e85fda73dc7a89ac55b97f4aa4 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Aug 26 18:44:53 2016 +0200 Bind by default to localhost. Add support for disabling the web UI. * Expose these options via configuration options This fixes #589 --- ooni/agent/agent.py | 9 +++++---- ooni/scripts/ooniprobe_agent.py | 6 +++--- ooni/settings.py | 10 +++++++++- ooni/ui/web/web.py | 3 ++- 4 files changed, 19 … [View More]insertions(+), 9 deletions(-) diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py index 0311cef..01e2162 100644 --- a/ooni/agent/agent.py +++ b/ooni/agent/agent.py @@ -14,10 +14,11 @@ class AgentService(service.MultiService): self.scheduler_service = SchedulerService(director) self.scheduler_service.setServiceParent(self) - self.web_ui_service = WebUIService(director, - self.scheduler_service, - web_ui_port) - self.web_ui_service.setServiceParent(self) + if not config.advanced.disabled_webui: + self.web_ui_service = WebUIService(director, + self.scheduler_service, + web_ui_port) + self.web_ui_service.setServiceParent(self) def startService(self): diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index 30df9b9..08fba89 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -11,14 +11,12 @@ from ooni.utils import log from ooni.settings import config from ooni.agent.agent import AgentService -WEB_UI_PORT = 8842 -WEB_UI_URL = "http://127.0.0.1:{0}".format(WEB_UI_PORT) class StartOoniprobeAgentPlugin: tapname = "ooniprobe" def makeService(self, so): - return AgentService(WEB_UI_PORT) + return AgentService(config.advanced.webui_port) class OoniprobeTwistdConfig(twistd.ServerOptions): subCommands = [ @@ -70,6 +68,8 @@ def start_agent(options=None): "StartOoniprobeAgent": StartOoniprobeAgentPlugin() } print("Starting ooniprobe agent.") + WEB_UI_URL = "http://{0}:{1}".format( + config.advanced.webui_address, config.advanced.webui_port) print("To view the GUI go to %s" % WEB_UI_URL) log.start() twistd.runApp(twistd_config) diff --git a/ooni/settings.py b/ooni/settings.py index 3da1da7..2bacd57 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -65,6 +65,11 @@ advanced: # insecure_backend: false # The preferred backend type, can be one of onion, https or cloudfront preferred_backend: {preferred_backend} + # The port and address for the Web UI + #webui_port: 8842 + #webui_address: "127.0.0.1" + # Should the Web UI be disabled + #disable_webui: false tor: #socks_port: 8801 #control_port: 8802 @@ -121,7 +126,10 @@ defaults = { "reporting_retries": 5, "reporting_concurrency": 7, "insecure_backend": False, - "preferred_backend": "onion" + "preferred_backend": "onion", + "webui_port": 8842, + "webui_address": "127.0.0.1", + "webui_disabled": False }, "tor": { "socks_port": None, diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py index 10bbef1..f36e0a2 100644 --- a/ooni/ui/web/web.py +++ b/ooni/ui/web/web.py @@ -19,7 +19,8 @@ class WebUIService(service.MultiService): web_ui_api = WebUIAPI(config, self.director, self.scheduler) self._port = reactor.listenTCP( self.port_number, - server.Site(web_ui_api.app.resource()) + server.Site(web_ui_api.app.resource()), + interface=config.advanced.webui_address ) def stopService(self): [View Less]

1 0

[ooni-probe/master] Don't look for the namebench csv file since we don't ship it
by art＠torproject.org 19 Sep '16

19 Sep '16

commit da00195094cdf22678d42608cdbc3bb22b9d757e Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Aug 10 18:11:06 2016 +0200 Don't look for the namebench csv file since we don't ship it * Include an empty file so that older ooniprobes don't end up always updating the resources on every cycle. --- ooni/deckgen/cli.py | 4 ---- setup.py | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py … [View More]

1 0

[ooni-probe/master] Add support for deck lifecycle in the web UI
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 7663106d25d272ffc7ff2208d95cddef3b685f62 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Aug 4 19:46:26 2016 +0200 Add support for deck lifecycle in the web UI * Fix a series of bugs found while testing * Bump the version number up one --- MANIFEST.in | 1 + data/decks/web-full.yaml | 23 ++++++ data/decks/web-no-invalid.yaml | 18 +++++ data/decks/web.yaml | 23 ------ ooni/__init__.py | 2 +- … [View More]ooni/agent/agent.py | 9 ++- ooni/agent/scheduler.py | 64 ++++++++++++++--- ooni/deck/deck.py | 9 ++- ooni/deck/store.py | 4 +- ooni/resources.py | 5 ++ ooni/scripts/oonideckgen.py | 20 ++---- ooni/scripts/ooniprobe.py | 3 +- ooni/scripts/oonireport.py | 7 +- ooni/settings.py | 154 +++++++++++++++++++---------------------- ooni/ui/cli.py | 37 ++++++++-- ooni/ui/web/client/index.html | 2 +- ooni/ui/web/server.py | 28 +++++++- ooni/ui/web/web.py | 5 +- ooni/utils/__init__.py | 4 +- setup.py | 29 +++----- 20 files changed, 271 insertions(+), 176 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 485e834..60d2ef9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,3 +9,4 @@ include data/ooniprobe.conf.sample include data/configs/lepidopter-ooniprobe.conf include data/configs/lepidopter-oonireport.conf include ooni/settings.ini +include ooni/ui/consent-form.md diff --git a/data/decks/web-full.yaml b/data/decks/web-full.yaml new file mode 100644 index 0000000..7812505 --- /dev/null +++ b/data/decks/web-full.yaml @@ -0,0 +1,23 @@ +--- +name: Full Web test deck +description: This deck runs HTTP Header Field Manipulation, HTTP Invalid + Request and the Web Connectivity test +schedule: "@daily" +tasks: +- name: Runs the HTTP Header Field Manipulation test + ooni: + test_name: http_header_field_manipulation + +- name: Runs the HTTP Invalid Request Line test + ooni: + test_name: http_invalid_request_line + +- name: Runs the Web Connectivity Test + ooni: + test_name: web_connectivity + file: $citizenlab_global_urls + +- name: Runs the Web Connectivity Test + ooni: + test_name: web_connectivity + file: $citizenlab_${probe_cc}_urls diff --git a/data/decks/web-no-invalid.yaml b/data/decks/web-no-invalid.yaml new file mode 100644 index 0000000..ea93488 --- /dev/null +++ b/data/decks/web-no-invalid.yaml @@ -0,0 +1,18 @@ +--- +name: Web test deck without HTTP Invalid Request Line +description: This deck runs HTTP Header Field Manipulation, and the Web Connectivity test +schedule: "@daily" +tasks: +- name: Runs the HTTP Header Field Manipulation test + ooni: + test_name: http_header_field_manipulation + +- name: Runs the Web Connectivity Test + ooni: + test_name: web_connectivity + file: $citizenlab_global_urls + +- name: Runs the Web Connectivity Test + ooni: + test_name: web_connectivity + file: $citizenlab_${probe_cc}_urls diff --git a/data/decks/web.yaml b/data/decks/web.yaml deleted file mode 100644 index c7b9bdc..0000000 --- a/data/decks/web.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -name: Web related ooniprobe tests -description: This deck runs HTTP Header Field Manipulation, HTTP Invalid - Request and the Web Connectivity test -schedule: "@daily" -tasks: -- name: Runs the HTTP Header Field Manipulation test - ooni: - test_name: http_header_field_manipulation - -- name: Runs the HTTP Invalid Request Line test - ooni: - test_name: http_invalid_request_line - -- name: Runs the Web Connectivity Test - ooni: - test_name: web_connectivity - file: $citizenlab_global_urls - -- name: Runs the Web Connectivity Test - ooni: - test_name: web_connectivity - file: $citizenlab_${probe_cc}_urls diff --git a/ooni/__init__.py b/ooni/__init__.py index 653a636..1a31608 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "2.0.0a0" +__version__ = "2.0.0a1" __all__ = [ 'agent', diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py index c2e7e26..0311cef 100644 --- a/ooni/agent/agent.py +++ b/ooni/agent/agent.py @@ -11,12 +11,15 @@ class AgentService(service.MultiService): director = Director() - self.web_ui_service = WebUIService(director, web_ui_port) - self.web_ui_service.setServiceParent(self) - self.scheduler_service = SchedulerService(director) self.scheduler_service.setServiceParent(self) + self.web_ui_service = WebUIService(director, + self.scheduler_service, + web_ui_port) + self.web_ui_service.setServiceParent(self) + + def startService(self): service.MultiService.startService(self) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 1f51bd4..74a1688 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -14,6 +14,29 @@ from ooni.contrib import croniter from ooni.geoip import probe_ip from ooni.measurements import list_measurements +class FileSystemlockAndMutex(object): + """ + This is a lock that is both a mutex lock and also on filesystem. + When you acquire it, it will first block on the mutex lock and then + once that is released it will attempt to acquire the lock on the + filesystem. + + It's a way to support concurrent usage of the DeferredFilesystemLock + without races. + """ + def __init__(self, file_path): + self._fs_lock = defer.DeferredFilesystemLock(file_path) + self._mutex = defer.DeferredLock() + + @defer.inlineCallbacks + def acquire(self): + yield self._mutex.acquire() + yield self._fs_lock.deferUntilLocked() + + def release(self): + self._fs_lock.unlock() + self._mutex.release() + class DidNotRun(Exception): pass @@ -33,7 +56,7 @@ class ScheduledTask(object): scheduler_directory = config.scheduler_directory self._last_run = FilePath(scheduler_directory).child(self.identifier) - self._last_run_lock = defer.DeferredFilesystemLock( + self._last_run_lock = FileSystemlockAndMutex( FilePath(scheduler_directory).child(self.identifier + ".lock").path ) @@ -63,9 +86,9 @@ class ScheduledTask(object): @defer.inlineCallbacks def run(self): - yield self._last_run_lock.deferUntilLocked() + yield self._last_run_lock.acquire() if not self.should_run: - self._last_run_lock.unlock() + self._last_run_lock.release() raise DidNotRun try: yield self.task() @@ -73,7 +96,7 @@ class ScheduledTask(object): except: raise finally: - self._last_run_lock.unlock() + self._last_run_lock.release() class UpdateInputsAndResources(ScheduledTask): @@ -140,6 +163,21 @@ class RunDeck(ScheduledTask): yield deck.setup() yield deck.run(self.director) + +class RefreshDeckList(ScheduledTask): + """ + This task is configured to refresh the list of decks that are enabled. + """ + identifier = 'refresh-deck-list' + schedule = '@hourly' + + def __init__(self, scheduler, schedule=None, identifier=None): + self.scheduler = scheduler + super(RefreshDeckList, self).__init__(schedule, identifier) + + def task(self): + self.scheduler.refresh_deck_list() + class SendHeartBeat(ScheduledTask): """ This task is used to send a heartbeat that the probe is still alive and @@ -188,6 +226,18 @@ class SchedulerService(service.MultiService): def schedule(self, task): self._scheduled_tasks.append(task) + def refresh_deck_list(self): + # Deletes all the RunDeck tasks and reschedules only the ones that + # are enabled. + for scheduled_task in self._scheduled_tasks[:]: + if isinstance(scheduled_task, RunDeck): + self._scheduled_tasks.remove(scheduled_task) + + for deck_id, deck in deck_store.list_enabled(): + if deck.schedule is None: + continue + self.schedule(RunDeck(self.director, deck_id, deck.schedule)) + def _task_did_not_run(self, failure, task): failure.trap(DidNotRun) log.debug("Did not run {0}".format(task.identifier)) @@ -214,13 +264,11 @@ class SchedulerService(service.MultiService): def startService(self): service.MultiService.startService(self) + self.refresh_deck_list() self.schedule(UpdateInputsAndResources()) self.schedule(UploadReports()) self.schedule(DeleteOldReports()) - for deck_id, deck in deck_store.list_enabled(): - if deck.schedule is None: - continue - self.schedule(RunDeck(self.director, deck_id, deck.schedule)) + self.schedule(RefreshDeckList(self)) self._looping_call.start(self.interval) diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index 1b2300a..75d6366 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -225,7 +225,12 @@ class NGDeck(object): measurement_id = task.id measurement_dir = self._measurement_path.child(measurement_id) - measurement_dir.createDirectory() + try: + measurement_dir.createDirectory() + except OSError as ose: + # Ignore 'File Exists' + if ose.errno != 17: + raise report_filename = measurement_dir.child("measurements.njson.progress").path pid_file = measurement_dir.child("running.pid") @@ -338,6 +343,8 @@ class DeckTask(object): if task_data.get('no-collector', False): collector_address = None + elif config.reports.upload is False: + collector_address = None net_test_loader = NetTestLoader( options_to_args(task_data), diff --git a/ooni/deck/store.py b/ooni/deck/store.py index 695f97d..2d24f29 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -140,7 +140,7 @@ class DeckStore(object): def list_enabled(self): decks = [] for deck_id, deck in self._list(): - if self.is_enabled(deck_id): + if not self.is_enabled(deck_id): continue decks.append((deck_id, deck)) return decks @@ -153,7 +153,7 @@ class DeckStore(object): if not deck_path.exists(): raise DeckNotFound(deck_id) deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml') - deck_enabled_path.linkTo(deck_path) + deck_path.linkTo(deck_enabled_path) def disable(self, deck_id): deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml') diff --git a/ooni/resources.py b/ooni/resources.py index aef0f13..9615c53 100644 --- a/ooni/resources.py +++ b/ooni/resources.py @@ -4,6 +4,11 @@ from twisted.python.filepath import FilePath from twisted.internet import defer from twisted.web.client import downloadPage, getPage, HTTPClientFactory +# WARNING: this script is being run as part of the post install procedure. +# Be sure to not import either in this module or in the imported modules +# dependencies other than twisted. If you end up including something that is +# not twisted, then you will need to add it to the setup_requires in setup.py. + from ooni.utils import log, gunzip, rename from ooni.settings import config diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py index 9b087f9..6c2882c 100644 --- a/ooni/scripts/oonideckgen.py +++ b/ooni/scripts/oonideckgen.py @@ -2,7 +2,6 @@ from __future__ import print_function import errno import os -import shutil import sys from twisted.internet import defer, task @@ -29,7 +28,7 @@ class Options(usage.Options): "submitting reports"], ["bouncer", None, None, "Specify a custom bouncer to use"], ["output", "o", None, - "Specify the directory where to write output."] + "Specify the path where we should be writing the deck to."] ] def opt_version(self): @@ -119,24 +118,13 @@ def oonideckgen(reactor): print("%s: --country-code must be 2 characters" % sys.argv[0]) sys.exit(2) - if not os.path.isdir(options['output']): - print("%s: %s is not a directory" % (sys.argv[0], - options['output'])) - sys.exit(3) + if os.path.isdir(options['output']): + options['output'] = os.path.join(options['output'], 'web-full.yaml') options['country-code'] = options['country-code'].lower() - output_dir = os.path.abspath(options['output']) - output_dir = os.path.join(output_dir, "deck") - - if os.path.isdir(output_dir): - print("Found previous deck deleting content of it") - shutil.rmtree(output_dir) - - options['output'] = output_dir - try: - os.makedirs(options['output']) + os.makedirs(os.path.dirname(options['output'])) except OSError as exception: if exception.errno != errno.EEXIST: raise diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py index 430252a..d67dd80 100644 --- a/ooni/scripts/ooniprobe.py +++ b/ooni/scripts/ooniprobe.py @@ -13,7 +13,8 @@ def ooniprobe(reactor): if global_options['queue']: return runWithDaemonDirector(global_options) elif global_options['initialize']: - return initializeOoniprobe(global_options) + initializeOoniprobe(global_options) + return defer.succeed(None) elif global_options['web-ui']: from ooni.scripts.ooniprobe_agent import WEB_UI_URL from ooni.scripts.ooniprobe_agent import status_agent, start_agent diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py index 13d8473..cd9b244 100644 --- a/ooni/scripts/oonireport.py +++ b/ooni/scripts/oonireport.py @@ -85,8 +85,7 @@ def upload(report_file, collector=None, bouncer=None, measurement_id=None): elif isinstance(collector_settings, str): collector_client = CollectorClient(address=collector_settings) else: - log.msg("Could not find %s in reporting.yaml. Looking up " - "collector with canonical bouncer." % report_file) + log.msg("Looking up collector with canonical bouncer." % report_file) collector_client = yield lookup_collector_client(report.header, CANONICAL_BOUNCER_ONION) @@ -267,8 +266,8 @@ class Options(usage.Options): def tor_check(): if not config.tor.socks_port: - print("Currently oonireport requires that you start Tor yourself " - "and set the socks_port inside of ooniprobe.conf") + log.err("Currently oonireport requires that you start Tor yourself " + "and set the socks_port inside of ooniprobe.conf") sys.exit(1) diff --git a/ooni/settings.py b/ooni/settings.py index 8bb3340..e7174e2 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -13,7 +13,6 @@ from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol from ooni.utils import Storage, log, get_ooni_root from ooni import errors - CONFIG_FILE_TEMPLATE = """\ # This is the configuration file for OONIProbe # This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html @@ -127,36 +126,81 @@ defaults = { "preferred_backend": "onion" }, "tor": { + "socks_port": None, + "control_port": None, + "bridges": None, + "data_dir": None, "timeout": 200, "torrc": {} } } +# This is the root of the ooniprobe source code tree +OONIPROBE_ROOT = get_ooni_root() + +IS_VIRTUALENV = False +if hasattr(sys, 'real_prefix'): + IS_VIRTUALENV = True + +# These are the the embedded settings +_SETTINGS_INI = os.path.join(OONIPROBE_ROOT, 'settings.ini') + +USR_SHARE_PATH = '/var/lib/ooni' +VAR_LIB_PATH = '/usr/share/ooni' +ETC_PATH = '/etc' + +if IS_VIRTUALENV: + _PREFIX = os.path.abspath(sys.prefix) + VAR_LIB_PATH = os.path.join( + _PREFIX, + 'var', 'lib', 'ooni' + ) + USR_SHARE_PATH = os.path.join( + _PREFIX, + 'usr', 'share', 'ooni' + ) + ETC_PATH = os.path.join( + _PREFIX, + 'etc' + ) +elif os.path.isfile(_SETTINGS_INI): + settings = SafeConfigParser() + with open(_SETTINGS_INI) as fp: + settings.readfp(fp) + + _USR_SHARE_PATH = settings.get('directories', 'usr_share') + if _USR_SHARE_PATH is not None: + USR_SHARE_PATH = _USR_SHARE_PATH + + _VAR_LIB_PATH = settings.get('directories', 'var_lib') + if _VAR_LIB_PATH is not None: + VAR_LIB_PATH = _VAR_LIB_PATH + + _ETC_PATH = settings.get('directories', 'etc') + if _ETC_PATH is not None: + ETC_PATH = _ETC_PATH + class OConfig(object): _custom_home = None def __init__(self): self.current_user = getpass.getuser() + self.global_options = {} - self.reports = Storage() + self.scapyFactory = None self.tor_state = None self.logging = True + + # These are the configuration options self.basic = Storage() self.advanced = Storage() + self.reports = Storage() self.tor = Storage() self.privacy = Storage() - self.set_paths() - def embedded_settings(self, category, option): - embedded_settings = os.path.join(get_ooni_root(), 'settings.ini') - if os.path.isfile(embedded_settings): - settings = SafeConfigParser() - with open(embedded_settings) as fp: - settings.readfp(fp) - return settings.get(category, option) - return None + self.set_paths() def is_initialized(self): # When this is false it means that the user has not gone @@ -170,64 +214,23 @@ class OConfig(object): with open(initialized_path, 'w+'): pass @property - def var_lib_path(self): - if hasattr(sys, 'real_prefix'): - # We are in a virtualenv use the /usr/share in the virtualenv - return os.path.join( - os.path.abspath(sys.prefix), - 'var', 'lib', 'ooni' - ) - var_lib_path = self.embedded_settings("directories", "var_lib") - if var_lib_path: - return os.path.abspath(var_lib_path) - return "/var/lib/ooni" - - @property def running_path(self): """ This is the directory used to store state application data. It defaults to /var/lib/ooni, but if that is not writeable we will use the ooni_home. """ - var_lib_path = self.var_lib_path - if os.access(var_lib_path, os.W_OK): - return var_lib_path + if os.access(VAR_LIB_PATH, os.W_OK): + return VAR_LIB_PATH return self.ooni_home @property - def usr_share_path(self): - if hasattr(sys, 'real_prefix'): - # We are in a virtualenv use the /usr/share in the virtualenv - return os.path.join( - os.path.abspath(sys.prefix), - 'usr', 'share', 'ooni' - ) - usr_share_path = self.embedded_settings("directories", "usr_share") - if usr_share_path: - return os.path.abspath(usr_share_path) - return "/usr/share/ooni" - - - @property - def etc_path(self): - if hasattr(sys, 'real_prefix'): - # We are in a virtualenv use the /usr/share in the virtualenv - return os.path.join( - os.path.abspath(sys.prefix), - 'usr', 'share', 'ooni' - ) - etc_path = self.embedded_settings("directories", "etc") - if etc_path: - return os.path.abspath(etc_path) - return "/etc" - - @property def data_directory_candidates(self): dirs = [ self.ooni_home, - self.var_lib_path, - self.usr_share_path, - os.path.join(get_ooni_root(), '..', 'data'), + VAR_LIB_PATH, + USR_SHARE_PATH, + os.path.join(OONIPROBE_ROOT, '..', 'data'), '/usr/share/' ] if os.getenv("OONI_DATA_DIR"): @@ -241,7 +244,7 @@ class OConfig(object): for target_dir in self.data_directory_candidates: if os.path.isdir(target_dir): return target_dir - return self.var_lib_path + return VAR_LIB_PATH @property def ooni_home(self): @@ -260,14 +263,14 @@ class OConfig(object): return file_path def set_paths(self): - self.nettest_directory = os.path.join(get_ooni_root(), 'nettests') - self.web_ui_directory = os.path.join(get_ooni_root(), 'ui', 'web', 'client') + self.nettest_directory = os.path.join(OONIPROBE_ROOT, 'nettests') + self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web','client') self.inputs_directory = os.path.join(self.running_path, 'inputs') self.scheduler_directory = os.path.join(self.running_path, 'scheduler') self.resources_directory = os.path.join(self.running_path, 'resources') - self.decks_available_directory = os.path.join(self.running_path, + self.decks_available_directory = os.path.join(USR_SHARE_PATH, 'decks-available') self.decks_enabled_directory = os.path.join(self.running_path, 'decks-enabled') @@ -317,6 +320,7 @@ class OConfig(object): def create_config_file(self, include_ip=False, include_asn=True, include_country=True, should_upload=True, preferred_backend="onion"): + self.initialize_ooni_home() def _bool_to_yaml(value): if value is True: return 'true' @@ -342,39 +346,23 @@ class OConfig(object): ) self.read_config_file() - def _create_config_file(self): - target_config_file = self.config_file - print "Creating it for you in '%s'." % target_config_file - sample_config_file = self.get_data_file_path('ooniprobe.conf.sample') - - with open(sample_config_file) as f: - with open(target_config_file, 'w+') as w: - for line in f: - if line.startswith(' logfile: '): - w.write(' logfile: %s\n' % ( - os.path.join(self.ooni_home, 'ooniprobe.log')) - ) - else: - w.write(line) - def read_config_file(self, check_incoherences=False): - #if not os.path.isfile(self.config_file): - # print "Configuration file does not exist." - # self._create_config_file() - # self.read_config_file() - configuration = {} + config_file = {} + log.debug("Reading config file from %s" % self.config_file) if os.path.isfile(self.config_file): with open(self.config_file) as f: config_file_contents = '\n'.join(f.readlines()) - configuration = yaml.safe_load(config_file_contents) + config_file = yaml.safe_load(config_file_contents) for category in defaults.keys(): + configuration[category] = {} for k, v in defaults[category].items(): try: - value = configuration.get(category, {})[k] + value = config_file.get(category, {})[k] except KeyError: value = v + configuration[category][k] = value getattr(self, category)[k] = value self.set_paths() diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 8cd3358..65e4a0f 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -10,7 +10,7 @@ from twisted.python import usage from twisted.internet import defer from ooni import errors, __version__ -from ooni.settings import config +from ooni.settings import config, OONIPROBE_ROOT from ooni.utils import log class LifetimeExceeded(Exception): pass @@ -182,7 +182,30 @@ def director_startup_other_failures(failure): def initializeOoniprobe(global_options): - # XXX print here the informed consent documentation. + print(""" + _ _ _ + __ _ _ _ ___ ___| |_(_)_ _ __ _ __| | + / _` | '_/ -_) -_) _| | ' \/ _` (_-<_| + \__, |_| \___\___|\__|_|_||_\__, /__(_) + |___/ |___/ ) + """) + print("It looks like this is the first time you are running ooniprobe") + print("Please take a minute to read through the informed consent documentation and " + "understand what are the risks associated with running ooniprobe.") + print("Press enter to continue...") + raw_input() + with open(os.path.join(OONIPROBE_ROOT, 'ui', 'consent-form.md')) as f: + consent_form_text = ''.join(f.readlines()) + from pydoc import pager + pager(consent_form_text) + + answer = "" + while answer.lower() != "yes": + print('Type "yes" if you are fully aware of the risks associated with using ooniprobe and you wish to proceed') + answer = raw_input("> ") + + print("") + print("Now help us configure some things!") answer = raw_input('Should we upload measurements to a collector? (Y/n) ') should_upload = True if answer.lower().startswith("n"): @@ -195,15 +218,15 @@ def initializeOoniprobe(global_options): answer = raw_input('Should we include your ASN (your network) in ' 'measurements? (Y/n) ') - include_asn = False + include_asn = True if answer.lower().startswith("n"): - include_asn = True + include_asn = False answer = raw_input('Should we include your Country in ' 'measurements? (Y/n) ') - include_country = False + include_country = True if answer.lower().startswith("n"): - include_country = True + include_country = False answer = raw_input('How would you like reports to be uploaded? (onion, ' 'https, cloudfronted) ') @@ -230,7 +253,7 @@ def setupGlobalOptions(logging, start_tor, check_incoherences): log.err("You first need to agree to the informed consent and setup " "ooniprobe to run it.") global_options['initialize'] = True - return + return global_options config.set_paths() config.initialize_ooni_home() diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index 6a7c149..ad2dc50 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?9c4ed560c98eaf61a836"></script></body> + <script type="text/javascript" src="app.bundle.js?f06164bd3b339e781c75"></script></body> </html> diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index ed2193e..bdafe75 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -149,8 +149,10 @@ class WebUIAPI(object): _reactor = reactor _enable_xsrf_protection = True - def __init__(self, config, director, _reactor=reactor): + def __init__(self, config, director, scheduler, _reactor=reactor): self.director = director + self.scheduler = scheduler + self.config = config self.measurement_path = FilePath(config.measurements_directory) @@ -301,17 +303,34 @@ class WebUIAPI(object): for deck_id, deck in self.director.deck_store.list(): deck_list['available'][deck_id] = { 'name': deck.name, - 'description': deck.description + 'description': deck.description, + 'schedule': deck.schedule, + 'enabled': self.director.deck_store.is_enabled(deck_id) } for deck_id, deck in self.director.deck_store.list_enabled(): deck_list['enabled'][deck_id] = { 'name': deck.name, - 'description': deck.description + 'description': deck.description, + 'schedule': deck.schedule, + 'enabled': True } return self.render_json(deck_list, request) + @app.route('/api/deck/<string:deck_id>/run', methods=["POST"]) + @xsrf_protect(check=True) + @requires_true(attrs=['_director_started', '_is_initialized']) + def api_deck_run(self, request, deck_id): + try: + deck = self.director.deck_store.get(deck_id) + except DeckNotFound: + raise WebUIError(404, "Deck not found") + + self.run_deck(deck) + + return self.render_json({"status": "starting"}, request) + @app.route('/api/deck/<string:deck_id>/enable', methods=["POST"]) @xsrf_protect(check=True) @requires_true(attrs=['_director_started', '_is_initialized']) @@ -321,6 +340,8 @@ class WebUIAPI(object): except DeckNotFound: raise WebUIError(404, "Deck not found") + self.scheduler.refresh_deck_list() + return self.render_json({"status": "enabled"}, request) @app.route('/api/deck/<string:deck_id>/disable', methods=["POST"]) @@ -331,6 +352,7 @@ class WebUIAPI(object): self.director.deck_store.disable(deck_id) except DeckNotFound: raise WebUIError(404, "Deck not found") + self.scheduler.refresh_deck_list() return self.render_json({"status": "disabled"}, request) diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py index eca75cb..10bbef1 100644 --- a/ooni/ui/web/web.py +++ b/ooni/ui/web/web.py @@ -6,16 +6,17 @@ from ooni.ui.web.server import WebUIAPI from ooni.settings import config class WebUIService(service.MultiService): - def __init__(self, director, port_number=8842): + def __init__(self, director, scheduler, port_number=8842): service.MultiService.__init__(self) self.director = director + self.scheduler = scheduler self.port_number = port_number def startService(self): service.MultiService.startService(self) - web_ui_api = WebUIAPI(config, self.director) + web_ui_api = WebUIAPI(config, self.director, self.scheduler) self._port = reactor.listenTCP( self.port_number, server.Site(web_ui_api.app.resource()) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index d672ca8..247758f 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -10,9 +10,6 @@ from zipfile import ZipFile from twisted.python.filepath import FilePath from twisted.python.runtime import platform -from ooni import errors - - class Storage(dict): """ A Storage object is like a dictionary except `obj.foo` can be used @@ -57,6 +54,7 @@ class Storage(dict): self[k] = v def checkForRoot(): + from ooni import errors if os.getuid() != 0: raise errors.InsufficientPrivileges diff --git a/setup.py b/setup.py index da48323..abbbf21 100644 --- a/setup.py +++ b/setup.py @@ -92,6 +92,8 @@ import os import shutil import tempfile import subprocess +from glob import glob + from ConfigParser import SafeConfigParser from os.path import join as pj @@ -138,30 +140,20 @@ class OoniInstall(install): else: var_path = pj(prefix, 'var', 'lib') - for root, dirs, file_names in os.walk('data/'): - files = [] - for file_name in file_names: - if file_name.endswith('.pyc'): - continue - elif file_name.endswith('.dat') and \ - file_name.startswith('Geo'): - continue - elif file_name == "ooniprobe.conf.sample": - files.append(self.gen_config(share_path)) - continue - files.append(pj(root, file_name)) - self.distribution.data_files.append( - [ - pj(share_path, 'ooni', root.replace('data/', '')), - files - ] + self.distribution.data_files.append( + ( + pj(share_path, 'ooni', 'decks-available'), + glob('data/decks/*') ) + ) settings = SafeConfigParser() settings.add_section("directories") settings.set("directories", "usr_share", os.path.join(share_path, "ooni")) settings.set("directories", "var_lib", os.path.join(var_path, "ooni")) + settings.set("directories", "etc", + os.path.join(var_path, "ooni")) with open("ooni/settings.ini", "w+") as fp: settings.write(fp) @@ -196,7 +188,7 @@ class OoniInstall(install): if is_lepidopter(): self.update_lepidopter_config() - +setup_requires = ['twisted'] install_requires = [] dependency_links = [] data_files = [] @@ -248,6 +240,7 @@ setup( include_package_data=True, dependency_links=dependency_links, install_requires=install_requires, + setup_requires=setup_requires, zip_safe=False, entry_points={ 'console_scripts': [ [View Less]

1 0

[ooni-probe/master] Merge pull request #581 from TheTorProject/feature/webui2-rb
by art＠torproject.org 19 Sep '16

19 Sep '16

1 0

[ooni-probe/master] Ignore global options that are None or set to zero.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 196fc17c2d62a8eb37e95e55f0dedaa2c53284f5 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Aug 29 09:39:14 2016 +0200 Ignore global options that are None or set to zero. * Always pop values from task_data to ensure the options are always clean. This closes #592 --- ooni/deck/deck.py | 16 ++++++++++------ ooni/tests/test_deck.py | 10 +++++++++- ooni/ui/cli.py | 4 ++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/… [View More]ooni/deck/deck.py b/ooni/deck/deck.py index 037c5d6..e4c35c4 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -317,12 +317,16 @@ class DeckTask(object): self._load(data) - def _get_option(self, name, task_data, default=None): + def _pop_option(self, name, task_data, default=None): try: - return self.global_options[name] + value = self.global_options[name] + if value in [None, 0]: + raise KeyError except KeyError: - return task_data.pop(name, - self.parent_metadata.get(name, default)) + value = task_data.pop(name, + self.parent_metadata.get(name, default)) + task_data.pop(name, None) + return value def _load_ooni(self, task_data): required_keys = ["test_name"] @@ -334,8 +338,8 @@ class DeckTask(object): nettest_path = nettest_to_path(task_data.pop("test_name"), self._arbitrary_paths) - annotations = self._get_option('annotations', task_data, {}) - collector_address = self._get_option('collector', task_data, None) + annotations = self._pop_option('annotations', task_data, {}) + collector_address = self._pop_option('collector', task_data, None) try: self.output_path = self.global_options['reportfile'] diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index 1bc6097..b2d05a6 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -14,7 +14,7 @@ from hashlib import sha256 from ooni import errors from ooni.deck.store import input_store from ooni.deck.backend import lookup_collector_and_test_helpers -from ooni.deck.deck import nettest_to_path, NGDeck +from ooni.deck.deck import nettest_to_path, NGDeck, options_to_args from ooni.deck.legacy import convert_legacy_deck from ooni.tests.bases import ConfigTestCase from ooni.tests.mocks import MockBouncerClient, MockCollectorClient @@ -330,3 +330,11 @@ class TestNGDeck(ConfigTestCase): "manipulation/http_header_field_manipulation", "blocking/web_connectivity" ]) + tasks = map(lambda task: task['ooni'], ng_deck['tasks']) + self.assertEqual( + tasks[2]['f'], + '/path/to/citizenlab-urls-global.txt') + + def test_options_to_args(self): + args = options_to_args({"f": "foobar.txt", "bar": None, "help": 0}) + print(args) diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 61f254b..3ad9605 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -338,8 +338,8 @@ def createDeck(global_options, url=None): raise SystemExit(3) except errors.OONIUsageError as e: - log.err(e) - print e.net_test_loader.usageOptions().getUsage() + log.exception(e) + map(log.msg, e.net_test_loader.usageOptions().getUsage().split("\n")) raise SystemExit(4) except errors.HTTPSCollectorUnsupported: [View Less]

1 0

[ooni-probe/master] settings.py: make sure default paths are okay
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 8048dbd1d1775dc658537b138e102a9e8319f147 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon Aug 8 19:24:29 2016 +0200 settings.py: make sure default paths are okay --- ooni/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/settings.py b/ooni/settings.py index 16de94d..3da1da7 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -143,8 +143,8 @@ if hasattr(sys, 'real_prefix'): # These are the the embedded settings … [View More]

1 0

[ooni-probe/master] Various bugfixes and stability fixes.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 42dde01be494c7902ad6860877e067ffb37f7513 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Aug 5 16:29:07 2016 +0200 Various bugfixes and stability fixes. --- Vagrantfile | 30 +++++++++++++----------------- ooni/agent/scheduler.py | 3 +++ ooni/deck/deck.py | 13 +++++++------ ooni/measurements.py | 6 +++++- ooni/settings.py | 4 ++-- ooni/ui/cli.py | 19 ------------------- ooni/ui/web/server.py | 2 +- setup.py … [View More] | 2 +- 8 files changed, 32 insertions(+), 47 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 0e75d55..71f7c7a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -17,6 +17,12 @@ apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libff cd /data/ooni-probe python setup.py install + +echo "To run ooniprobe:" +echo "1. vagrant ssh probe" +echo "2. ooniprobe-agent start" +echo "3. connect to the web UI on your host machine at http://localhost:8042/" + SCRIPT $setup_oonibackend = <<SCRIPT @@ -47,6 +53,13 @@ cp oonib.conf.example /etc/oonibackend.conf echo "Installing ooni-backend" python setup.py install +echo "" +echo "To run oonibackend:" +echo "1. vagrant ssh backend" +echo "2. vi /etc/oonibackend.conf # possibly" +echo "3. cd /data/ooni-backend" +echo "4. sudo ./bin/oonib -c /etc/oonibackend.conf" + SCRIPT Vagrant.configure("2") do |config| @@ -68,21 +81,4 @@ Vagrant.configure("2") do |config| end end - if File.directory?("../ooni-backend") - config.vm.provision "shell", inline: <<-EOF - echo "To run oonibackend:" - echo "1. vagrant ssh backend" - echo "2. vi /etc/oonibackend.conf # possibly" - echo "3. cd /data/ooni-backend" - echo "4. sudo ./bin/oonib -c /etc/oonibackend.conf" - EOF - end - - config.vm.provision "shell", inline: <<-EOF - echo "To run ooniprobe:" - echo "1. vagrant ssh probe" - echo "2. oonideckgen" - echo "3. ooniprobe -i deck-*/*.deck" - EOF - end diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 74a1688..3389db1 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -70,6 +70,7 @@ class ScheduledTask(object): @property def last_run(self): + self._last_run.restat(False) if not self._last_run.exists(): return datetime.fromtimestamp(0) with self._last_run.open('r') as in_file: @@ -208,6 +209,8 @@ def run_system_tasks(no_input_store=False): log.debug("Running task {0}".format(task.identifier)) try: yield task.run() + except DidNotRun: + log.debug("Did not run {0}".format(task.identifier)) except Exception as exc: log.err("Failed to run task {0}".format(task.identifier)) log.exception(exc) diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index 75d6366..037c5d6 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -259,8 +259,9 @@ class NGDeck(object): try: yield task.setup() except InputNotFound: - log.msg("Skipping this task because the input cannot be found") - self._skip = True + log.msg("Skipping the task {0} because the input cannot be " + "found".format(task.id)) + task.skip = True self._is_setup = True @defer.inlineCallbacks @@ -271,8 +272,8 @@ class NGDeck(object): yield director.start_tor() yield self.query_bouncer() for task in self._tasks: - if task._skip is True: - log.msg("Skipping running {0}".format(task.name)) + if task.skip is True: + log.msg("Skipping running {0}".format(task.id)) continue if task.type == "ooni": yield self._run_ooni_task(task, director) @@ -294,9 +295,9 @@ class DeckTask(object): self.cwd = cwd self.data = deepcopy(data) - self._skip = False + self.skip = False - self.id = "" + self.id = "invalid" self.type = None self.metadata = {} diff --git a/ooni/measurements.py b/ooni/measurements.py index cdf6b14..6d90c1b 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -3,6 +3,7 @@ import json import signal from twisted.python.filepath import FilePath +from ooni.utils import log from ooni.settings import config class MeasurementInProgress(Exception): @@ -118,7 +119,10 @@ def list_measurements(): measurements = [] measurement_path = FilePath(config.measurements_directory) for measurement_id in measurement_path.listdir(): - measurements.append(get_measurement(measurement_id)) + try: + measurements.append(get_measurement(measurement_id)) + except: + log.err("Failed to get metadata for measurement {0}".format(measurement_id)) return measurements if __name__ == "__main__": diff --git a/ooni/settings.py b/ooni/settings.py index e7174e2..16de94d 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -9,9 +9,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from os.path import abspath, expanduser -from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol from ooni.utils import Storage, log, get_ooni_root -from ooni import errors CONFIG_FILE_TEMPLATE = """\ # This is the configuration file for OONIProbe @@ -380,6 +378,7 @@ class OConfig(object): self.log_incoherences(incoherent) def log_incoherences(self, incoherences): + from ooni import errors if len(incoherences) > 0: if len(incoherences) > 1: incoherent_pretty = ", ".join(incoherences[:-1]) + ' and ' + incoherences[-1] @@ -393,6 +392,7 @@ class OConfig(object): """ Called only when we must start tor by director.start """ + from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol incoherent = [] if not self.advanced.start_tor: if self.tor.socks_port is None: diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 65e4a0f..61f254b 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -74,15 +74,6 @@ class Options(usage.Options): def __init__(self): usage.Options.__init__(self) - def getUsage(self, width=None): - return super(Options, self).getUsage(width) + """ -To get started you may want to run: - -$ oonideckgen - -This will tell you how to run ooniprobe :) -""" - def opt_spew(self): """ Print an insanely verbose log of everything that happens. @@ -109,9 +100,6 @@ This will tell you how to run ooniprobe :) def parseOptions(): - print "WARNING: running ooniprobe involves some risk that varies greatly" - print " from country to country. You should be aware of this when" - print " running the tool. Read more about this in the manpage or README." cmd_line_options = Options() if len(sys.argv) == 1: cmd_line_options.getUsage() @@ -182,13 +170,6 @@ def director_startup_other_failures(failure): def initializeOoniprobe(global_options): - print(""" - _ _ _ - __ _ _ _ ___ ___| |_(_)_ _ __ _ __| | - / _` | '_/ -_) -_) _| | ' \/ _` (_-<_| - \__, |_| \___\___|\__|_|_||_\__, /__(_) - |___/ |___/ ) - """) print("It looks like this is the first time you are running ooniprobe") print("Please take a minute to read through the informed consent documentation and " "understand what are the risks associated with running ooniprobe.") diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index bdafe75..ee33fa2 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -196,7 +196,7 @@ class WebUIAPI(object): } def handle_director_event(self, event): - log.msg("Handling event {0}".format(event.type)) + log.debug("Handling event {0}".format(event.type)) self.director_event_poller.notify(event) def director_started(self, _): diff --git a/setup.py b/setup.py index abbbf21..7612b39 100644 --- a/setup.py +++ b/setup.py @@ -188,7 +188,7 @@ class OoniInstall(install): if is_lepidopter(): self.update_lepidopter_config() -setup_requires = ['twisted'] +setup_requires = ['twisted', 'pyyaml'] install_requires = [] dependency_links = [] data_files = [] [View Less]

1 0

[ooni-probe/master] Add deck configuration step to the setup wizard
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 0c27e0602cfcfb172bc09d1ca2c35a612803f783 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Aug 31 16:32:38 2016 +0200 Add deck configuration step to the setup wizard --- ooni/ui/web/server.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 6f9ef69..2e89370 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -190,7 +190,8 @@ class WebUIAPI(object)… [View More]: def status(self): quota_warning = None try: - with open(os.path.join(config.running_dir, "quota_warning")) as in_file: + with open(os.path.join(config.running_path, + "quota_warning")) as in_file: quota_warning = in_file.read() except IOError as ioe: if ioe.errno != errno.ENOENT: @@ -259,6 +260,21 @@ class WebUIAPI(object): d.addCallback(got_status_update) return d + @app.route('/api/initialize', methods=["GET"]) + @xsrf_protect(check=False) + @requires_false(attrs=['_is_initialized']) + def api_initialize_get(self, request): + available_decks = [] + for deck_id, deck in self.director.deck_store.list(): + available_decks.append({ + 'name': deck.name, + 'description': deck.description, + 'schedule': deck.schedule, + 'enabled': self.director.deck_store.is_enabled(deck_id), + 'id': deck_id + }) + return self.render_json({"available_decks": available_decks}, request) + @app.route('/api/initialize', methods=["POST"]) @xsrf_protect(check=True) @requires_false(attrs=['_is_initialized']) @@ -278,6 +294,26 @@ class WebUIAPI(object): raise WebUIError(400, 'Missing required key {0}'.format( required_key)) config.create_config_file(**options) + try: + deck_config = initial_configuration['deck_config'] + except KeyError: + raise WebUIError(400, 'Missing enabled decks') + + for deck_id, enabled in deck_config.items(): + try: + if enabled is True: + self.director.deck_store.enable(deck_id) + elif enabled is False: + try: + self.director.deck_store.disable(deck_id) + except DeckNotFound: + # We ignore these errors, because it could be that a deck + # that is marked as disabled is already disabled + pass + except DeckNotFound: + raise WebUIError(404, 'Deck not found') + + self.scheduler.refresh_deck_list() config.set_initialized() self._is_initialized = True [View Less]

1 0

[ooni-probe/master] web-full.yaml: add missing word
by art＠torproject.org 19 Sep '16

19 Sep '16

commit eb7971a108201bed8f6735a14f73000072ab9960 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon Aug 8 18:46:27 2016 +0200 web-full.yaml: add missing word --- data/decks/web-full.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/decks/web-full.yaml b/data/decks/web-full.yaml index 7812505..ccb470d 100644 --- a/data/decks/web-full.yaml +++ b/data/decks/web-full.yaml @@ -1,7 +1,7 @@ --- name: Full Web test deck description: This deck runs HTTP … [View More]

1 0