[tor-commits] [ooni-probe/master] Use NGDeck also for the command line version of ooniprobe
art at torproject.org
art at torproject.org
Mon Sep 19 12:14:24 UTC 2016
commit 76cc0d4989de0a777fa62f4de5b60f07e0701da7
Author: Arturo Filastò <arturo at filasto.net>
Date: Tue Jul 26 02:05:58 2016 +0200
Use NGDeck also for the command line version of ooniprobe
* Remove code that is now dead.
---
bin/Makefile | 81 --------
bin/ooniprobe | 24 ---
bin/ooniprobe-dev | 10 -
bin/test/ooni/__init__.py | 1 -
ooni/backend_client.py | 95 +--------
ooni/deck.py | 356 +++++++++++++++++++++------------
ooni/director.py | 2 -
ooni/geoip.py | 6 +-
ooni/reporter.py | 2 +-
ooni/scripts/oonideckgen.py | 2 +-
ooni/scripts/ooniprobe.py | 19 ++
ooni/scripts/oonireport.py | 2 +-
ooni/templates/tcpt.py | 1 -
ooni/tests/mocks.py | 2 +-
ooni/tests/test_backend_client.py | 16 --
ooni/tests/test_deck.py | 243 +++++++++++-----------
ooni/ui/cli.py | 124 +++++-------
ooni/ui/web/client/index.html | 2 +-
scripts/__init__.py | 0
scripts/set_caps/Makefile | 81 ++++++++
scripts/set_caps/__init__.py | 0
scripts/set_caps/test/__init__.py | 0
scripts/set_caps/test/ooni/__init__.py | 1 +
23 files changed, 517 insertions(+), 553 deletions(-)
diff --git a/bin/Makefile b/bin/Makefile
deleted file mode 100644
index bc80843..0000000
--- a/bin/Makefile
+++ /dev/null
@@ -1,81 +0,0 @@
-# Wrappers for running ooniprobe as a non-root user.
-#
-# Build-Depends: cython, pythonX.Y-dev, libcap2-bin
-# Depends: libpythonX.Y
-#
-# $ make && make check
-# $ sudo make install # after installing the rest of ooni-probe
-# $ make installcheck_unsafe # runs complete tests as non-root
-#
-# `make` builds a program that has file capabilities set on it. This is just
-# ./ooniprobe compiled into a C program using Cython, so that one can set
-# capabilities directly on the resulting binary. This way, we avoid the need
-# for a separate child python interpreter with its own capabilities. Another
-# advantage is that libpython.so (needed by the program) would be automatically
-# upgraded by the system package manager. The version of python is hard-coded
-# into the wrapper at build time; making this dynamic is possible, but much
-# more complex and not yet implemented.
-#
-# Execution may additionally be limited to a particular unix group by using
-# chgrp(1) and chmod(1) to 'o-x,g+x' after installation.
-#
-
-# GNU Makefile conventions, see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
-prefix = /usr/local
-exec_prefix = $(prefix)
-bindir = $(exec_prefix)/bin
-
-INSTALL = install
-PYTHON = python
-PYTHON_CONFIG = python-config
-CYTHON = cython
-SETCAP = setcap
-
-INSTALL_PROGRAM = $(INSTALL)
-PY_CFLAGS = $(shell $(PYTHON_CONFIG) --cflags)
-PY_LDFLAGS = $(shell $(PYTHON_CONFIG) --ldflags)
-
-BUILDDIR := ./build
-SCRIPTDIR := .
-TESTDIR := ./test
-CAP_SCRIPT := ooniprobe
-CAP_NEEDED := cap_net_admin,cap_net_raw
-
-# Unfortunately cython --embed ignores the arguments in the shebang line
-# So we need to patch the generated code ourselves.
-CYTHON_PRE_MAIN = extern int Py_IgnoreEnvironmentFlag; \
- Py_IgnoreEnvironmentFlag++; \
- extern int Py_NoUserSiteDirectory; \
- Py_NoUserSiteDirectory++;
-
-all: $(BUILDDIR)/$(CAP_SCRIPT)
-
-$(BUILDDIR)/$(CAP_SCRIPT): $(BUILDDIR)/$(CAP_SCRIPT).c Makefile
- $(CC) $(PY_CFLAGS) $(PY_LDFLAGS) "$<" -o "$@"
-
-$(BUILDDIR)/$(CAP_SCRIPT).c: $(SCRIPTDIR)/$(CAP_SCRIPT) Makefile
- mkdir -p "$(BUILDDIR)"
- $(CYTHON) "$<" --embed=CYTHON_MAIN_SENTINEL -Werror -Wextra -o "$@"
- sed -i \
- -e 's/\(.*CYTHON_MAIN_SENTINEL.*{\)/\1 $(CYTHON_PRE_MAIN)/g' \
- -e '/CYTHON_MAIN_SENTINEL[^{]*$$/,/{/s/{/{ $(CYTHON_PRE_MAIN)/g' \
- -e 's/CYTHON_MAIN_SENTINEL/main/g' "$@"
-
-check: $(BUILDDIR)/$(CAP_SCRIPT)
- # test that setcapped binary ignores PYTHONPATH
- BIN="$$(realpath "$<")" && cd "$(TESTDIR)" && PYTHONPATH=. $$BIN --version
-
-install: $(BUILDDIR)/$(CAP_SCRIPT)
- mkdir -p "$(DESTDIR)$(bindir)"
- $(INSTALL_PROGRAM) -t "$(DESTDIR)$(bindir)" "$(BUILDDIR)/$(CAP_SCRIPT)"
- $(SETCAP) "$(CAP_NEEDED)"+eip "$(DESTDIR)$(bindir)/$(CAP_SCRIPT)"
-
-installcheck_unsafe: $(BUILDDIR)/$(CAP_SCRIPT)
- # run a standard check. note that because of hardcoded paths (for security)
- # this can only work after you've installed your development copy
- "./$<" -i /usr/share/ooni/decks/complete.deck
-
-clean:
- rm -rf "$(BUILDDIR)"
-
-.PHONY: clean all check install installcheck%
diff --git a/bin/ooniprobe b/bin/ooniprobe
deleted file mode 100755
index 0274899..0000000
--- a/bin/ooniprobe
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-import sys
-
-from twisted.internet import reactor
-
-from ooni.ui.cli import runWithDaemonDirector, runWithDirector
-from ooni.ui.cli import setupGlobalOptions
-
-exit_code=0
-
-global_options = setupGlobalOptions(logging=True, start_tor=True,
- check_incoherences=True)
-if global_options['queue']:
- d = runWithDaemonDirector(global_options)
-else:
- d = runWithDirector(global_options)
- at d.addBoth
-def cb(result):
- global exit_code
- if result is not None:
- exit_code=1
- reactor.stop()
-reactor.run()
-sys.exit(exit_code)
diff --git a/bin/ooniprobe-dev b/bin/ooniprobe-dev
deleted file mode 100755
index c44a8a6..0000000
--- a/bin/ooniprobe-dev
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# Developer script for running ooniprobe directly from the repository.
-# We don't automatically add "$PWD" to PYTHONPATH as that is a security risk
-# when run as /usr/bin/ooniprobe on an end user's computer.
-ROOTDIR=$(cd $(dirname $(dirname $0)) && pwd -P)
-if [ $? -ne 0 ]; then
- echo "$0: cannot determine toplevel directory" 1>&2
- exit 1
-fi
-PYTHONPATH="$ROOTDIR" exec $ROOTDIR/bin/ooniprobe "$@"
diff --git a/bin/test/ooni/__init__.py b/bin/test/ooni/__init__.py
deleted file mode 100644
index f2cd353..0000000
--- a/bin/test/ooni/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-raise ValueError("test failed! wrapper did not ignore polluted PWD. either the wrapper is faulty, or ooni is still unpatched (Tor bug #13581)")
diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index 506aba4..7721f6f 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -134,6 +134,8 @@ class OONIBClient(object):
return finished
def queryBackend(self, method, urn, query=None, retries=3):
+ log.debug("Querying backend {0}{1} with {2}".format(self.base_address,
+ urn, query))
bodyProducer = None
if query:
bodyProducer = StringProducer(json.dumps(query))
@@ -212,105 +214,12 @@ class CollectorClient(OONIBClient):
return d
- def getInput(self, input_hash):
- from ooni.deck import InputFile
-
- input_file = InputFile(input_hash)
- if input_file.descriptorCached:
- return defer.succeed(input_file)
- else:
- d = self.queryBackend('GET', '/input/' + input_hash)
-
- @d.addCallback
- def cb(descriptor):
- input_file.load(descriptor)
- input_file.save()
- return input_file
-
- @d.addErrback
- def err(err):
- log.err("Failed to get descriptor for input %s" % input_hash)
- log.exception(err)
-
- return d
-
- def getInputList(self):
- return self.queryBackend('GET', '/input')
-
- def downloadInput(self, input_hash):
- from ooni.deck import InputFile
-
- input_file = InputFile(input_hash)
-
- if input_file.fileCached:
- return defer.succeed(input_file)
- else:
- d = self.download('/input/' + input_hash + '/file', input_file.cached_file)
-
- @d.addCallback
- def cb(res):
- input_file.verify()
- return input_file
-
- @d.addErrback
- def err(err):
- log.err("Failed to download the input file %s" % input_hash)
- log.exception(err)
-
- return d
-
def getInputPolicy(self):
return self.queryBackend('GET', '/policy/input')
def getNettestPolicy(self):
return self.queryBackend('GET', '/policy/nettest')
- def getDeckList(self):
- return self.queryBackend('GET', '/deck')
-
- def getDeck(self, deck_hash):
- from ooni.deck import Deck
-
- deck = Deck(deck_hash)
- if deck.descriptorCached:
- return defer.succeed(deck)
- else:
- d = self.queryBackend('GET', '/deck/' + deck_hash)
-
- @d.addCallback
- def cb(descriptor):
- deck.load(descriptor)
- deck.save()
- return deck
-
- @d.addErrback
- def err(err):
- log.err("Failed to get descriptor for deck %s" % deck_hash)
- log.exception(err)
-
- return d
-
- def downloadDeck(self, deck_hash):
- from ooni.deck import Deck
-
- deck = Deck(deck_hash)
- if deck.fileCached:
- return defer.succeed(deck)
- else:
- d = self.download('/deck/' + deck_hash + '/file', deck.cached_file)
-
- @d.addCallback
- def cb(res):
- deck.verify()
- return deck
-
- @d.addErrback
- def err(err):
- log.err("Failed to download the deck %s" % deck_hash)
- log.exception(err)
-
- return d
-
def createReport(self, test_details):
request = {
'software_name': test_details['software_name'],
diff --git a/ooni/deck.py b/ooni/deck.py
index 1746d26..eacd256 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -24,59 +24,6 @@ from ooni.utils import log
from ooni.results import generate_summary
-class InputFile(object):
- def __init__(self, input_hash, base_path=config.inputs_directory):
- self.id = input_hash
- cache_path = os.path.join(os.path.abspath(base_path), input_hash)
- self.cached_file = cache_path
- self.cached_descriptor = cache_path + '.desc'
-
- @property
- def descriptorCached(self):
- if os.path.exists(self.cached_descriptor):
- with open(self.cached_descriptor) as f:
- descriptor = json.load(f)
- self.load(descriptor)
- return True
- return False
-
- @property
- def fileCached(self):
- if os.path.exists(self.cached_file):
- try:
- self.verify()
- except AssertionError:
- log.err("The input %s failed validation."
- "Going to consider it not cached." % self.id)
- return False
- return True
- return False
-
- def save(self):
- with open(self.cached_descriptor, 'w+') as f:
- json.dump({
- 'name': self.name,
- 'id': self.id,
- 'version': self.version,
- 'author': self.author,
- 'date': self.date,
- 'description': self.description
- }, f)
-
- def load(self, descriptor):
- self.name = descriptor['name']
- self.version = descriptor['version']
- self.author = descriptor['author']
- self.date = descriptor['date']
- self.description = descriptor['description']
-
- def verify(self):
- digest = os.path.basename(self.cached_file)
- with open(self.cached_file) as f:
- file_hash = sha256(f.read())
- assert file_hash.hexdigest() == digest
-
-
def nettest_to_path(path, allow_arbitrary_paths=False):
"""
Takes as input either a path or a nettest name.
@@ -136,6 +83,7 @@ def get_preferred_bouncer():
else:
return BouncerClient(bouncer_address)
+<<<<<<< d0fb4f37530aeb6b69fbc2985019464f8ec10312
class Deck(InputFile):
# this exists so we can mock it out in unittests
_BouncerClient = BouncerClient
@@ -274,6 +222,7 @@ class Deck(InputFile):
i['test_options'][i['key']] = input_file.cached_file
+ at defer.inlineCallbacks
def lookup_collector_and_test_helpers(net_test_loaders,
bouncer,
preferred_backend,
@@ -329,19 +278,21 @@ def lookup_collector_and_test_helpers(net_test_loaders,
net_test_loader.testName)
collector, test_helpers = \
- find_collector_and_test_helpers(test_name=net_test_loader.testName,
- test_version=net_test_loader.testVersion,
- input_files=net_test_loader.inputFiles)
+ find_collector_and_test_helpers(
+ test_name=net_test_loader.testName,
+ test_version=net_test_loader.testVersion,
+ input_files=net_test_loader.inputFiles
+ )
for option, name in net_test_loader.missingTestHelpers:
test_helper_address_or_settings = test_helpers[name]
net_test_loader.localOptions[option] = test_helper_address_or_settings
net_test_loader.testHelpers[option] = test_helper_address_or_settings
- if not net_test_loader.collector:
+ if not net_test_loader.collector and not no_collector:
+ log.debug("Using collector {0}".format(collector))
net_test_loader.collector = collector
-
@defer.inlineCallbacks
def get_reachable_test_helpers_and_collectors(net_tests, preferred_backend):
for net_test in net_tests:
@@ -579,10 +530,29 @@ def options_to_args(options, prepath=None):
continue
if k == "file":
v = resolve_file_path(v, prepath)
- args.append('--'+k)
+ if v == False or v == 0:
+ continue
+ if (len(k)) == 1:
+ args.append('-'+k)
+ else:
+ args.append('--'+k)
+ if isinstance(v, bool) or isinstance(v, int):
+ continue
args.append(v)
return args
+def normalize_options(options):
+ """
+ Takes some options that have a mixture of - and _ and returns the
+ equivalent options with only '_'.
+ """
+ normalized_opts = {}
+ for k, v in options.items():
+ normalized_key = k.replace('-', '_')
+ assert normalized_key not in normalized_opts, "The key {0} cannot be normalized".format(k)
+ normalized_opts[normalized_key] = v
+ return normalized_opts
+
class UnknownTaskKey(Exception):
pass
@@ -593,8 +563,14 @@ class DeckTask(object):
_metadata_keys = ["name"]
_supported_tasks = ["ooni"]
- def __init__(self, data, parent_metadata={}, cwd=None):
- self.parent_metadata = parent_metadata
+ def __init__(self, data,
+ parent_metadata={},
+ global_options={},
+ cwd=None,
+ arbitrary_paths=False):
+
+ self.parent_metadata = normalize_options(parent_metadata)
+ self.global_options = global_options
self.cwd = cwd
self.data = deepcopy(data)
@@ -605,10 +581,16 @@ class DeckTask(object):
self.requires_tor = False
self.requires_bouncer = False
+ # If this is set to true a deck can specify any path. It should only
+ # be run against trusted decks or when you create a deck
+ # programmaticaly to a run test specified from the command line.
+ self._arbitrary_paths = arbitrary_paths
+
self.ooni = {
'bouncer_client': None,
'test_details': {}
}
+ self.output_path = None
self._load(data)
@@ -619,7 +601,8 @@ class DeckTask(object):
raise MissingTaskDataKey(required_key)
# This raises e.NetTestNotFound, we let it go onto the caller
- nettest_path = nettest_to_path(task_data.pop("test_name"))
+ nettest_path = nettest_to_path(task_data.pop("test_name"),
+ self._arbitrary_paths)
try:
annotations = task_data.pop('annotations')
@@ -631,8 +614,16 @@ class DeckTask(object):
except KeyError:
collector_address = self.parent_metadata.get('collector', None)
+ try:
+ self.output_path = task_data.pop('reportfile')
+ except KeyError:
+ self.output_path = self.global_options.get('reportfile', None)
+
+ if task_data.get('no-collector', False):
+ collector_address = None
+
net_test_loader = NetTestLoader(
- options_to_args(task_data),
+ options_to_args(task_data, self.cwd),
annotations=annotations,
test_file=nettest_path
)
@@ -658,11 +649,14 @@ class DeckTask(object):
self.requires_bouncer = True
self.ooni['net_test_loader'] = net_test_loader
- # Need to ensure that this is called only once we have looked up the
- # probe IP address and have geoip data.
- self.ooni['test_details'] = net_test_loader.getTestDetails()
+
+ def _setup_ooni(self):
+ self.ooni['test_details'] = self.ooni['net_test_loader'].getTestDetails()
self.id = generate_filename(self.ooni['test_details'])
+ def setup(self):
+ getattr(self, "_setup_"+self.type)()
+
def _load(self, data):
for key in self._metadata_keys:
try:
@@ -678,11 +672,81 @@ class DeckTask(object):
assert len(data) == 0
+class NotAnOption(Exception):
+ pass
+
+def subargs_to_options(subargs):
+ options = {}
+
+ def parse_option_name(arg):
+ if arg.startswith("--"):
+ return arg[2:]
+ elif arg.startswith("-"):
+ return arg[1:]
+ raise NotAnOption
+
+ subargs = iter(reversed(subargs))
+ for subarg in subargs:
+ try:
+ value = subarg
+ name = parse_option_name(subarg)
+ options[name] = True
+ except NotAnOption:
+ try:
+ name = parse_option_name(subargs.next())
+ options[name] = value
+ except StopIteration:
+ break
+
+ return options
+
+def convert_legacy_deck(deck_data):
+ """
+ I take a legacy deck list and convert it to the new deck format.
+
+ :param deck_data: in the legacy format
+ :return: deck_data in the new format
+ """
+ assert isinstance(deck_data, list), "Legacy decks are lists"
+ new_deck_data = {}
+ new_deck_data["name"] = "Legacy deck"
+ new_deck_data["description"] = "This is a legacy deck converted to the " \
+ "new format"
+ new_deck_data["bouncer"] = None
+ new_deck_data["tasks"] = []
+ for deck_item in deck_data:
+ deck_task = {"ooni": {}}
+
+ options = deck_item["options"]
+ deck_task["ooni"]["test_name"] = options.pop("test_file")
+ deck_task["ooni"]["annotations"] = options.pop("annotations", {})
+ deck_task["ooni"]["collector"] = options.pop("collector", None)
+
+ # XXX here we end up picking only the last not none bouncer_address
+ bouncer_address = options.pop("bouncer", None)
+ if bouncer_address is not None:
+ new_deck_data["bouncer"] = bouncer_address
+
+ subargs = options.pop("subargs", [])
+ for name, value in subargs_to_options(subargs).items():
+ deck_task["ooni"][name] = value
+
+ for name, value in options.items():
+ deck_task["ooni"][name] = value
+
+ new_deck_data["tasks"].append(deck_task)
+
+ return new_deck_data
+
class NGDeck(object):
- def __init__(self, deck_data=None,
- deck_path=None, no_collector=False):
+ def __init__(self,
+ deck_data=None,
+ deck_path=None,
+ global_options={},
+ no_collector=False,
+ arbitrary_paths=False):
# Used to resolve relative paths inside of decks.
- self.deck_directory = None
+ self.deck_directory = os.getcwd()
self.requires_tor = False
self.no_collector = no_collector
self.name = ""
@@ -690,8 +754,12 @@ class NGDeck(object):
self.schedule = None
self.metadata = {}
+ self.global_options = normalize_options(global_options)
self.bouncer = None
+ self._arbitrary_paths = arbitrary_paths
+ self._is_setup = False
+
self._measurement_path = FilePath(config.measurements_directory)
self._tasks = []
self.task_ids = []
@@ -701,35 +769,24 @@ class NGDeck(object):
elif deck_data is not None:
self.load(deck_data)
- def open(self, deck_path):
+ def open(self, deck_path, global_options=None):
with open(deck_path) as fh:
deck_data = yaml.safe_load(fh)
- self.load(deck_data)
+ self.deck_directory = os.path.abspath(os.path.dirname(deck_path))
+ self.load(deck_data, global_options)
- def write(self, fh):
- """
- Writes a properly formatted deck to the supplied file handle.
- :param fh: an open file handle
- :return:
- """
- deck_data = {
- "name": self.name,
- "description": self.description,
- "tasks": [task.data for task in self._tasks]
- }
- if self.schedule is not None:
- deck_data["schedule"] = self.schedule
- for key, value in self.metadata.items():
- deck_data[key] = value
+ def load(self, deck_data, global_options=None):
+ if global_options is not None:
+ self.global_options = global_options
- fh.write("---\n")
- yaml.safe_dump(deck_data, fh, default_flow_style=False)
+ if isinstance(deck_data, list):
+ deck_data = convert_legacy_deck(deck_data)
- def load(self, deck_data):
self.name = deck_data.pop("name", "Un-named Deck")
self.description = deck_data.pop("description", "No description")
- bouncer_address = deck_data.pop("bouncer", None)
+ bouncer_address = self.global_options.get('bouncer',
+ deck_data.pop("bouncer", None))
if bouncer_address is None:
self.bouncer = get_preferred_bouncer()
elif isinstance(bouncer_address, dict):
@@ -743,8 +800,17 @@ class NGDeck(object):
for key, metadata in deck_data.items():
self.metadata[key] = metadata
+ # We override the task metadata with the global options if present
+ self.metadata.update(self.global_options)
+
for task_data in tasks_data:
- deck_task = DeckTask(task_data, self.metadata, self.deck_directory)
+ deck_task = DeckTask(
+ data=task_data,
+ parent_metadata=self.metadata,
+ global_options=self.global_options,
+ cwd=self.deck_directory,
+ arbitrary_paths=self._arbitrary_paths
+ )
if deck_task.requires_tor:
self.requires_tor = True
if (deck_task.requires_bouncer and
@@ -753,6 +819,32 @@ class NGDeck(object):
self._tasks.append(deck_task)
self.task_ids.append(deck_task.id)
+ if self.metadata.get('no_collector', False):
+ self.no_collector = True
+
+ @property
+ def tasks(self):
+ return self._tasks
+
+ def write(self, fh):
+ """
+ Writes a properly formatted deck to the supplied file handle.
+ :param fh: an open file handle
+ :return:
+ """
+ deck_data = {
+ "name": self.name,
+ "description": self.description,
+ "tasks": [task.data for task in self._tasks]
+ }
+ if self.schedule is not None:
+ deck_data["schedule"] = self.schedule
+ for key, value in self.metadata.items():
+ deck_data[key] = value
+
+ fh.write("---\n")
+ yaml.safe_dump(deck_data, fh, default_flow_style=False)
+
@defer.inlineCallbacks
def query_bouncer(self):
preferred_backend = config.advanced.get(
@@ -772,60 +864,74 @@ class NGDeck(object):
preferred_backend,
self.no_collector
)
-
- def _measurement_completed(self, result, measurement_id):
- log.msg("{0}".format(result))
- measurement_dir = self._measurement_path.child(measurement_id)
- measurement_dir.child("measurements.njson.progress").moveTo(
- measurement_dir.child("measurements.njson")
- )
- generate_summary(
- measurement_dir.child("measurements.njson").path,
- measurement_dir.child("summary.json").path
- )
- measurement_dir.child("running.pid").remove()
-
- def _measurement_failed(self, failure, measurement_id):
- measurement_dir = self._measurement_path.child(measurement_id)
- measurement_dir.child("running.pid").remove()
- # XXX do we also want to delete measurements.njson.progress?
+ defer.returnValue(net_test_loaders)
+
+ def _measurement_completed(self, result, task):
+ if not task.output_path:
+ measurement_id = task.id
+ measurement_dir = self._measurement_path.child(measurement_id)
+ measurement_dir.child("measurements.njson.progress").moveTo(
+ measurement_dir.child("measurements.njson")
+ )
+ generate_summary(
+ measurement_dir.child("measurements.njson").path,
+ measurement_dir.child("summary.json").path
+ )
+ measurement_dir.child("running.pid").remove()
+
+ def _measurement_failed(self, failure, task):
+ if not task.output_path:
+ # XXX do we also want to delete measurements.njson.progress?
+ measurement_id = task.id
+ measurement_dir = self._measurement_path.child(measurement_id)
+ measurement_dir.child("running.pid").remove()
return failure
def _run_ooni_task(self, task, director):
net_test_loader = task.ooni["net_test_loader"]
test_details = task.ooni["test_details"]
- measurement_id = task.id
- measurement_dir = self._measurement_path.child(measurement_id)
- measurement_dir.createDirectory()
+ report_filename = task.output_path
+ if not task.output_path:
+ measurement_id = task.id
- report_filename = measurement_dir.child("measurements.njson.progress").path
- pid_file = measurement_dir.child("running.pid")
+ measurement_dir = self._measurement_path.child(measurement_id)
+ measurement_dir.createDirectory()
- with pid_file.open('w') as out_file:
- out_file.write("{0}".format(os.getpid()))
+ report_filename = measurement_dir.child("measurements.njson.progress").path
+ pid_file = measurement_dir.child("running.pid")
+
+ with pid_file.open('w') as out_file:
+ out_file.write("{0}".format(os.getpid()))
d = director.start_net_test_loader(
net_test_loader,
report_filename,
+ collector_client=net_test_loader.collector,
test_details=test_details
)
- d.addCallback(self._measurement_completed, measurement_id)
- d.addErrback(self._measurement_failed, measurement_id)
+ d.addCallback(self._measurement_completed, task)
+ d.addErrback(self._measurement_failed, task)
return d
+ def setup(self):
+ """
+ This method needs to be called before you are able to run a deck.
+ """
+ for task in self._tasks:
+ task.setup()
+ self._is_setup = True
+
@defer.inlineCallbacks
def run(self, director):
- tasks = []
- preferred_backend = config.advanced.get("preferred_backend", "onion")
+ assert self._is_setup, "You must call setup() before you can run a " \
+ "deck"
+ if self.requires_tor:
+ yield director.start_tor()
yield self.query_bouncer()
for task in self._tasks:
- if task.requires_tor:
- yield director.start_tor()
- elif task.requires_bouncer and preferred_backend == "onion":
- yield director.start_tor()
if task.type == "ooni":
- tasks.append(self._run_ooni_task(task, director))
- defer.returnValue(tasks)
+ yield self._run_ooni_task(task, director)
+ self._is_setup = False
input_store = InputStore()
diff --git a/ooni/director.py b/ooni/director.py
index 793975e..c239601 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -424,9 +424,7 @@ class Director(object):
log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
try:
yield start_tor(tor_config)
- log.err("Calling tor callback")
self._tor_starting.callback(self._tor_state)
- log.err("called")
except Exception as exc:
log.err("Failed to start tor")
log.exc(exc)
diff --git a/ooni/geoip.py b/ooni/geoip.py
index 28e0e1e..f118268 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -1,6 +1,7 @@
from __future__ import absolute_import
import re
import os
+import json
import random
from hashlib import sha256
@@ -137,11 +138,12 @@ class UbuntuGeoIP(HTTPGeoIPLookupper):
return probe_ip
class DuckDuckGoGeoIP(HTTPGeoIPLookupper):
- url = "https://duckduckgo.com/?q=ip&ia=answer"
+ url = "https://api.duckduckgo.com/?q=ip&format=json"
def parseResponse(self, response_body):
+ j = json.loads(response_body)
regexp = "Your IP address is (.*) in "
- probe_ip = re.search(regexp, response_body).group(1)
+ probe_ip = re.search(regexp, j['Answer']).group(1)
return probe_ip
class ProbeIP(object):
diff --git a/ooni/reporter.py b/ooni/reporter.py
index f07b3cf..e2f155f 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -534,7 +534,7 @@ class Report(object):
collector:
The address of the oonib collector for this report.
- no_yamloo:
+ no_njson:
If we should disable reporting to disk.
"""
self.test_details = test_details
diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
index fa675f9..b980a2c 100644
--- a/ooni/scripts/oonideckgen.py
+++ b/ooni/scripts/oonideckgen.py
@@ -92,7 +92,7 @@ def get_user_country_code():
@defer.inlineCallbacks
-def oonideckgen():
+def oonideckgen(reactor):
options = Options()
try:
options.parseOptions()
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
new file mode 100644
index 0000000..24493da
--- /dev/null
+++ b/ooni/scripts/ooniprobe.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+from twisted.internet import task
+
+def ooniprobe(reactor):
+ from ooni.ui.cli import runWithDaemonDirector, runWithDirector
+ from ooni.ui.cli import setupGlobalOptions
+
+ global_options = setupGlobalOptions(logging=True, start_tor=True,
+ check_incoherences=True)
+ if global_options['queue']:
+ return runWithDaemonDirector(global_options)
+ else:
+ return runWithDirector(global_options)
+
+def run():
+ task.react(ooniprobe)
+
+if __name__ == "__main__":
+ run()
diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py
index 6facadf..d75c925 100644
--- a/ooni/scripts/oonireport.py
+++ b/ooni/scripts/oonireport.py
@@ -202,7 +202,7 @@ def tor_check():
sys.exit(1)
-def oonireport(args=sys.argv[1:]):
+def oonireport(reactor, args=sys.argv[1:]):
options = Options()
try:
options.parseOptions(args)
diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py
index 5e1d21a..bb080e6 100644
--- a/ooni/templates/tcpt.py
+++ b/ooni/templates/tcpt.py
@@ -96,4 +96,3 @@ class TCPTest(NetTestCase):
d2.addCallback(connected)
d2.addErrback(errback)
return d1
-
diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py
index c15beec..d623e0b 100644
--- a/ooni/tests/mocks.py
+++ b/ooni/tests/mocks.py
@@ -191,7 +191,7 @@ class MockTaskManager(TaskManager):
class MockBouncerClient(object):
def __init__(self, *args, **kw):
- pass
+ self.backend_type = "onion"
def lookupTestHelpers(self, required_test_helpers):
ret = {
diff --git a/ooni/tests/test_backend_client.py b/ooni/tests/test_backend_client.py
index 212abb6..fab1fa2 100644
--- a/ooni/tests/test_backend_client.py
+++ b/ooni/tests/test_backend_client.py
@@ -60,22 +60,6 @@ class TestEnd2EndBackendClient(ConfigTestCase):
def test_download_input(self):
yield self.collector_client.downloadInput(input_id)
- @defer.inlineCallbacks
- def test_get_deck_list(self):
- deck_list = yield self.collector_client.getDeckList()
- self.assertTrue(isinstance(deck_list, list))
-
- @defer.inlineCallbacks
- def test_get_deck_descriptor(self):
- deck_descriptor = yield self.collector_client.getDeck(deck_id)
- for key in ['name', 'description',
- 'version', 'author', 'date', 'id']:
- self.assertTrue(hasattr(deck_descriptor, key))
-
- @defer.inlineCallbacks
- def test_download_deck(self):
- yield self.collector_client.downloadDeck(deck_id)
-
def test_lookup_invalid_helpers(self):
bouncer_client = BouncerClient('http://127.0.0.1:8888')
return self.failUnlessFailure(
diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py
index 455f4e1..3d07959 100644
--- a/ooni/tests/test_deck.py
+++ b/ooni/tests/test_deck.py
@@ -1,15 +1,20 @@
import os
+from StringIO import StringIO
from copy import deepcopy
+import yaml
+
+from mock import patch, MagicMock
+
from twisted.internet import defer
from twisted.trial import unittest
from hashlib import sha256
from ooni import errors
from ooni.deck import input_store, lookup_collector_and_test_helpers
-from ooni.nettest import NetTestLoader
-from ooni.deck import InputFile, Deck, nettest_to_path, DeckTask, NGDeck
+from ooni.deck import nettest_to_path, NGDeck
+from ooni.deck import convert_legacy_deck
from ooni.tests.bases import ConfigTestCase
from ooni.tests.mocks import MockBouncerClient, MockCollectorClient
@@ -80,45 +85,6 @@ class BaseTestCase(unittest.TestCase):
"""
super(BaseTestCase, self).setUp()
-
-class TestInputFile(BaseTestCase):
- def tearDown(self):
- if self.filename != "":
- os.remove(self.filename)
-
- def test_file_cached(self):
- self.filename = file_hash = sha256(self.dummy_deck_content).hexdigest()
- input_file = InputFile(file_hash, base_path='.')
- with open(file_hash, 'w+') as f:
- f.write(self.dummy_deck_content)
- assert input_file.fileCached
-
- def test_file_invalid_hash(self):
- self.filename = invalid_hash = 'a' * 64
- with open(invalid_hash, 'w+') as f:
- f.write("b" * 100)
- input_file = InputFile(invalid_hash, base_path='.')
- self.assertRaises(AssertionError, input_file.verify)
-
- def test_save_descriptor(self):
- descriptor = {
- 'name': 'spam',
- 'id': 'spam',
- 'version': 'spam',
- 'author': 'spam',
- 'date': 'spam',
- 'description': 'spam'
- }
- file_id = 'a' * 64
- self.filename = file_id + '.desc'
- input_file = InputFile(file_id, base_path='.')
- input_file.load(descriptor)
- input_file.save()
- assert os.path.isfile(self.filename)
-
- assert input_file.descriptorCached
-
-
class TestDeck(BaseTestCase, ConfigTestCase):
def setUp(self):
super(TestDeck, self).setUp()
@@ -137,60 +103,47 @@ class TestDeck(BaseTestCase, ConfigTestCase):
super(TestDeck, self).tearDown()
def test_open_deck(self):
- deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
- decks_directory=".")
- deck.loadDeck(self.deck_file)
- assert len(deck.netTestLoaders) == 1
+ deck = NGDeck()
+ deck.open(self.deck_file)
+ assert len(deck.tasks.ooni['net_test_loaders']) == 1
def test_load_deck_with_global_options(self):
global_options = {
"annotations": {"spam": "ham"},
"collector": "httpo://thirteenchars123.onion"
}
- deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
- decks_directory=".")
- deck.loadDeck(self.deck_file,
- global_options=global_options)
+ deck = NGDeck(global_options=global_options)
+ deck.open(self.deck_file)
self.assertEqual(
- deck.netTestLoaders[0].annotations,
+ deck.tasks.ooni['net_test_loaders'][0].annotations,
global_options['annotations']
)
self.assertEqual(
- deck.netTestLoaders[0].collector.base_address,
+ deck.tasks.ooni['net_test_loaders'][0].collector.base_address,
global_options['collector'].replace("httpo://", "http://")
)
- def test_save_deck_descriptor(self):
- deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
- decks_directory=".")
- deck.loadDeck(self.deck_file)
- deck.load({'name': 'spam',
- 'id': 'spam',
- 'version': 'spam',
- 'author': 'spam',
- 'date': 'spam',
- 'description': 'spam'
- })
- deck.save()
- self.filename = self.deck_file + ".desc"
- deck.verify()
-
+ @patch('ooni.deck.BouncerClient', MockBouncerClient)
+ @patch('ooni.deck.CollectorClient', MockCollectorClient)
@defer.inlineCallbacks
def test_lookup_test_helpers_and_collector(self):
- deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
- decks_directory=".")
+ deck = NGDeck()
deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
- deck._BouncerClient = MockBouncerClient
- deck._CollectorClient = MockCollectorClient
- deck.loadDeck(self.deck_file)
+ deck.open(self.deck_file)
- self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
+ self.assertEqual(
+ len(deck.tasks.ooni['net_test_loaders'][0].missingTestHelpers), 1)
- yield lookup_collector_and_test_helpers(deck.preferred_backend,
- deck.netTestLoaders)
+ yield lookup_collector_and_test_helpers(
+ net_test_loaders=deck.netTestLoaders,
+ preferred_backend=deck.preferred_backend,
+ bouncer=deck.bouncer
+ )
- self.assertEqual(deck.netTestLoaders[0].collector.settings['address'],
- 'httpo://thirteenchars123.onion')
+ self.assertEqual(
+ deck.tasks.ooni['net_test_loaders'][0].collector.settings['address'],
+ 'httpo://thirteenchars123.onion'
+ )
self.assertEqual(deck.netTestLoaders[0].localOptions['backend'],
'127.0.0.1')
@@ -202,15 +155,15 @@ class TestDeck(BaseTestCase, ConfigTestCase):
self.deck_file = os.path.join(self.cwd, deck_hash)
with open(self.deck_file, 'w+') as f:
f.write(self.dummy_deck_content_with_many_tests)
- deck = Deck(decks_directory=".")
- deck.loadDeck(self.deck_file)
+ deck = NGDeck()
+ deck.open(self.deck_file)
self.assertEqual(
- deck.netTestLoaders[0].localOptions['backend'],
+ deck.tasks[0].ooni['net_test_loader'].localOptions['backend'],
'1.1.1.1'
)
self.assertEqual(
- deck.netTestLoaders[1].localOptions['backend'],
+ deck.tasks[1].ooni['net_test_loader'].localOptions['backend'],
'2.2.2.2'
)
@@ -222,58 +175,65 @@ class TestDeck(BaseTestCase, ConfigTestCase):
nettest_to_path,
"invalid_test")
+ @patch('ooni.deck.BouncerClient', MockBouncerClient)
+ @patch('ooni.deck.CollectorClient', MockCollectorClient)
@defer.inlineCallbacks
def test_lookup_test_helpers_and_collector_cloudfront(self):
self.config.advanced.preferred_backend = "cloudfront"
- deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
- decks_directory=".")
- deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
- deck._BouncerClient = MockBouncerClient
- deck._CollectorClient = MockCollectorClient
- deck.loadDeck(self.deck_file)
-
- self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
-
- yield lookup_collector_and_test_helpers(deck.preferred_backend,
- deck.netTestLoaders)
+ deck = NGDeck()
+ deck.open(self.deck_file)
+ first_net_test_loader = deck.tasks[0].ooni['net_test_loader']
+ net_test_loaders = map(lambda task: task.ooni['net_test_loader'],
+ deck.tasks)
+ self.assertEqual(len(first_net_test_loader.missingTestHelpers), 1)
+
+ yield lookup_collector_and_test_helpers(
+ net_test_loaders=net_test_loaders ,
+ preferred_backend='cloudfront',
+ bouncer=deck.bouncer
+ )
self.assertEqual(
- deck.netTestLoaders[0].collector.settings['address'],
+ first_net_test_loader.collector.settings['address'],
'https://address.cloudfront.net'
)
self.assertEqual(
- deck.netTestLoaders[0].collector.settings['front'],
+ first_net_test_loader.collector.settings['front'],
'front.cloudfront.net'
)
self.assertEqual(
- deck.netTestLoaders[0].localOptions['backend'],
+ first_net_test_loader.localOptions['backend'],
'127.0.0.1'
)
-
+ @patch('ooni.deck.BouncerClient', MockBouncerClient)
+ @patch('ooni.deck.CollectorClient', MockCollectorClient)
@defer.inlineCallbacks
def test_lookup_test_helpers_and_collector_https(self):
self.config.advanced.preferred_backend = "https"
- deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
- decks_directory=".")
- deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
- deck._BouncerClient = MockBouncerClient
- deck._CollectorClient = MockCollectorClient
- deck.loadDeck(self.deck_file)
+ deck = NGDeck()
+ deck.open(self.deck_file)
+
+ first_net_test_loader = deck.tasks[0].ooni['net_test_loader']
+ net_test_loaders = map(lambda task: task.ooni['net_test_loader'],
+ deck.tasks)
- self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
+ self.assertEqual(len(first_net_test_loader .missingTestHelpers), 1)
- yield lookup_collector_and_test_helpers(deck.preferred_backend,
- deck.netTestLoaders)
+ yield lookup_collector_and_test_helpers(
+ net_test_loaders=net_test_loaders,
+ preferred_backend='https',
+ bouncer=deck.bouncer
+ )
self.assertEqual(
- deck.netTestLoaders[0].collector.settings['address'],
+ first_net_test_loader.collector.settings['address'],
'https://collector.ooni.io'
)
self.assertEqual(
- deck.netTestLoaders[0].localOptions['backend'],
+ first_net_test_loader.localOptions['backend'],
'127.0.0.1'
)
@@ -301,20 +261,65 @@ DECK_DATA = {
deepcopy(TASK_DATA)
]
}
+
+LEGACY_DECK = """
+- options:
+ annotations: null
+ bouncer: null
+ collector: null
+ no-collector: 0
+ no-geoip: 0
+ no-yamloo: 0
+ reportfile: null
+ subargs: [--flag, --key, value]
+ test_file: manipulation/http_invalid_request_line
+ verbose: 0
+- options:
+ annotations: null
+ bouncer: null
+ collector: null
+ no-collector: 0
+ no-geoip: 0
+ no-yamloo: 0
+ reportfile: null
+ subargs: []
+ test_file: manipulation/http_header_field_manipulation
+ verbose: 0
+- options:
+ annotations: null
+ bouncer: null
+ collector: null
+ no-collector: 0
+ no-geoip: 0
+ no-yamloo: 0
+ reportfile: null
+ subargs: [-f, /path/to/citizenlab-urls-global.txt]
+ test_file: blocking/web_connectivity
+ verbose: 0
+"""
+
class TestNGDeck(ConfigTestCase):
- skip = True
def test_deck_task(self):
- if self.skip:
- self.skipTest("Skip is set to true")
- yield input_store.update("ZZ")
- deck_task = DeckTask(TASK_DATA)
- self.assertIsInstance(deck_task.ooni["net_test_loader"],
- NetTestLoader)
+ #yield input_store.update("ZZ")
+ #deck_task = DeckTask(TASK_DATA)
+ #self.assertIsInstance(deck_task.ooni["net_test_loader"],
+ # NetTestLoader)
+ pass
- @defer.inlineCallbacks
def test_deck_load(self):
- if self.skip:
- self.skipTest("Skip is set to true")
- yield input_store.update("ZZ")
- deck = NGDeck(deck_data=DECK_DATA)
- self.assertEqual(len(deck.tasks), 1)
+ #yield input_store.update("ZZ")
+ #deck = NGDeck(deck_data=DECK_DATA)
+ #self.assertEqual(len(deck.tasks), 1)
+ pass
+
+ def test_convert_legacy_deck(self):
+ legacy_deck = yaml.safe_load(StringIO(LEGACY_DECK))
+ ng_deck = convert_legacy_deck(legacy_deck)
+ self.assertEqual(len(ng_deck['tasks']), 3)
+ task_names = map(lambda task: task['ooni']['test_name'],
+ ng_deck['tasks'])
+ self.assertItemsEqual(task_names, [
+ "manipulation/http_invalid_request_line",
+ "manipulation/http_header_field_manipulation",
+ "blocking/web_connectivity"
+ ])
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index fe24bf6..6550b3d 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -26,19 +26,18 @@ class Options(usage.Options):
optFlags = [["help", "h"],
["no-collector", "n", "Disable writing to collector"],
- ["no-yamloo", "N", "Disable writing to YAML file"],
+ ["no-njson", "N", "Disable writing to disk"],
["no-geoip", "g", "Disable geoip lookup on start"],
["list", "s", "List the currently installed ooniprobe "
"nettests"],
- ["printdeck", "p", "Print the equivalent deck for the "
- "provided command"],
["verbose", "v", "Show more verbose information"]
]
optParameters = [
- ["reportfile", "o", None, "Specify the report file name to write to."],
+ ["reportfile", "o", None, "Specify the report file name to write "
+ "to."],
["testdeck", "i", None, "Specify as input a test deck: a yaml file "
- "containing the tests to run and their "
+ "containing the tests to run and their "
"arguments."],
["collector", "c", None, "Specify the address of the collector for "
"test results. In most cases a user will "
@@ -132,7 +131,8 @@ def director_startup_handled_failures(failure):
errors.CouldNotFindTestCollector,
errors.ProbeIPUnknown,
errors.InvalidInputFile,
- errors.ConfigFileIncoherent)
+ errors.ConfigFileIncoherent,
+ SystemExit)
if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
@@ -236,64 +236,71 @@ def setupCollector(global_options, collector_client):
return collector_client
def createDeck(global_options, url=None):
- from ooni.nettest import NetTestLoader
- from ooni.deck import Deck, nettest_to_path
- from ooni.backend_client import CollectorClient
+ from ooni.deck import NGDeck, subargs_to_options
if url:
log.msg("Creating deck for: %s" % (url))
- if global_options['no-yamloo']:
- log.msg("Will not write to a yamloo report file")
-
- deck = Deck(bouncer=global_options['bouncer'],
- no_collector=global_options['no-collector'])
-
+ test_deck_path = global_options.pop('testdeck', None)
+ test_name = global_options.pop('test_file', None)
+ no_collector = global_options.pop('no-collector', False)
try:
- if global_options['testdeck']:
- deck.loadDeck(global_options['testdeck'], global_options)
+ if test_deck_path is not None:
+ deck = NGDeck(
+ global_options=global_options,
+ no_collector=no_collector
+ )
+ deck.open(test_deck_path)
else:
+ deck = NGDeck(
+ global_options=global_options,
+ no_collector=no_collector,
+ arbitrary_paths=True
+ )
log.debug("No test deck detected")
- test_file = nettest_to_path(global_options['test_file'], True)
if url is not None:
args = ('-u', url)
else:
args = tuple()
if any(global_options['subargs']):
args = global_options['subargs'] + args
- net_test_loader = NetTestLoader(args,
- test_file=test_file,
- annotations=global_options['annotations'])
- if global_options['collector']:
- net_test_loader.collector = \
- CollectorClient(global_options['collector'])
- deck.insert(net_test_loader)
+
+ test_options = subargs_to_options(args)
+ test_options['test_name'] = test_name
+ deck.load({
+ "tasks": [
+ {"ooni": test_options}
+ ]
+ })
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
- sys.exit(2)
+ raise SystemExit(2)
+
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
- sys.exit(3)
+ raise SystemExit(3)
+
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
- sys.exit(4)
+ raise SystemExit(4)
+
except errors.HTTPSCollectorUnsupported:
log.err("HTTPS collectors require a twisted version of at least 14.0.2.")
- sys.exit(6)
+ raise SystemExit(6)
except errors.InsecureBackend:
log.err("Attempting to report to an insecure collector.")
log.err("To enable reporting to insecure collector set the "
"advanced->insecure_backend option to true in "
"your ooniprobe.conf file.")
- sys.exit(7)
+ raise SystemExit(7)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
- sys.exit(5)
+ raise SystemExit(5)
return deck
@@ -301,45 +308,21 @@ def createDeck(global_options, url=None):
def runTestWithDirector(director, global_options, url=None, start_tor=True):
deck = createDeck(global_options, url=url)
- start_tor |= deck.requiresTor
-
- d = director.start(start_tor=start_tor,
- check_incoherences=global_options['check_incoherences'])
-
- def setup_nettest(_):
+ d = director.start()
+ @defer.inlineCallbacks
+ def post_director_start(_):
try:
- return deck.setup()
+ deck.setup()
+ yield deck.run(director)
except errors.UnableToLoadDeckInput as error:
- return defer.failure.Failure(error)
+ raise defer.failure.Failure(error)
except errors.NoReachableTestHelpers as error:
- return defer.failure.Failure(error)
+ raise defer.failure.Failure(error)
except errors.NoReachableCollectors as error:
- return defer.failure.Failure(error)
+ raise defer.failure.Failure(error)
+ except SystemExit as error:
+ raise error
- # Wait until director has started up (including bootstrapping Tor)
- # before adding tests
- @defer.inlineCallbacks
- def post_director_start(_):
- for net_test_loader in deck.netTestLoaders:
- # Decks can specify different collectors
- # for each net test, so that each NetTest
- # may be paired with a test_helper and its collector
- # However, a user can override this behavior by
- # specifying a collector from the command-line (-c).
- # If a collector is not specified in the deck, or the
- # deck is a singleton, the default collector set in
- # ooniprobe.conf will be used
- collector_client = None
- if not global_options['no-collector']:
- collector_client = setupCollector(global_options,
- net_test_loader.collector)
-
- yield director.start_net_test_loader(net_test_loader,
- global_options['reportfile'],
- collector_client,
- global_options['no-yamloo'])
-
- d.addCallback(setup_nettest)
d.addCallback(post_director_start)
d.addErrback(director_startup_handled_failures)
d.addErrback(director_startup_other_failures)
@@ -379,14 +362,7 @@ def runWithDirector(global_options):
print "Note: Third party tests require an external "\
"application to run properly."
- sys.exit(0)
-
- elif global_options['printdeck']:
- del global_options['printdeck']
- print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
- print yaml.safe_dump([{'options': global_options}]).strip()
-
- sys.exit(0)
+ raise SystemExit(0)
if global_options.get('annotations') is not None:
global_options['annotations'] = setupAnnotations(global_options)
@@ -427,7 +403,7 @@ def runWithDaemonDirector(global_options):
except ImportError:
print "Pika is required for queue connection."
print "Install with \"pip install pika\"."
- sys.exit(7)
+ raise SystemExit(7)
director = Director()
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index e306ef2..e90ef83 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
<app>
Loading...
</app>
- <script type="text/javascript" src="app.bundle.js?7ed7d7510803fa1a4ad8"></script></body>
+ <script type="text/javascript" src="app.bundle.js?6b15c1dd202a0f5a80e7"></script></body>
</html>
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/set_caps/Makefile b/scripts/set_caps/Makefile
new file mode 100644
index 0000000..bc80843
--- /dev/null
+++ b/scripts/set_caps/Makefile
@@ -0,0 +1,81 @@
+# Wrappers for running ooniprobe as a non-root user.
+#
+# Build-Depends: cython, pythonX.Y-dev, libcap2-bin
+# Depends: libpythonX.Y
+#
+# $ make && make check
+# $ sudo make install # after installing the rest of ooni-probe
+# $ make installcheck_unsafe # runs complete tests as non-root
+#
+# `make` builds a program that has file capabilities set on it. This is just
+# ./ooniprobe compiled into a C program using Cython, so that one can set
+# capabilities directly on the resulting binary. This way, we avoid the need
+# for a separate child python interpreter with its own capabilities. Another
+# advantage is that libpython.so (needed by the program) would be automatically
+# upgraded by the system package manager. The version of python is hard-coded
+# into the wrapper at build time; making this dynamic is possible, but much
+# more complex and not yet implemented.
+#
+# Execution may additionally be limited to a particular unix group by using
+# chgrp(1) and chmod(1) to 'o-x,g+x' after installation.
+#
+
+# GNU Makefile conventions, see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
+prefix = /usr/local
+exec_prefix = $(prefix)
+bindir = $(exec_prefix)/bin
+
+INSTALL = install
+PYTHON = python
+PYTHON_CONFIG = python-config
+CYTHON = cython
+SETCAP = setcap
+
+INSTALL_PROGRAM = $(INSTALL)
+PY_CFLAGS = $(shell $(PYTHON_CONFIG) --cflags)
+PY_LDFLAGS = $(shell $(PYTHON_CONFIG) --ldflags)
+
+BUILDDIR := ./build
+SCRIPTDIR := .
+TESTDIR := ./test
+CAP_SCRIPT := ooniprobe
+CAP_NEEDED := cap_net_admin,cap_net_raw
+
+# Unfortunately cython --embed ignores the arguments in the shebang line
+# So we need to patch the generated code ourselves.
+CYTHON_PRE_MAIN = extern int Py_IgnoreEnvironmentFlag; \
+ Py_IgnoreEnvironmentFlag++; \
+ extern int Py_NoUserSiteDirectory; \
+ Py_NoUserSiteDirectory++;
+
+all: $(BUILDDIR)/$(CAP_SCRIPT)
+
+$(BUILDDIR)/$(CAP_SCRIPT): $(BUILDDIR)/$(CAP_SCRIPT).c Makefile
+ $(CC) $(PY_CFLAGS) $(PY_LDFLAGS) "$<" -o "$@"
+
+$(BUILDDIR)/$(CAP_SCRIPT).c: $(SCRIPTDIR)/$(CAP_SCRIPT) Makefile
+ mkdir -p "$(BUILDDIR)"
+ $(CYTHON) "$<" --embed=CYTHON_MAIN_SENTINEL -Werror -Wextra -o "$@"
+ sed -i \
+ -e 's/\(.*CYTHON_MAIN_SENTINEL.*{\)/\1 $(CYTHON_PRE_MAIN)/g' \
+ -e '/CYTHON_MAIN_SENTINEL[^{]*$$/,/{/s/{/{ $(CYTHON_PRE_MAIN)/g' \
+ -e 's/CYTHON_MAIN_SENTINEL/main/g' "$@"
+
+check: $(BUILDDIR)/$(CAP_SCRIPT)
+ # test that setcapped binary ignores PYTHONPATH
+ BIN="$$(realpath "$<")" && cd "$(TESTDIR)" && PYTHONPATH=. $$BIN --version
+
+install: $(BUILDDIR)/$(CAP_SCRIPT)
+ mkdir -p "$(DESTDIR)$(bindir)"
+ $(INSTALL_PROGRAM) -t "$(DESTDIR)$(bindir)" "$(BUILDDIR)/$(CAP_SCRIPT)"
+ $(SETCAP) "$(CAP_NEEDED)"+eip "$(DESTDIR)$(bindir)/$(CAP_SCRIPT)"
+
+installcheck_unsafe: $(BUILDDIR)/$(CAP_SCRIPT)
+ # run a standard check. note that because of hardcoded paths (for security)
+ # this can only work after you've installed your development copy
+ "./$<" -i /usr/share/ooni/decks/complete.deck
+
+clean:
+ rm -rf "$(BUILDDIR)"
+
+.PHONY: clean all check install installcheck%
diff --git a/scripts/set_caps/__init__.py b/scripts/set_caps/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/set_caps/test/__init__.py b/scripts/set_caps/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/set_caps/test/ooni/__init__.py b/scripts/set_caps/test/ooni/__init__.py
new file mode 100644
index 0000000..f2cd353
--- /dev/null
+++ b/scripts/set_caps/test/ooni/__init__.py
@@ -0,0 +1 @@
+raise ValueError("test failed! wrapper did not ignore polluted PWD. either the wrapper is faulty, or ooni is still unpatched (Tor bug #13581)")
More information about the tor-commits
mailing list