[tor-commits] [ooni-probe/master] Use NGDeck also for the command line version of ooniprobe

commit 76cc0d4989de0a777fa62f4de5b60f07e0701da7 Author: Arturo Filastò arturo@filasto.net Date: Tue Jul 26 02:05:58 2016 +0200

Use NGDeck also for the command line version of ooniprobe

* Remove code that is now dead. --- bin/Makefile | 81 -------- bin/ooniprobe | 24 --- bin/ooniprobe-dev | 10 - bin/test/ooni/__init__.py | 1 - ooni/backend_client.py | 95 +-------- ooni/deck.py | 356 +++++++++++++++++++++------------ ooni/director.py | 2 - ooni/geoip.py | 6 +- ooni/reporter.py | 2 +- ooni/scripts/oonideckgen.py | 2 +- ooni/scripts/ooniprobe.py | 19 ++ ooni/scripts/oonireport.py | 2 +- ooni/templates/tcpt.py | 1 - ooni/tests/mocks.py | 2 +- ooni/tests/test_backend_client.py | 16 -- ooni/tests/test_deck.py | 243 +++++++++++----------- ooni/ui/cli.py | 124 +++++------- ooni/ui/web/client/index.html | 2 +- scripts/__init__.py | 0 scripts/set_caps/Makefile | 81 ++++++++ scripts/set_caps/__init__.py | 0 scripts/set_caps/test/__init__.py | 0 scripts/set_caps/test/ooni/__init__.py | 1 + 23 files changed, 517 insertions(+), 553 deletions(-)

diff --git a/bin/Makefile b/bin/Makefile deleted file mode 100644 index bc80843..0000000 --- a/bin/Makefile +++ /dev/null @@ -1,81 +0,0 @@ -# Wrappers for running ooniprobe as a non-root user. -# -# Build-Depends: cython, pythonX.Y-dev, libcap2-bin -# Depends: libpythonX.Y -# -# $ make && make check -# $ sudo make install # after installing the rest of ooni-probe -# $ make installcheck_unsafe # runs complete tests as non-root -# -# `make` builds a program that has file capabilities set on it. This is just -# ./ooniprobe compiled into a C program using Cython, so that one can set -# capabilities directly on the resulting binary. This way, we avoid the need -# for a separate child python interpreter with its own capabilities. Another -# advantage is that libpython.so (needed by the program) would be automatically -# upgraded by the system package manager. The version of python is hard-coded -# into the wrapper at build time; making this dynamic is possible, but much -# more complex and not yet implemented. -# -# Execution may additionally be limited to a particular unix group by using -# chgrp(1) and chmod(1) to 'o-x,g+x' after installation. -# - -# GNU Makefile conventions, see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html -prefix = /usr/local -exec_prefix = $(prefix) -bindir = $(exec_prefix)/bin - -INSTALL = install -PYTHON = python -PYTHON_CONFIG = python-config -CYTHON = cython -SETCAP = setcap - -INSTALL_PROGRAM = $(INSTALL) -PY_CFLAGS = $(shell $(PYTHON_CONFIG) --cflags) -PY_LDFLAGS = $(shell $(PYTHON_CONFIG) --ldflags) - -BUILDDIR := ./build -SCRIPTDIR := . -TESTDIR := ./test -CAP_SCRIPT := ooniprobe -CAP_NEEDED := cap_net_admin,cap_net_raw - -# Unfortunately cython --embed ignores the arguments in the shebang line -# So we need to patch the generated code ourselves. -CYTHON_PRE_MAIN = extern int Py_IgnoreEnvironmentFlag; \ - Py_IgnoreEnvironmentFlag++; \ - extern int Py_NoUserSiteDirectory; \ - Py_NoUserSiteDirectory++; - -all: $(BUILDDIR)/$(CAP_SCRIPT) - -$(BUILDDIR)/$(CAP_SCRIPT): $(BUILDDIR)/$(CAP_SCRIPT).c Makefile - $(CC) $(PY_CFLAGS) $(PY_LDFLAGS) "$<" -o "$@" - -$(BUILDDIR)/$(CAP_SCRIPT).c: $(SCRIPTDIR)/$(CAP_SCRIPT) Makefile - mkdir -p "$(BUILDDIR)" - $(CYTHON) "$<" --embed=CYTHON_MAIN_SENTINEL -Werror -Wextra -o "$@" - sed -i \ - -e 's/(.*CYTHON_MAIN_SENTINEL.*{)/\1 $(CYTHON_PRE_MAIN)/g' \ - -e '/CYTHON_MAIN_SENTINEL[^{]*$$/,/{/s/{/{ $(CYTHON_PRE_MAIN)/g' \ - -e 's/CYTHON_MAIN_SENTINEL/main/g' "$@" - -check: $(BUILDDIR)/$(CAP_SCRIPT) - # test that setcapped binary ignores PYTHONPATH - BIN="$$(realpath "$<")" && cd "$(TESTDIR)" && PYTHONPATH=. $$BIN --version - -install: $(BUILDDIR)/$(CAP_SCRIPT) - mkdir -p "$(DESTDIR)$(bindir)" - $(INSTALL_PROGRAM) -t "$(DESTDIR)$(bindir)" "$(BUILDDIR)/$(CAP_SCRIPT)" - $(SETCAP) "$(CAP_NEEDED)"+eip "$(DESTDIR)$(bindir)/$(CAP_SCRIPT)" - -installcheck_unsafe: $(BUILDDIR)/$(CAP_SCRIPT) - # run a standard check. note that because of hardcoded paths (for security) - # this can only work after you've installed your development copy - "./$<" -i /usr/share/ooni/decks/complete.deck - -clean: - rm -rf "$(BUILDDIR)" - -.PHONY: clean all check install installcheck% diff --git a/bin/ooniprobe b/bin/ooniprobe deleted file mode 100755 index 0274899..0000000 --- a/bin/ooniprobe +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -import sys - -from twisted.internet import reactor - -from ooni.ui.cli import runWithDaemonDirector, runWithDirector -from ooni.ui.cli import setupGlobalOptions - -exit_code=0 - -global_options = setupGlobalOptions(logging=True, start_tor=True, - check_incoherences=True) -if global_options['queue']: - d = runWithDaemonDirector(global_options) -else: - d = runWithDirector(global_options) -@d.addBoth -def cb(result): - global exit_code - if result is not None: - exit_code=1 - reactor.stop() -reactor.run() -sys.exit(exit_code) diff --git a/bin/ooniprobe-dev b/bin/ooniprobe-dev deleted file mode 100755 index c44a8a6..0000000 --- a/bin/ooniprobe-dev +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -# Developer script for running ooniprobe directly from the repository. -# We don't automatically add "$PWD" to PYTHONPATH as that is a security risk -# when run as /usr/bin/ooniprobe on an end user's computer. -ROOTDIR=$(cd $(dirname $(dirname $0)) && pwd -P) -if [ $? -ne 0 ]; then - echo "$0: cannot determine toplevel directory" 1>&2 - exit 1 -fi -PYTHONPATH="$ROOTDIR" exec $ROOTDIR/bin/ooniprobe "$@" diff --git a/bin/test/ooni/__init__.py b/bin/test/ooni/__init__.py deleted file mode 100644 index f2cd353..0000000 --- a/bin/test/ooni/__init__.py +++ /dev/null @@ -1 +0,0 @@ -raise ValueError("test failed! wrapper did not ignore polluted PWD. either the wrapper is faulty, or ooni is still unpatched (Tor bug #13581)") diff --git a/ooni/backend_client.py b/ooni/backend_client.py index 506aba4..7721f6f 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -134,6 +134,8 @@ class OONIBClient(object): return finished

def queryBackend(self, method, urn, query=None, retries=3): + log.debug("Querying backend {0}{1} with {2}".format(self.base_address, + urn, query)) bodyProducer = None if query: bodyProducer = StringProducer(json.dumps(query)) @@ -212,105 +214,12 @@ class CollectorClient(OONIBClient):

return d

- def getInput(self, input_hash): - from ooni.deck import InputFile - - input_file = InputFile(input_hash) - if input_file.descriptorCached: - return defer.succeed(input_file) - else: - d = self.queryBackend('GET', '/input/' + input_hash) - - @d.addCallback - def cb(descriptor): - input_file.load(descriptor) - input_file.save() - return input_file - - @d.addErrback - def err(err): - log.err("Failed to get descriptor for input %s" % input_hash) - log.exception(err) - - return d - - def getInputList(self): - return self.queryBackend('GET', '/input') - - def downloadInput(self, input_hash): - from ooni.deck import InputFile - - input_file = InputFile(input_hash) - - if input_file.fileCached: - return defer.succeed(input_file) - else: - d = self.download('/input/' + input_hash + '/file', input_file.cached_file) - - @d.addCallback - def cb(res): - input_file.verify() - return input_file - - @d.addErrback - def err(err): - log.err("Failed to download the input file %s" % input_hash) - log.exception(err) - - return d - def getInputPolicy(self): return self.queryBackend('GET', '/policy/input')

def getNettestPolicy(self): return self.queryBackend('GET', '/policy/nettest')

- def getDeckList(self): - return self.queryBackend('GET', '/deck') - - def getDeck(self, deck_hash): - from ooni.deck import Deck - - deck = Deck(deck_hash) - if deck.descriptorCached: - return defer.succeed(deck) - else: - d = self.queryBackend('GET', '/deck/' + deck_hash) - - @d.addCallback - def cb(descriptor): - deck.load(descriptor) - deck.save() - return deck - - @d.addErrback - def err(err): - log.err("Failed to get descriptor for deck %s" % deck_hash) - log.exception(err) - - return d - - def downloadDeck(self, deck_hash): - from ooni.deck import Deck - - deck = Deck(deck_hash) - if deck.fileCached: - return defer.succeed(deck) - else: - d = self.download('/deck/' + deck_hash + '/file', deck.cached_file) - - @d.addCallback - def cb(res): - deck.verify() - return deck - - @d.addErrback - def err(err): - log.err("Failed to download the deck %s" % deck_hash) - log.exception(err) - - return d - def createReport(self, test_details): request = { 'software_name': test_details['software_name'], diff --git a/ooni/deck.py b/ooni/deck.py index 1746d26..eacd256 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -24,59 +24,6 @@ from ooni.utils import log

from ooni.results import generate_summary

-class InputFile(object): - def __init__(self, input_hash, base_path=config.inputs_directory): - self.id = input_hash - cache_path = os.path.join(os.path.abspath(base_path), input_hash) - self.cached_file = cache_path - self.cached_descriptor = cache_path + '.desc' - - @property - def descriptorCached(self): - if os.path.exists(self.cached_descriptor): - with open(self.cached_descriptor) as f: - descriptor = json.load(f) - self.load(descriptor) - return True - return False - - @property - def fileCached(self): - if os.path.exists(self.cached_file): - try: - self.verify() - except AssertionError: - log.err("The input %s failed validation." - "Going to consider it not cached." % self.id) - return False - return True - return False - - def save(self): - with open(self.cached_descriptor, 'w+') as f: - json.dump({ - 'name': self.name, - 'id': self.id, - 'version': self.version, - 'author': self.author, - 'date': self.date, - 'description': self.description - }, f) - - def load(self, descriptor): - self.name = descriptor['name'] - self.version = descriptor['version'] - self.author = descriptor['author'] - self.date = descriptor['date'] - self.description = descriptor['description'] - - def verify(self): - digest = os.path.basename(self.cached_file) - with open(self.cached_file) as f: - file_hash = sha256(f.read()) - assert file_hash.hexdigest() == digest - - def nettest_to_path(path, allow_arbitrary_paths=False): """ Takes as input either a path or a nettest name. @@ -136,6 +83,7 @@ def get_preferred_bouncer(): else: return BouncerClient(bouncer_address)

+<<<<<<< d0fb4f37530aeb6b69fbc2985019464f8ec10312 class Deck(InputFile): # this exists so we can mock it out in unittests _BouncerClient = BouncerClient @@ -274,6 +222,7 @@ class Deck(InputFile): i['test_options'][i['key']] = input_file.cached_file

+@defer.inlineCallbacks def lookup_collector_and_test_helpers(net_test_loaders, bouncer, preferred_backend, @@ -329,19 +278,21 @@ def lookup_collector_and_test_helpers(net_test_loaders, net_test_loader.testName)

collector, test_helpers = \ - find_collector_and_test_helpers(test_name=net_test_loader.testName, - test_version=net_test_loader.testVersion, - input_files=net_test_loader.inputFiles) + find_collector_and_test_helpers( + test_name=net_test_loader.testName, + test_version=net_test_loader.testVersion, + input_files=net_test_loader.inputFiles + )

for option, name in net_test_loader.missingTestHelpers: test_helper_address_or_settings = test_helpers[name] net_test_loader.localOptions[option] = test_helper_address_or_settings net_test_loader.testHelpers[option] = test_helper_address_or_settings

- if not net_test_loader.collector: + if not net_test_loader.collector and not no_collector: + log.debug("Using collector {0}".format(collector)) net_test_loader.collector = collector

- @defer.inlineCallbacks def get_reachable_test_helpers_and_collectors(net_tests, preferred_backend): for net_test in net_tests: @@ -579,10 +530,29 @@ def options_to_args(options, prepath=None): continue if k == "file": v = resolve_file_path(v, prepath) - args.append('--'+k) + if v == False or v == 0: + continue + if (len(k)) == 1: + args.append('-'+k) + else: + args.append('--'+k) + if isinstance(v, bool) or isinstance(v, int): + continue args.append(v) return args

+def normalize_options(options): + """ + Takes some options that have a mixture of - and _ and returns the + equivalent options with only '_'. + """ + normalized_opts = {} + for k, v in options.items(): + normalized_key = k.replace('-', '_') + assert normalized_key not in normalized_opts, "The key {0} cannot be normalized".format(k) + normalized_opts[normalized_key] = v + return normalized_opts + class UnknownTaskKey(Exception): pass

@@ -593,8 +563,14 @@ class DeckTask(object): _metadata_keys = ["name"] _supported_tasks = ["ooni"]

- def __init__(self, data, parent_metadata={}, cwd=None): - self.parent_metadata = parent_metadata + def __init__(self, data, + parent_metadata={}, + global_options={}, + cwd=None, + arbitrary_paths=False): + + self.parent_metadata = normalize_options(parent_metadata) + self.global_options = global_options self.cwd = cwd self.data = deepcopy(data)

@@ -605,10 +581,16 @@ class DeckTask(object): self.requires_tor = False self.requires_bouncer = False

+ # If this is set to true a deck can specify any path. It should only + # be run against trusted decks or when you create a deck + # programmaticaly to a run test specified from the command line. + self._arbitrary_paths = arbitrary_paths + self.ooni = { 'bouncer_client': None, 'test_details': {} } + self.output_path = None

self._load(data)

@@ -619,7 +601,8 @@ class DeckTask(object): raise MissingTaskDataKey(required_key)

# This raises e.NetTestNotFound, we let it go onto the caller - nettest_path = nettest_to_path(task_data.pop("test_name")) + nettest_path = nettest_to_path(task_data.pop("test_name"), + self._arbitrary_paths)

try: annotations = task_data.pop('annotations') @@ -631,8 +614,16 @@ class DeckTask(object): except KeyError: collector_address = self.parent_metadata.get('collector', None)

+ try: + self.output_path = task_data.pop('reportfile') + except KeyError: + self.output_path = self.global_options.get('reportfile', None) + + if task_data.get('no-collector', False): + collector_address = None + net_test_loader = NetTestLoader( - options_to_args(task_data), + options_to_args(task_data, self.cwd), annotations=annotations, test_file=nettest_path ) @@ -658,11 +649,14 @@ class DeckTask(object): self.requires_bouncer = True

self.ooni['net_test_loader'] = net_test_loader - # Need to ensure that this is called only once we have looked up the - # probe IP address and have geoip data. - self.ooni['test_details'] = net_test_loader.getTestDetails() + + def _setup_ooni(self): + self.ooni['test_details'] = self.ooni['net_test_loader'].getTestDetails() self.id = generate_filename(self.ooni['test_details'])

+ def setup(self): + getattr(self, "_setup_"+self.type)() + def _load(self, data): for key in self._metadata_keys: try: @@ -678,11 +672,81 @@ class DeckTask(object):

assert len(data) == 0

+class NotAnOption(Exception): + pass + +def subargs_to_options(subargs): + options = {} + + def parse_option_name(arg): + if arg.startswith("--"): + return arg[2:] + elif arg.startswith("-"): + return arg[1:] + raise NotAnOption + + subargs = iter(reversed(subargs)) + for subarg in subargs: + try: + value = subarg + name = parse_option_name(subarg) + options[name] = True + except NotAnOption: + try: + name = parse_option_name(subargs.next()) + options[name] = value + except StopIteration: + break + + return options + +def convert_legacy_deck(deck_data): + """ + I take a legacy deck list and convert it to the new deck format. + + :param deck_data: in the legacy format + :return: deck_data in the new format + """ + assert isinstance(deck_data, list), "Legacy decks are lists" + new_deck_data = {} + new_deck_data["name"] = "Legacy deck" + new_deck_data["description"] = "This is a legacy deck converted to the " \ + "new format" + new_deck_data["bouncer"] = None + new_deck_data["tasks"] = [] + for deck_item in deck_data: + deck_task = {"ooni": {}} + + options = deck_item["options"] + deck_task["ooni"]["test_name"] = options.pop("test_file") + deck_task["ooni"]["annotations"] = options.pop("annotations", {}) + deck_task["ooni"]["collector"] = options.pop("collector", None) + + # XXX here we end up picking only the last not none bouncer_address + bouncer_address = options.pop("bouncer", None) + if bouncer_address is not None: + new_deck_data["bouncer"] = bouncer_address + + subargs = options.pop("subargs", []) + for name, value in subargs_to_options(subargs).items(): + deck_task["ooni"][name] = value + + for name, value in options.items(): + deck_task["ooni"][name] = value + + new_deck_data["tasks"].append(deck_task) + + return new_deck_data + class NGDeck(object): - def __init__(self, deck_data=None, - deck_path=None, no_collector=False): + def __init__(self, + deck_data=None, + deck_path=None, + global_options={}, + no_collector=False, + arbitrary_paths=False): # Used to resolve relative paths inside of decks. - self.deck_directory = None + self.deck_directory = os.getcwd() self.requires_tor = False self.no_collector = no_collector self.name = "" @@ -690,8 +754,12 @@ class NGDeck(object): self.schedule = None

self.metadata = {} + self.global_options = normalize_options(global_options) self.bouncer = None

+ self._arbitrary_paths = arbitrary_paths + self._is_setup = False + self._measurement_path = FilePath(config.measurements_directory) self._tasks = [] self.task_ids = [] @@ -701,35 +769,24 @@ class NGDeck(object): elif deck_data is not None: self.load(deck_data)

- def open(self, deck_path): + def open(self, deck_path, global_options=None): with open(deck_path) as fh: deck_data = yaml.safe_load(fh) - self.load(deck_data) + self.deck_directory = os.path.abspath(os.path.dirname(deck_path)) + self.load(deck_data, global_options)

- def write(self, fh): - """ - Writes a properly formatted deck to the supplied file handle. - :param fh: an open file handle - :return: - """ - deck_data = { - "name": self.name, - "description": self.description, - "tasks": [task.data for task in self._tasks] - } - if self.schedule is not None: - deck_data["schedule"] = self.schedule - for key, value in self.metadata.items(): - deck_data[key] = value + def load(self, deck_data, global_options=None): + if global_options is not None: + self.global_options = global_options

- fh.write("---\n") - yaml.safe_dump(deck_data, fh, default_flow_style=False) + if isinstance(deck_data, list): + deck_data = convert_legacy_deck(deck_data)

- def load(self, deck_data): self.name = deck_data.pop("name", "Un-named Deck") self.description = deck_data.pop("description", "No description")

- bouncer_address = deck_data.pop("bouncer", None) + bouncer_address = self.global_options.get('bouncer', + deck_data.pop("bouncer", None)) if bouncer_address is None: self.bouncer = get_preferred_bouncer() elif isinstance(bouncer_address, dict): @@ -743,8 +800,17 @@ class NGDeck(object): for key, metadata in deck_data.items(): self.metadata[key] = metadata

+ # We override the task metadata with the global options if present + self.metadata.update(self.global_options) + for task_data in tasks_data: - deck_task = DeckTask(task_data, self.metadata, self.deck_directory) + deck_task = DeckTask( + data=task_data, + parent_metadata=self.metadata, + global_options=self.global_options, + cwd=self.deck_directory, + arbitrary_paths=self._arbitrary_paths + ) if deck_task.requires_tor: self.requires_tor = True if (deck_task.requires_bouncer and @@ -753,6 +819,32 @@ class NGDeck(object): self._tasks.append(deck_task) self.task_ids.append(deck_task.id)

+ if self.metadata.get('no_collector', False): + self.no_collector = True + + @property + def tasks(self): + return self._tasks + + def write(self, fh): + """ + Writes a properly formatted deck to the supplied file handle. + :param fh: an open file handle + :return: + """ + deck_data = { + "name": self.name, + "description": self.description, + "tasks": [task.data for task in self._tasks] + } + if self.schedule is not None: + deck_data["schedule"] = self.schedule + for key, value in self.metadata.items(): + deck_data[key] = value + + fh.write("---\n") + yaml.safe_dump(deck_data, fh, default_flow_style=False) + @defer.inlineCallbacks def query_bouncer(self): preferred_backend = config.advanced.get( @@ -772,60 +864,74 @@ class NGDeck(object): preferred_backend, self.no_collector ) - - def _measurement_completed(self, result, measurement_id): - log.msg("{0}".format(result)) - measurement_dir = self._measurement_path.child(measurement_id) - measurement_dir.child("measurements.njson.progress").moveTo( - measurement_dir.child("measurements.njson") - ) - generate_summary( - measurement_dir.child("measurements.njson").path, - measurement_dir.child("summary.json").path - ) - measurement_dir.child("running.pid").remove() - - def _measurement_failed(self, failure, measurement_id): - measurement_dir = self._measurement_path.child(measurement_id) - measurement_dir.child("running.pid").remove() - # XXX do we also want to delete measurements.njson.progress? + defer.returnValue(net_test_loaders) + + def _measurement_completed(self, result, task): + if not task.output_path: + measurement_id = task.id + measurement_dir = self._measurement_path.child(measurement_id) + measurement_dir.child("measurements.njson.progress").moveTo( + measurement_dir.child("measurements.njson") + ) + generate_summary( + measurement_dir.child("measurements.njson").path, + measurement_dir.child("summary.json").path + ) + measurement_dir.child("running.pid").remove() + + def _measurement_failed(self, failure, task): + if not task.output_path: + # XXX do we also want to delete measurements.njson.progress? + measurement_id = task.id + measurement_dir = self._measurement_path.child(measurement_id) + measurement_dir.child("running.pid").remove() return failure

def _run_ooni_task(self, task, director): net_test_loader = task.ooni["net_test_loader"] test_details = task.ooni["test_details"] - measurement_id = task.id

- measurement_dir = self._measurement_path.child(measurement_id) - measurement_dir.createDirectory() + report_filename = task.output_path + if not task.output_path: + measurement_id = task.id

- report_filename = measurement_dir.child("measurements.njson.progress").path - pid_file = measurement_dir.child("running.pid") + measurement_dir = self._measurement_path.child(measurement_id) + measurement_dir.createDirectory()

- with pid_file.open('w') as out_file: - out_file.write("{0}".format(os.getpid())) + report_filename = measurement_dir.child("measurements.njson.progress").path + pid_file = measurement_dir.child("running.pid") + + with pid_file.open('w') as out_file: + out_file.write("{0}".format(os.getpid()))

d = director.start_net_test_loader( net_test_loader, report_filename, + collector_client=net_test_loader.collector, test_details=test_details ) - d.addCallback(self._measurement_completed, measurement_id) - d.addErrback(self._measurement_failed, measurement_id) + d.addCallback(self._measurement_completed, task) + d.addErrback(self._measurement_failed, task) return d

+ def setup(self): + """ + This method needs to be called before you are able to run a deck. + """ + for task in self._tasks: + task.setup() + self._is_setup = True + @defer.inlineCallbacks def run(self, director): - tasks = [] - preferred_backend = config.advanced.get("preferred_backend", "onion") + assert self._is_setup, "You must call setup() before you can run a " \ + "deck" + if self.requires_tor: + yield director.start_tor() yield self.query_bouncer() for task in self._tasks: - if task.requires_tor: - yield director.start_tor() - elif task.requires_bouncer and preferred_backend == "onion": - yield director.start_tor() if task.type == "ooni": - tasks.append(self._run_ooni_task(task, director)) - defer.returnValue(tasks) + yield self._run_ooni_task(task, director) + self._is_setup = False

input_store = InputStore() diff --git a/ooni/director.py b/ooni/director.py index 793975e..c239601 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -424,9 +424,7 @@ class Director(object): log.debug("Setting SOCKS port as %s" % tor_config.SocksPort) try: yield start_tor(tor_config) - log.err("Calling tor callback") self._tor_starting.callback(self._tor_state) - log.err("called") except Exception as exc: log.err("Failed to start tor") log.exc(exc) diff --git a/ooni/geoip.py b/ooni/geoip.py index 28e0e1e..f118268 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import re import os +import json import random

from hashlib import sha256 @@ -137,11 +138,12 @@ class UbuntuGeoIP(HTTPGeoIPLookupper): return probe_ip

class DuckDuckGoGeoIP(HTTPGeoIPLookupper): - url = "https://duckduckgo.com/?q=ip&ia=answer" + url = "https://api.duckduckgo.com/?q=ip&format=json"

def parseResponse(self, response_body): + j = json.loads(response_body) regexp = "Your IP address is (.*) in " - probe_ip = re.search(regexp, response_body).group(1) + probe_ip = re.search(regexp, j['Answer']).group(1) return probe_ip

class ProbeIP(object): diff --git a/ooni/reporter.py b/ooni/reporter.py index f07b3cf..e2f155f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -534,7 +534,7 @@ class Report(object): collector: The address of the oonib collector for this report.

- no_yamloo: + no_njson: If we should disable reporting to disk. """ self.test_details = test_details diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py index fa675f9..b980a2c 100644 --- a/ooni/scripts/oonideckgen.py +++ b/ooni/scripts/oonideckgen.py @@ -92,7 +92,7 @@ def get_user_country_code():

@defer.inlineCallbacks -def oonideckgen(): +def oonideckgen(reactor): options = Options() try: options.parseOptions() diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py new file mode 100644 index 0000000..24493da --- /dev/null +++ b/ooni/scripts/ooniprobe.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +from twisted.internet import task + +def ooniprobe(reactor): + from ooni.ui.cli import runWithDaemonDirector, runWithDirector + from ooni.ui.cli import setupGlobalOptions + + global_options = setupGlobalOptions(logging=True, start_tor=True, + check_incoherences=True) + if global_options['queue']: + return runWithDaemonDirector(global_options) + else: + return runWithDirector(global_options) + +def run(): + task.react(ooniprobe) + +if __name__ == "__main__": + run() diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py index 6facadf..d75c925 100644 --- a/ooni/scripts/oonireport.py +++ b/ooni/scripts/oonireport.py @@ -202,7 +202,7 @@ def tor_check(): sys.exit(1)

-def oonireport(args=sys.argv[1:]): +def oonireport(reactor, args=sys.argv[1:]): options = Options() try: options.parseOptions(args) diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py index 5e1d21a..bb080e6 100644 --- a/ooni/templates/tcpt.py +++ b/ooni/templates/tcpt.py @@ -96,4 +96,3 @@ class TCPTest(NetTestCase): d2.addCallback(connected) d2.addErrback(errback) return d1 - diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py index c15beec..d623e0b 100644 --- a/ooni/tests/mocks.py +++ b/ooni/tests/mocks.py @@ -191,7 +191,7 @@ class MockTaskManager(TaskManager):

class MockBouncerClient(object): def __init__(self, *args, **kw): - pass + self.backend_type = "onion"

def lookupTestHelpers(self, required_test_helpers): ret = { diff --git a/ooni/tests/test_backend_client.py b/ooni/tests/test_backend_client.py index 212abb6..fab1fa2 100644 --- a/ooni/tests/test_backend_client.py +++ b/ooni/tests/test_backend_client.py @@ -60,22 +60,6 @@ class TestEnd2EndBackendClient(ConfigTestCase): def test_download_input(self): yield self.collector_client.downloadInput(input_id)

- @defer.inlineCallbacks - def test_get_deck_list(self): - deck_list = yield self.collector_client.getDeckList() - self.assertTrue(isinstance(deck_list, list)) - - @defer.inlineCallbacks - def test_get_deck_descriptor(self): - deck_descriptor = yield self.collector_client.getDeck(deck_id) - for key in ['name', 'description', - 'version', 'author', 'date', 'id']: - self.assertTrue(hasattr(deck_descriptor, key)) - - @defer.inlineCallbacks - def test_download_deck(self): - yield self.collector_client.downloadDeck(deck_id) - def test_lookup_invalid_helpers(self): bouncer_client = BouncerClient('http://127.0.0.1:8888') return self.failUnlessFailure( diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index 455f4e1..3d07959 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -1,15 +1,20 @@ import os

+from StringIO import StringIO from copy import deepcopy

+import yaml + +from mock import patch, MagicMock + from twisted.internet import defer from twisted.trial import unittest

from hashlib import sha256 from ooni import errors from ooni.deck import input_store, lookup_collector_and_test_helpers -from ooni.nettest import NetTestLoader -from ooni.deck import InputFile, Deck, nettest_to_path, DeckTask, NGDeck +from ooni.deck import nettest_to_path, NGDeck +from ooni.deck import convert_legacy_deck from ooni.tests.bases import ConfigTestCase from ooni.tests.mocks import MockBouncerClient, MockCollectorClient

@@ -80,45 +85,6 @@ class BaseTestCase(unittest.TestCase): """ super(BaseTestCase, self).setUp()

- -class TestInputFile(BaseTestCase): - def tearDown(self): - if self.filename != "": - os.remove(self.filename) - - def test_file_cached(self): - self.filename = file_hash = sha256(self.dummy_deck_content).hexdigest() - input_file = InputFile(file_hash, base_path='.') - with open(file_hash, 'w+') as f: - f.write(self.dummy_deck_content) - assert input_file.fileCached - - def test_file_invalid_hash(self): - self.filename = invalid_hash = 'a' * 64 - with open(invalid_hash, 'w+') as f: - f.write("b" * 100) - input_file = InputFile(invalid_hash, base_path='.') - self.assertRaises(AssertionError, input_file.verify) - - def test_save_descriptor(self): - descriptor = { - 'name': 'spam', - 'id': 'spam', - 'version': 'spam', - 'author': 'spam', - 'date': 'spam', - 'description': 'spam' - } - file_id = 'a' * 64 - self.filename = file_id + '.desc' - input_file = InputFile(file_id, base_path='.') - input_file.load(descriptor) - input_file.save() - assert os.path.isfile(self.filename) - - assert input_file.descriptorCached - - class TestDeck(BaseTestCase, ConfigTestCase): def setUp(self): super(TestDeck, self).setUp() @@ -137,60 +103,47 @@ class TestDeck(BaseTestCase, ConfigTestCase): super(TestDeck, self).tearDown()

def test_open_deck(self): - deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, - decks_directory=".") - deck.loadDeck(self.deck_file) - assert len(deck.netTestLoaders) == 1 + deck = NGDeck() + deck.open(self.deck_file) + assert len(deck.tasks.ooni['net_test_loaders']) == 1

def test_load_deck_with_global_options(self): global_options = { "annotations": {"spam": "ham"}, "collector": "httpo://thirteenchars123.onion" } - deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, - decks_directory=".") - deck.loadDeck(self.deck_file, - global_options=global_options) + deck = NGDeck(global_options=global_options) + deck.open(self.deck_file) self.assertEqual( - deck.netTestLoaders[0].annotations, + deck.tasks.ooni['net_test_loaders'][0].annotations, global_options['annotations'] ) self.assertEqual( - deck.netTestLoaders[0].collector.base_address, + deck.tasks.ooni['net_test_loaders'][0].collector.base_address, global_options['collector'].replace("httpo://", "http://") )

- def test_save_deck_descriptor(self): - deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, - decks_directory=".") - deck.loadDeck(self.deck_file) - deck.load({'name': 'spam', - 'id': 'spam', - 'version': 'spam', - 'author': 'spam', - 'date': 'spam', - 'description': 'spam' - }) - deck.save() - self.filename = self.deck_file + ".desc" - deck.verify() - + @patch('ooni.deck.BouncerClient', MockBouncerClient) + @patch('ooni.deck.CollectorClient', MockCollectorClient) @defer.inlineCallbacks def test_lookup_test_helpers_and_collector(self): - deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, - decks_directory=".") + deck = NGDeck() deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS) - deck._BouncerClient = MockBouncerClient - deck._CollectorClient = MockCollectorClient - deck.loadDeck(self.deck_file) + deck.open(self.deck_file)

- self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1) + self.assertEqual( + len(deck.tasks.ooni['net_test_loaders'][0].missingTestHelpers), 1)

- yield lookup_collector_and_test_helpers(deck.preferred_backend, - deck.netTestLoaders) + yield lookup_collector_and_test_helpers( + net_test_loaders=deck.netTestLoaders, + preferred_backend=deck.preferred_backend, + bouncer=deck.bouncer + )

- self.assertEqual(deck.netTestLoaders[0].collector.settings['address'], - 'httpo://thirteenchars123.onion') + self.assertEqual( + deck.tasks.ooni['net_test_loaders'][0].collector.settings['address'], + 'httpo://thirteenchars123.onion' + )

self.assertEqual(deck.netTestLoaders[0].localOptions['backend'], '127.0.0.1') @@ -202,15 +155,15 @@ class TestDeck(BaseTestCase, ConfigTestCase): self.deck_file = os.path.join(self.cwd, deck_hash) with open(self.deck_file, 'w+') as f: f.write(self.dummy_deck_content_with_many_tests) - deck = Deck(decks_directory=".") - deck.loadDeck(self.deck_file) + deck = NGDeck() + deck.open(self.deck_file)

self.assertEqual( - deck.netTestLoaders[0].localOptions['backend'], + deck.tasks[0].ooni['net_test_loader'].localOptions['backend'], '1.1.1.1' ) self.assertEqual( - deck.netTestLoaders[1].localOptions['backend'], + deck.tasks[1].ooni['net_test_loader'].localOptions['backend'], '2.2.2.2' )

@@ -222,58 +175,65 @@ class TestDeck(BaseTestCase, ConfigTestCase): nettest_to_path, "invalid_test")

+ @patch('ooni.deck.BouncerClient', MockBouncerClient) + @patch('ooni.deck.CollectorClient', MockCollectorClient) @defer.inlineCallbacks def test_lookup_test_helpers_and_collector_cloudfront(self): self.config.advanced.preferred_backend = "cloudfront" - deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, - decks_directory=".") - deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS) - deck._BouncerClient = MockBouncerClient - deck._CollectorClient = MockCollectorClient - deck.loadDeck(self.deck_file) - - self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1) - - yield lookup_collector_and_test_helpers(deck.preferred_backend, - deck.netTestLoaders) + deck = NGDeck() + deck.open(self.deck_file) + first_net_test_loader = deck.tasks[0].ooni['net_test_loader'] + net_test_loaders = map(lambda task: task.ooni['net_test_loader'], + deck.tasks) + self.assertEqual(len(first_net_test_loader.missingTestHelpers), 1) + + yield lookup_collector_and_test_helpers( + net_test_loaders=net_test_loaders , + preferred_backend='cloudfront', + bouncer=deck.bouncer + )

self.assertEqual( - deck.netTestLoaders[0].collector.settings['address'], + first_net_test_loader.collector.settings['address'], 'https://address.cloudfront.net' ) self.assertEqual( - deck.netTestLoaders[0].collector.settings['front'], + first_net_test_loader.collector.settings['front'], 'front.cloudfront.net' )

self.assertEqual( - deck.netTestLoaders[0].localOptions['backend'], + first_net_test_loader.localOptions['backend'], '127.0.0.1' )

- + @patch('ooni.deck.BouncerClient', MockBouncerClient) + @patch('ooni.deck.CollectorClient', MockCollectorClient) @defer.inlineCallbacks def test_lookup_test_helpers_and_collector_https(self): self.config.advanced.preferred_backend = "https" - deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, - decks_directory=".") - deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS) - deck._BouncerClient = MockBouncerClient - deck._CollectorClient = MockCollectorClient - deck.loadDeck(self.deck_file) + deck = NGDeck() + deck.open(self.deck_file) + + first_net_test_loader = deck.tasks[0].ooni['net_test_loader'] + net_test_loaders = map(lambda task: task.ooni['net_test_loader'], + deck.tasks)

- self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1) + self.assertEqual(len(first_net_test_loader .missingTestHelpers), 1)

- yield lookup_collector_and_test_helpers(deck.preferred_backend, - deck.netTestLoaders) + yield lookup_collector_and_test_helpers( + net_test_loaders=net_test_loaders, + preferred_backend='https', + bouncer=deck.bouncer + )

self.assertEqual( - deck.netTestLoaders[0].collector.settings['address'], + first_net_test_loader.collector.settings['address'], 'https://collector.ooni.io' )

self.assertEqual( - deck.netTestLoaders[0].localOptions['backend'], + first_net_test_loader.localOptions['backend'], '127.0.0.1' )

@@ -301,20 +261,65 @@ DECK_DATA = { deepcopy(TASK_DATA) ] } + +LEGACY_DECK = """ +- options: + annotations: null + bouncer: null + collector: null + no-collector: 0 + no-geoip: 0 + no-yamloo: 0 + reportfile: null + subargs: [--flag, --key, value] + test_file: manipulation/http_invalid_request_line + verbose: 0 +- options: + annotations: null + bouncer: null + collector: null + no-collector: 0 + no-geoip: 0 + no-yamloo: 0 + reportfile: null + subargs: [] + test_file: manipulation/http_header_field_manipulation + verbose: 0 +- options: + annotations: null + bouncer: null + collector: null + no-collector: 0 + no-geoip: 0 + no-yamloo: 0 + reportfile: null + subargs: [-f, /path/to/citizenlab-urls-global.txt] + test_file: blocking/web_connectivity + verbose: 0 +""" + class TestNGDeck(ConfigTestCase): - skip = True def test_deck_task(self): - if self.skip: - self.skipTest("Skip is set to true") - yield input_store.update("ZZ") - deck_task = DeckTask(TASK_DATA) - self.assertIsInstance(deck_task.ooni["net_test_loader"], - NetTestLoader) + #yield input_store.update("ZZ") + #deck_task = DeckTask(TASK_DATA) + #self.assertIsInstance(deck_task.ooni["net_test_loader"], + # NetTestLoader) + pass

- @defer.inlineCallbacks def test_deck_load(self): - if self.skip: - self.skipTest("Skip is set to true") - yield input_store.update("ZZ") - deck = NGDeck(deck_data=DECK_DATA) - self.assertEqual(len(deck.tasks), 1) + #yield input_store.update("ZZ") + #deck = NGDeck(deck_data=DECK_DATA) + #self.assertEqual(len(deck.tasks), 1) + pass + + def test_convert_legacy_deck(self): + legacy_deck = yaml.safe_load(StringIO(LEGACY_DECK)) + ng_deck = convert_legacy_deck(legacy_deck) + self.assertEqual(len(ng_deck['tasks']), 3) + task_names = map(lambda task: task['ooni']['test_name'], + ng_deck['tasks']) + self.assertItemsEqual(task_names, [ + "manipulation/http_invalid_request_line", + "manipulation/http_header_field_manipulation", + "blocking/web_connectivity" + ]) diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index fe24bf6..6550b3d 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -26,19 +26,18 @@ class Options(usage.Options):

optFlags = [["help", "h"], ["no-collector", "n", "Disable writing to collector"], - ["no-yamloo", "N", "Disable writing to YAML file"], + ["no-njson", "N", "Disable writing to disk"], ["no-geoip", "g", "Disable geoip lookup on start"], ["list", "s", "List the currently installed ooniprobe " "nettests"], - ["printdeck", "p", "Print the equivalent deck for the " - "provided command"], ["verbose", "v", "Show more verbose information"] ]

optParameters = [ - ["reportfile", "o", None, "Specify the report file name to write to."], + ["reportfile", "o", None, "Specify the report file name to write " + "to."], ["testdeck", "i", None, "Specify as input a test deck: a yaml file " - "containing the tests to run and their " + "containing the tests to run and their " "arguments."], ["collector", "c", None, "Specify the address of the collector for " "test results. In most cases a user will " @@ -132,7 +131,8 @@ def director_startup_handled_failures(failure): errors.CouldNotFindTestCollector, errors.ProbeIPUnknown, errors.InvalidInputFile, - errors.ConfigFileIncoherent) + errors.ConfigFileIncoherent, + SystemExit)

if isinstance(failure.value, errors.TorNotRunning): log.err("Tor does not appear to be running") @@ -236,64 +236,71 @@ def setupCollector(global_options, collector_client): return collector_client

def createDeck(global_options, url=None): - from ooni.nettest import NetTestLoader - from ooni.deck import Deck, nettest_to_path - from ooni.backend_client import CollectorClient + from ooni.deck import NGDeck, subargs_to_options

if url: log.msg("Creating deck for: %s" % (url))

- if global_options['no-yamloo']: - log.msg("Will not write to a yamloo report file") - - deck = Deck(bouncer=global_options['bouncer'], - no_collector=global_options['no-collector']) - + test_deck_path = global_options.pop('testdeck', None) + test_name = global_options.pop('test_file', None) + no_collector = global_options.pop('no-collector', False) try: - if global_options['testdeck']: - deck.loadDeck(global_options['testdeck'], global_options) + if test_deck_path is not None: + deck = NGDeck( + global_options=global_options, + no_collector=no_collector + ) + deck.open(test_deck_path) else: + deck = NGDeck( + global_options=global_options, + no_collector=no_collector, + arbitrary_paths=True + ) log.debug("No test deck detected") - test_file = nettest_to_path(global_options['test_file'], True) if url is not None: args = ('-u', url) else: args = tuple() if any(global_options['subargs']): args = global_options['subargs'] + args - net_test_loader = NetTestLoader(args, - test_file=test_file, - annotations=global_options['annotations']) - if global_options['collector']: - net_test_loader.collector = \ - CollectorClient(global_options['collector']) - deck.insert(net_test_loader) + + test_options = subargs_to_options(args) + test_options['test_name'] = test_name + deck.load({ + "tasks": [ + {"ooni": test_options} + ] + }) except errors.MissingRequiredOption as option_name: log.err('Missing required option: "%s"' % option_name) incomplete_net_test_loader = option_name.net_test_loader print incomplete_net_test_loader.usageOptions().getUsage() - sys.exit(2) + raise SystemExit(2) + except errors.NetTestNotFound as path: log.err('Requested NetTest file not found (%s)' % path) - sys.exit(3) + raise SystemExit(3) + except errors.OONIUsageError as e: log.err(e) print e.net_test_loader.usageOptions().getUsage() - sys.exit(4) + raise SystemExit(4) + except errors.HTTPSCollectorUnsupported: log.err("HTTPS collectors require a twisted version of at least 14.0.2.") - sys.exit(6) + raise SystemExit(6) except errors.InsecureBackend: log.err("Attempting to report to an insecure collector.") log.err("To enable reporting to insecure collector set the " "advanced->insecure_backend option to true in " "your ooniprobe.conf file.") - sys.exit(7) + raise SystemExit(7) except Exception as e: if config.advanced.debug: log.exception(e) log.err(e) - sys.exit(5) + raise SystemExit(5)

return deck

@@ -301,45 +308,21 @@ def createDeck(global_options, url=None): def runTestWithDirector(director, global_options, url=None, start_tor=True): deck = createDeck(global_options, url=url)

- start_tor |= deck.requiresTor - - d = director.start(start_tor=start_tor, - check_incoherences=global_options['check_incoherences']) - - def setup_nettest(_): + d = director.start() + @defer.inlineCallbacks + def post_director_start(_): try: - return deck.setup() + deck.setup() + yield deck.run(director) except errors.UnableToLoadDeckInput as error: - return defer.failure.Failure(error) + raise defer.failure.Failure(error) except errors.NoReachableTestHelpers as error: - return defer.failure.Failure(error) + raise defer.failure.Failure(error) except errors.NoReachableCollectors as error: - return defer.failure.Failure(error) + raise defer.failure.Failure(error) + except SystemExit as error: + raise error

- # Wait until director has started up (including bootstrapping Tor) - # before adding tests - @defer.inlineCallbacks - def post_director_start(_): - for net_test_loader in deck.netTestLoaders: - # Decks can specify different collectors - # for each net test, so that each NetTest - # may be paired with a test_helper and its collector - # However, a user can override this behavior by - # specifying a collector from the command-line (-c). - # If a collector is not specified in the deck, or the - # deck is a singleton, the default collector set in - # ooniprobe.conf will be used - collector_client = None - if not global_options['no-collector']: - collector_client = setupCollector(global_options, - net_test_loader.collector) - - yield director.start_net_test_loader(net_test_loader, - global_options['reportfile'], - collector_client, - global_options['no-yamloo']) - - d.addCallback(setup_nettest) d.addCallback(post_director_start) d.addErrback(director_startup_handled_failures) d.addErrback(director_startup_other_failures) @@ -379,14 +362,7 @@ def runWithDirector(global_options): print "Note: Third party tests require an external "\ "application to run properly."

- sys.exit(0) - - elif global_options['printdeck']: - del global_options['printdeck'] - print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments" - print yaml.safe_dump([{'options': global_options}]).strip() - - sys.exit(0) + raise SystemExit(0)

if global_options.get('annotations') is not None: global_options['annotations'] = setupAnnotations(global_options) @@ -427,7 +403,7 @@ def runWithDaemonDirector(global_options): except ImportError: print "Pika is required for queue connection." print "Install with "pip install pika"." - sys.exit(7) + raise SystemExit(7)

director = Director()

diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index e306ef2..e90ef83 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?7ed7d7510803fa1a4ad8"></script></body> + <script type="text/javascript" src="app.bundle.js?6b15c1dd202a0f5a80e7"></script></body> </html> diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/set_caps/Makefile b/scripts/set_caps/Makefile new file mode 100644 index 0000000..bc80843 --- /dev/null +++ b/scripts/set_caps/Makefile @@ -0,0 +1,81 @@ +# Wrappers for running ooniprobe as a non-root user. +# +# Build-Depends: cython, pythonX.Y-dev, libcap2-bin +# Depends: libpythonX.Y +# +# $ make && make check +# $ sudo make install # after installing the rest of ooni-probe +# $ make installcheck_unsafe # runs complete tests as non-root +# +# `make` builds a program that has file capabilities set on it. This is just +# ./ooniprobe compiled into a C program using Cython, so that one can set +# capabilities directly on the resulting binary. This way, we avoid the need +# for a separate child python interpreter with its own capabilities. Another +# advantage is that libpython.so (needed by the program) would be automatically +# upgraded by the system package manager. The version of python is hard-coded +# into the wrapper at build time; making this dynamic is possible, but much +# more complex and not yet implemented. +# +# Execution may additionally be limited to a particular unix group by using +# chgrp(1) and chmod(1) to 'o-x,g+x' after installation. +# + +# GNU Makefile conventions, see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html +prefix = /usr/local +exec_prefix = $(prefix) +bindir = $(exec_prefix)/bin + +INSTALL = install +PYTHON = python +PYTHON_CONFIG = python-config +CYTHON = cython +SETCAP = setcap + +INSTALL_PROGRAM = $(INSTALL) +PY_CFLAGS = $(shell $(PYTHON_CONFIG) --cflags) +PY_LDFLAGS = $(shell $(PYTHON_CONFIG) --ldflags) + +BUILDDIR := ./build +SCRIPTDIR := . +TESTDIR := ./test +CAP_SCRIPT := ooniprobe +CAP_NEEDED := cap_net_admin,cap_net_raw + +# Unfortunately cython --embed ignores the arguments in the shebang line +# So we need to patch the generated code ourselves. +CYTHON_PRE_MAIN = extern int Py_IgnoreEnvironmentFlag; \ + Py_IgnoreEnvironmentFlag++; \ + extern int Py_NoUserSiteDirectory; \ + Py_NoUserSiteDirectory++; + +all: $(BUILDDIR)/$(CAP_SCRIPT) + +$(BUILDDIR)/$(CAP_SCRIPT): $(BUILDDIR)/$(CAP_SCRIPT).c Makefile + $(CC) $(PY_CFLAGS) $(PY_LDFLAGS) "$<" -o "$@" + +$(BUILDDIR)/$(CAP_SCRIPT).c: $(SCRIPTDIR)/$(CAP_SCRIPT) Makefile + mkdir -p "$(BUILDDIR)" + $(CYTHON) "$<" --embed=CYTHON_MAIN_SENTINEL -Werror -Wextra -o "$@" + sed -i \ + -e 's/(.*CYTHON_MAIN_SENTINEL.*{)/\1 $(CYTHON_PRE_MAIN)/g' \ + -e '/CYTHON_MAIN_SENTINEL[^{]*$$/,/{/s/{/{ $(CYTHON_PRE_MAIN)/g' \ + -e 's/CYTHON_MAIN_SENTINEL/main/g' "$@" + +check: $(BUILDDIR)/$(CAP_SCRIPT) + # test that setcapped binary ignores PYTHONPATH + BIN="$$(realpath "$<")" && cd "$(TESTDIR)" && PYTHONPATH=. $$BIN --version + +install: $(BUILDDIR)/$(CAP_SCRIPT) + mkdir -p "$(DESTDIR)$(bindir)" + $(INSTALL_PROGRAM) -t "$(DESTDIR)$(bindir)" "$(BUILDDIR)/$(CAP_SCRIPT)" + $(SETCAP) "$(CAP_NEEDED)"+eip "$(DESTDIR)$(bindir)/$(CAP_SCRIPT)" + +installcheck_unsafe: $(BUILDDIR)/$(CAP_SCRIPT) + # run a standard check. note that because of hardcoded paths (for security) + # this can only work after you've installed your development copy + "./$<" -i /usr/share/ooni/decks/complete.deck + +clean: + rm -rf "$(BUILDDIR)" + +.PHONY: clean all check install installcheck% diff --git a/scripts/set_caps/__init__.py b/scripts/set_caps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/set_caps/test/__init__.py b/scripts/set_caps/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/set_caps/test/ooni/__init__.py b/scripts/set_caps/test/ooni/__init__.py new file mode 100644 index 0000000..f2cd353 --- /dev/null +++ b/scripts/set_caps/test/ooni/__init__.py @@ -0,0 +1 @@ +raise ValueError("test failed! wrapper did not ignore polluted PWD. either the wrapper is faulty, or ooni is still unpatched (Tor bug #13581)")