tor-commits September 2016

tor-commits@lists.torproject.org

20 participants
1187 discussions

[ooni-probe/master] Bump to alpha.2
by art＠torproject.org 19 Sep '16

19 Sep '16

commit f3d3bb10d52e472c880eaa0ca3f69cc3fe14dc6d Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Aug 8 16:42:16 2016 +0200 Bump to alpha.2 * Use semantic versioning --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 1a31608..9eecb67 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "2.0.0a1" +__version__ = "2.0.0-alpha.2" __all__ = [ 'agent',

1 0

[ooni-probe/master] Implement disk quota management
by art＠torproject.org 19 Sep '16

19 Sep '16

commit def4c929d162852ace8a016bb3352677eec5bcde Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Aug 30 02:09:55 2016 +0200 Implement disk quota management This is related to the feature described in here: https://github.com/TheTorProject/lepidopter/issues/53 --- ooni/agent/scheduler.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++ ooni/measurements.py | 20 +++++++++++---- ooni/settings.py | 6 ++++- ooni/tests/test_utils.py | 34 ++++++++++++++++++++++++ ooni/ui/web/server.py | 12 ++++++++- ooni/utils/files.py | 34 ++++++++++++++++++++++++ 6 files changed, 166 insertions(+), 7 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 3389db1..54551f6 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -1,3 +1,6 @@ +import os +import errno + from datetime import datetime from twisted.application import service @@ -8,6 +11,7 @@ from twisted.python.filepath import FilePath from ooni.scripts import oonireport from ooni import resources from ooni.utils import log, SHORT_DATE +from ooni.utils.files import human_size_to_bytes, directory_usage from ooni.deck.store import input_store, deck_store from ooni.settings import config from ooni.contrib import croniter @@ -146,6 +150,66 @@ class DeleteOldReports(ScheduledTask): measurement_path.child(measurement['id']).remove() +class CheckMeasurementQuota(ScheduledTask): + """ + This task is run to ensure we don't run out of disk space and deletes + older reports to avoid filling the quota. + """ + identifier = 'check-measurement-quota' + schedule = '@hourly' + _warn_when = 0.8 + + def task(self): + if config.basic.measurement_quota is None: + return + maximum_bytes = human_size_to_bytes(config.basic.measurement_quota) + available_bytes = directory_usage(config.measurements_directory) + warning_path = os.path.join(config.running_path, 'quota_warning') + + if (float(available_bytes) / float(maximum_bytes)) >= self._warn_when: + log.warn("You are about to reach the maximum allowed quota. Be careful") + with open(warning_path, "w") as out_file: + out_file.write("{0} {1}".split(available_bytes, maximum_bytes)) + else: + try: + os.remove(warning_path) + except OSError as ose: + if ose.errno != errno.ENOENT: + raise + + if float(available_bytes) < float(maximum_bytes): + # We are within the allow quota exit. + return + + # We should begin to delete old reports + amount_to_delete = float(maximum_bytes) - float(available_bytes) + amount_deleted = 0 + measurement_path = FilePath(config.measurements_directory) + + kept_measurements = [] + stale_measurements = [] + remaining_measurements = [] + measurements_by_date = sorted(list_measurements(compute_size=True), + key=lambda k: k['test_start_time']) + for measurement in measurements_by_date: + if measurement['keep'] is True: + kept_measurements.append(measurement) + elif measurement['stale'] is True: + stale_measurements.append(measurement) + else: + remaining_measurements.append(measurement) + + # This is the order in which we should begin deleting measurements. + ordered_measurements = (stale_measurements + + remaining_measurements + + kept_measurements) + while amount_deleted < amount_to_delete: + measurement = ordered_measurements.pop(0) + log.warn("Deleting report {0}".format(measurement["id"])) + measurement_path.child(measurement['id']).remove() + amount_deleted += measurement['size'] + + class RunDeck(ScheduledTask): """ This will run the decks that have been configured on the system as the @@ -196,6 +260,7 @@ SYSTEM_TASKS = [ UpdateInputsAndResources ] + @defer.inlineCallbacks def run_system_tasks(no_input_store=False): task_classes = SYSTEM_TASKS[:] @@ -215,6 +280,7 @@ def run_system_tasks(no_input_store=False): log.err("Failed to run task {0}".format(task.identifier)) log.exception(exc) + class SchedulerService(service.MultiService): """ This service is responsible for running the periodic tasks. @@ -271,6 +337,7 @@ class SchedulerService(service.MultiService): self.schedule(UpdateInputsAndResources()) self.schedule(UploadReports()) self.schedule(DeleteOldReports()) + self.schedule(CheckMeasurementQuota()) self.schedule(RefreshDeckList(self)) self._looping_call.start(self.interval) diff --git a/ooni/measurements.py b/ooni/measurements.py index 6d90c1b..fd722ee 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -4,6 +4,7 @@ import signal from twisted.python.filepath import FilePath from ooni.utils import log +from ooni.utils.files import directory_usage from ooni.settings import config class MeasurementInProgress(Exception): @@ -61,7 +62,8 @@ def generate_summary(input_file, output_file): class MeasurementNotFound(Exception): pass -def get_measurement(measurement_id): +def get_measurement(measurement_id, compute_size=False): + size = -1 measurement_path = FilePath(config.measurements_directory) measurement = measurement_path.child(measurement_id) if not measurement.exists(): @@ -70,6 +72,7 @@ def get_measurement(measurement_id): running = False completed = True keep = False + stale = False if measurement.child("measurements.njson.progress").exists(): completed = False # XXX this is done quite often around the code, probably should @@ -80,10 +83,14 @@ def get_measurement(measurement_id): os.kill(pid, signal.SIG_DFL) running = True except OSError: - pass + stale = True if measurement.child("keep").exists(): keep = True + + if compute_size is True: + size = directory_usage(measurement.path) + test_start_time, country_code, asn, test_name = \ measurement_id.split("-")[:4] return { @@ -94,7 +101,9 @@ def get_measurement(measurement_id): "id": measurement_id, "completed": completed, "keep": keep, - "running": running + "running": running, + "stale": stale, + "size": size } @@ -115,12 +124,13 @@ def get_summary(measurement_id): with summary.open("r") as f: return json.load(f) -def list_measurements(): + +def list_measurements(compute_size=False): measurements = [] measurement_path = FilePath(config.measurements_directory) for measurement_id in measurement_path.listdir(): try: - measurements.append(get_measurement(measurement_id)) + measurements.append(get_measurement(measurement_id, compute_size)) except: log.err("Failed to get metadata for measurement {0}".format(measurement_id)) return measurements diff --git a/ooni/settings.py b/ooni/settings.py index 2bacd57..c36f3f7 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -20,6 +20,9 @@ basic: # Where OONIProbe should be writing it's log file logfile: {logfile} loglevel: WARNING + # The maximum amount of data to store on disk. Once the quota is reached, + # we will start deleting older reports. + # measurement_quota: 1G privacy: # Should we include the IP address of the probe in the report? includeip: {include_ip} @@ -99,7 +102,8 @@ tor: defaults = { "basic": { "loglevel": "WARNING", - "logfile": "ooniprobe.log" + "logfile": "ooniprobe.log", + "measurement_quota": "1G" }, "privacy": { "includeip": False, diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index ea1d858..4ef2926 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -1,7 +1,10 @@ import os +import tempfile + from twisted.trial import unittest from ooni.utils import log, generate_filename, net +from ooni.utils.files import human_size_to_bytes, directory_usage class TestUtils(unittest.TestCase): @@ -43,3 +46,34 @@ class TestUtils(unittest.TestCase): def test_get_addresses(self): addresses = net.getAddresses() assert isinstance(addresses, list) + + def test_human_size(self): + self.assertEqual( + human_size_to_bytes("1G"), + 1024**3 + ) + self.assertEqual( + human_size_to_bytes("1.3M"), + 1.3 * 1024**2 + ) + self.assertEqual( + human_size_to_bytes("1.2K"), + 1.2 * 1024 + ) + self.assertEqual( + human_size_to_bytes("1"), + 1.0 + ) + self.assertEqual( + human_size_to_bytes("100.2"), + 100.2 + ) + + def test_directory_usage(self): + tmp_dir = tempfile.mkdtemp() + with open(os.path.join(tmp_dir, "something.txt"), "w") as out_file: + out_file.write("A"*1000) + os.mkdir(os.path.join(tmp_dir, "subdir")) + with open(os.path.join(tmp_dir, "subdir", "something.txt"), "w") as out_file: + out_file.write("A"*1000) + self.assertEqual(directory_usage(tmp_dir), 1000*2) diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index ee33fa2..6f9ef69 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -1,6 +1,8 @@ from __future__ import print_function +import os import json +import errno import string from functools import wraps from random import SystemRandom @@ -186,13 +188,21 @@ class WebUIAPI(object): @property def status(self): + quota_warning = None + try: + with open(os.path.join(config.running_dir, "quota_warning")) as in_file: + quota_warning = in_file.read() + except IOError as ioe: + if ioe.errno != errno.ENOENT: + raise return { "software_version": ooniprobe_version, "software_name": "ooniprobe", "asn": probe_ip.geodata['asn'], "country_code": probe_ip.geodata['countrycode'], "director_started": self._director_started, - "initialized": self._is_initialized + "initialized": self._is_initialized, + "quota_warning": quota_warning } def handle_director_event(self, event): diff --git a/ooni/utils/files.py b/ooni/utils/files.py new file mode 100644 index 0000000..aefb831 --- /dev/null +++ b/ooni/utils/files.py @@ -0,0 +1,34 @@ +import os +import re + +HUMAN_SIZE = re.compile("(\d+\.?\d*G)|(\d+\.?\d*M)|(\d+\.?\d*K)|(\d+\.?\d*)") + +class InvalidFormat(Exception): + pass + +def human_size_to_bytes(human_size): + """ + Converts a size specified in a human friendly way (for example 1G, 10M, + 30K) into bytes. + """ + gb, mb, kb, b = HUMAN_SIZE.match(human_size).groups() + if gb is not None: + b = float(gb[:-1]) * (1024 ** 3) + elif mb is not None: + b = float(mb[:-1]) * (1024 ** 2) + elif kb is not None: + b = float(kb[:-1]) * 1024 + elif b is not None: + b = float(b) + else: + raise InvalidFormat + return b + + +def directory_usage(path): + total_usage = 0 + for root, dirs, filenames in os.walk(path): + for filename in filenames: + fp = os.path.join(root, filename) + total_usage += os.path.getsize(fp) + return total_usage

1 0

[ooni-probe/master] Merge branch 'master' into v2.0.0-alpha
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 847a951b40d1355c7f18d8e91001ad9a1e5fc984 Merge: 8048dbd da00195 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Aug 26 18:33:38 2016 +0200 Merge branch 'master' into v2.0.0-alpha * master: Don't look for the namebench csv file since we don't ship it

1 0

[ooni-probe/master] Add support for initialization of ooniprobe
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 27a299c5c7d195860e63aa0b1d316f3255e4857d Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Aug 3 13:05:31 2016 +0200 Add support for initialization of ooniprobe --- data/decks/web.yaml | 1 + ooni/agent/scheduler.py | 25 +++--- ooni/scripts/ooniprobe.py | 4 +- ooni/settings.py | 191 ++++++++++++++++++++++++++++++++++++++---- ooni/tests/__init__.py | 7 -- ooni/ui/cli.py | 52 +++++++++++- ooni/ui/web/client/index.html | 2 +- ooni/ui/web/server.py | 82 +++++++++++++++++- ooni/utils/__init__.py | 1 - 9 files changed, 329 insertions(+), 36 deletions(-) diff --git a/data/decks/web.yaml b/data/decks/web.yaml index a81b8f8..c7b9bdc 100644 --- a/data/decks/web.yaml +++ b/data/decks/web.yaml @@ -2,6 +2,7 @@ name: Web related ooniprobe tests description: This deck runs HTTP Header Field Manipulation, HTTP Invalid Request and the Web Connectivity test +schedule: "@daily" tasks: - name: Runs the HTTP Header Field Manipulation test ooni: diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index a6d689f..1f51bd4 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -22,9 +22,11 @@ class ScheduledTask(object): schedule = None identifier = None - def __init__(self, schedule=None): + def __init__(self, schedule=None, identifier=None): if schedule is not None: self.schedule = schedule + if identifier is not None: + self.identifier = identifier assert self.identifier is not None, "self.identifier must be set" assert self.schedule is not None, "self.schedule must be set" @@ -120,23 +122,23 @@ class DeleteOldReports(ScheduledTask): measurement_path.child(measurement['id']).remove() -class RunDecks(ScheduledTask): +class RunDeck(ScheduledTask): """ This will run the decks that have been configured on the system as the decks to run by default. """ - schedule = '@daily' - identifier = 'run-decks' - def __init__(self, director, schedule=None): - super(RunDecks, self).__init__(schedule) + def __init__(self, director, deck_id, schedule): + self.deck_id = deck_id self.director = director + identifier = 'run-deck-' + deck_id + super(RunDeck, self).__init__(schedule, identifier) @defer.inlineCallbacks def task(self): - for deck_id, deck in deck_store.list_enabled(): - yield deck.setup() - yield deck.run(self.director) + deck = deck_store.get(self.deck_id) + yield deck.setup() + yield deck.run(self.director) class SendHeartBeat(ScheduledTask): """ @@ -215,7 +217,10 @@ class SchedulerService(service.MultiService): self.schedule(UpdateInputsAndResources()) self.schedule(UploadReports()) self.schedule(DeleteOldReports()) - self.schedule(RunDecks(self.director)) + for deck_id, deck in deck_store.list_enabled(): + if deck.schedule is None: + continue + self.schedule(RunDeck(self.director, deck_id, deck.schedule)) self._looping_call.start(self.interval) diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py index f5d5b59..430252a 100644 --- a/ooni/scripts/ooniprobe.py +++ b/ooni/scripts/ooniprobe.py @@ -6,12 +6,14 @@ from twisted.internet import task, defer def ooniprobe(reactor): from ooni.ui.cli import runWithDaemonDirector, runWithDirector - from ooni.ui.cli import setupGlobalOptions + from ooni.ui.cli import setupGlobalOptions, initializeOoniprobe global_options = setupGlobalOptions(logging=True, start_tor=True, check_incoherences=True) if global_options['queue']: return runWithDaemonDirector(global_options) + elif global_options['initialize']: + return initializeOoniprobe(global_options) elif global_options['web-ui']: from ooni.scripts.ooniprobe_agent import WEB_UI_URL from ooni.scripts.ooniprobe_agent import status_agent, start_agent diff --git a/ooni/settings.py b/ooni/settings.py index 2161560..8bb3340 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -13,6 +13,125 @@ from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol from ooni.utils import Storage, log, get_ooni_root from ooni import errors + +CONFIG_FILE_TEMPLATE = """\ +# This is the configuration file for OONIProbe +# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html +# Keep in mind that indentation matters. + +basic: + # Where OONIProbe should be writing it's log file + logfile: {logfile} + loglevel: WARNING +privacy: + # Should we include the IP address of the probe in the report? + includeip: {include_ip} + # Should we include the ASN of the probe in the report? + includeasn: {include_asn} + # Should we include the country as reported by GeoIP in the report? + includecountry: {include_country} + # Should we collect a full packet capture on the client? + #includepcap: false +reports: + # Should we place a unique ID inside of every report + #unique_id: true + # This is a prefix for each packet capture file (.pcap) per test: + #pcap: null + #collector: null + # Should we be uploading reports to the collector by default? + upload: {should_upload} +advanced: + #debug: false + # enable if auto detection fails + #tor_binary: /usr/sbin/tor + #obfsproxy_binary: /usr/bin/obfsproxy + # For auto detection + # interface: auto + # Of specify a specific interface + # interface: wlan0 + # If you do not specify start_tor, you will have to have Tor running and + # explicitly set the control port and SOCKS port + #start_tor: true + # After how many seconds we should give up on a particular measurement + #measurement_timeout: 120 + # After how many retries we should give up on a measurement + #measurement_retries: 2 + # How many measurements to perform concurrently + #measurement_concurrency: 4 + # After how may seconds we should give up reporting + #reporting_timeout: 360 + # After how many retries to give up on reporting + #reporting_retries: 5 + # How many reports to perform concurrently + #reporting_concurrency: 7 + # If we should support communicating to plaintext backends (via HTTP) + # insecure_backend: false + # The preferred backend type, can be one of onion, https or cloudfront + preferred_backend: {preferred_backend} +tor: + #socks_port: 8801 + #control_port: 8802 + # Specify the absolute path to the Tor bridges to use for testing + #bridges: bridges.list + # Specify path of the tor datadirectory. + # This should be set to something to avoid having Tor download each time + # the descriptors and consensus data. + #data_dir: ~/.tor/ + # + # This is the timeout after which we consider to to not have + # bootstrapped properly. + #timeout: 200 + torrc: + #HTTPProxy: host:port + #HTTPProxyAuthenticator: user:password + #HTTPSProxy: host:port + #HTTPSProxyAuthenticator: user:password + #UseBridges: 1 + #Bridge: + #- "meek_lite 0.0.2.0:1 url=https://meek-reflect.appspot.com/ front=www.google.com" + #- "meek_lite 0.0.2.0:2 url=https://d2zfqthxsdq309.cloudfront.net/ front=a0.awsstatic.com" + #- "meek_lite 0.0.2.0:3 url=https://az786092.vo.msecnd.net/ front=ajax.aspnetcdn.com" + #ClientTransportPlugin: "meek_lite exec /usr/bin/obfs4proxy" +""" + +defaults = { + "basic": { + "loglevel": "WARNING", + "logfile": "ooniprobe.log" + }, + "privacy": { + "includeip": False, + "includeasn": True, + "includecountry": True, + "includepcap": False + }, + "reports": { + "unique_id": True, + "pcap": None, + "collector": None, + "upload": True + }, + "advanced": { + "debug": False, + "tor_binary": None, + "obfsproxy_binary": None, + "interface": "auto", + "start_tor": True, + "measurement_timeout": 120, + "measurement_retries": 2, + "measurement_concurrency": 4, + "reporting_timeout": 360, + "reporting_retries": 5, + "reporting_concurrency": 7, + "insecure_backend": False, + "preferred_backend": "onion" + }, + "tor": { + "timeout": 200, + "torrc": {} + } +} + class OConfig(object): _custom_home = None @@ -39,6 +158,17 @@ class OConfig(object): return settings.get(category, option) return None + def is_initialized(self): + # When this is false it means that the user has not gone + # through the steps of acquiring informed consent and + # initializing this ooniprobe installation. + initialized_path = os.path.join(self.running_path, 'initialized') + return os.path.exists(initialized_path) + + def set_initialized(self): + initialized_path = os.path.join(self.running_path, 'initialized') + with open(initialized_path, 'w+'): pass + @property def var_lib_path(self): if hasattr(sys, 'real_prefix'): @@ -149,7 +279,8 @@ class OConfig(object): config_file = self.global_options['configfile'] self.config_file = expanduser(config_file) else: - self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf') + self.config_file = os.path.join(self.running_path, + 'ooniprobe.conf') if 'logfile' in self.basic: self.basic.logfile = expanduser( @@ -183,6 +314,33 @@ class OConfig(object): if exc.errno != 17: raise + def create_config_file(self, include_ip=False, include_asn=True, + include_country=True, should_upload=True, + preferred_backend="onion"): + def _bool_to_yaml(value): + if value is True: + return 'true' + elif value is False: + return 'false' + else: + return 'null' + # Convert the boolean value to their YAML string representation + include_ip = _bool_to_yaml(include_ip ) + include_asn = _bool_to_yaml(include_asn) + include_country = _bool_to_yaml(include_country) + should_upload = _bool_to_yaml(should_upload) + + logfile = os.path.join(self.running_path, 'ooniprobe.log') + with open(self.config_file, 'w+') as out_file: + out_file.write( + CONFIG_FILE_TEMPLATE.format(logfile=logfile, + include_ip=include_ip, + include_asn=include_asn, + include_country=include_country, + should_upload=should_upload, + preferred_backend=preferred_backend) + ) + self.read_config_file() def _create_config_file(self): target_config_file = self.config_file @@ -200,19 +358,24 @@ class OConfig(object): w.write(line) def read_config_file(self, check_incoherences=False): - if not os.path.isfile(self.config_file): - print "Configuration file does not exist." - self._create_config_file() - self.read_config_file() - - with open(self.config_file) as f: - config_file_contents = '\n'.join(f.readlines()) - configuration = yaml.safe_load(config_file_contents) - - for setting in configuration.keys(): - if setting in dir(self) and configuration[setting] is not None: - for k, v in configuration[setting].items(): - getattr(self, setting)[k] = v + #if not os.path.isfile(self.config_file): + # print "Configuration file does not exist." + # self._create_config_file() + # self.read_config_file() + + configuration = {} + if os.path.isfile(self.config_file): + with open(self.config_file) as f: + config_file_contents = '\n'.join(f.readlines()) + configuration = yaml.safe_load(config_file_contents) + + for category in defaults.keys(): + for k, v in defaults[category].items(): + try: + value = configuration.get(category, {})[k] + except KeyError: + value = v + getattr(self, category)[k] = value self.set_paths() if check_incoherences: diff --git a/ooni/tests/__init__.py b/ooni/tests/__init__.py index e7fd48b..b5dbab4 100644 --- a/ooni/tests/__init__.py +++ b/ooni/tests/__init__.py @@ -1,11 +1,4 @@ import socket -from ooni.settings import config - -config.initialize_ooni_home('ooni_home') -config.read_config_file() -config.logging = False -config.advanced.debug = False - def is_internet_connected(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 3eccf9a..8cd3358 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -30,7 +30,9 @@ class Options(usage.Options): ["list", "s", "List the currently installed ooniprobe " "nettests"], ["verbose", "v", "Show more verbose information"], - ["web-ui", "w", "Start the web UI"] + ["web-ui", "w", "Start the web UI"], + ["initialize", "z", "Initialize ooniprobe to begin running " + "it"], ] optParameters = [ @@ -178,10 +180,58 @@ def director_startup_other_failures(failure): log.err("An unhandled exception occurred while starting the director!") log.exception(failure) + +def initializeOoniprobe(global_options): + # XXX print here the informed consent documentation. + answer = raw_input('Should we upload measurements to a collector? (Y/n) ') + should_upload = True + if answer.lower().startswith("n"): + should_upload = False + + answer = raw_input('Should we include your IP in measurements? (y/N) ') + include_ip = False + if answer.lower().startswith("y"): + include_ip = True + + answer = raw_input('Should we include your ASN (your network) in ' + 'measurements? (Y/n) ') + include_asn = False + if answer.lower().startswith("n"): + include_asn = True + + answer = raw_input('Should we include your Country in ' + 'measurements? (Y/n) ') + include_country = False + if answer.lower().startswith("n"): + include_country = True + + answer = raw_input('How would you like reports to be uploaded? (onion, ' + 'https, cloudfronted) ') + + preferred_backend = 'onion' + if answer.lower().startswith("https"): + preferred_backend = 'https' + elif answer.lower().startswith("cloudfronted"): + preferred_backend = 'cloudfronted' + + config.create_config_file(include_ip=include_ip, + include_asn=include_asn, + include_country=include_country, + should_upload=should_upload, + preferred_backend=preferred_backend) + config.set_initialized() + def setupGlobalOptions(logging, start_tor, check_incoherences): global_options = parseOptions() config.global_options = global_options + + if not config.is_initialized(): + log.err("You first need to agree to the informed consent and setup " + "ooniprobe to run it.") + global_options['initialize'] = True + return + config.set_paths() config.initialize_ooni_home() try: diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index e363ba0..6a7c149 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?9d3ccb3bc67af5ed4453"></script></body> + <script type="text/javascript" src="app.bundle.js?9c4ed560c98eaf61a836"></script></body> </html> diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index a03daa4..ed2193e 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -71,6 +71,39 @@ def xsrf_protect(check=True): return deco +def _requires_value(value, attrs=[]): + + def deco(f): + + @wraps(f) + def wrapper(instance, request, *a, **kw): + for attr in attrs: + attr_value = getattr(instance, attr) + if attr_value is not value: + raise WebUIError(400, "{0} must be {1}".format(attr, + value)) + return f(instance, request, *a, **kw) + + return wrapper + + return deco + +def requires_true(attrs=[]): + """ + This decorator is used to require that a certain set of class attributes are + set to True. + Otherwise it will trigger a WebUIError. + """ + return _requires_value(True, attrs) + +def requires_false(attrs=[]): + """ + This decorator is used to require that a certain set of class attributes are + set to False. + Otherwise it will trigger a WebUIError. + """ + return _requires_value(False, attrs) + class LongPoller(object): def __init__(self, timeout, _reactor=reactor): @@ -128,6 +161,7 @@ class WebUIAPI(object): for _ in range(30)]) self._director_started = False + self._is_initialized = config.is_initialized() self.status_poller = LongPoller( self._long_polling_timeout, _reactor) @@ -139,6 +173,10 @@ class WebUIAPI(object): self.status_poller.start() self.director.subscribe(self.handle_director_event) + if self._is_initialized: + self.start_director() + + def start_director(self): d = self.director.start() d.addCallback(self.director_started) @@ -151,7 +189,8 @@ class WebUIAPI(object): "software_name": "ooniprobe", "asn": probe_ip.geodata['asn'], "country_code": probe_ip.geodata['countrycode'], - "director_started": self._director_started + "director_started": self._director_started, + "initialized": self._is_initialized } def handle_director_event(self, event): @@ -208,8 +247,36 @@ class WebUIAPI(object): d.addCallback(got_status_update) return d + @app.route('/api/initialize', methods=["POST"]) + @xsrf_protect(check=True) + @requires_false(attrs=['_is_initialized']) + def api_initialize(self, request): + try: + initial_configuration = json.load(request.content) + except ValueError: + raise WebUIError(400, 'Invalid JSON message recevied') + + required_keys = ['include_ip', 'include_asn', 'include_country', + 'should_upload', 'preferred_backend'] + options = {} + for required_key in required_keys: + try: + options[required_key] = initial_configuration[required_key] + except KeyError: + raise WebUIError(400, 'Missing required key {0}'.format( + required_key)) + config.create_config_file(**options) + config.set_initialized() + + self._is_initialized = True + + self.status_poller.notify() + self.start_director() + return self.render_json({"result": "ok"}, request) + @app.route('/api/deck/<string:deck_id>/start', methods=["POST"]) @xsrf_protect(check=True) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_deck_start(self, request, deck_id): try: deck = self.director.deck_store.get(deck_id) @@ -225,6 +292,7 @@ class WebUIAPI(object): @app.route('/api/deck', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_deck_list(self, request): deck_list = { 'available': {}, @@ -246,6 +314,7 @@ class WebUIAPI(object): @app.route('/api/deck/<string:deck_id>/enable', methods=["POST"]) @xsrf_protect(check=True) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_deck_enable(self, request, deck_id): try: self.director.deck_store.enable(deck_id) @@ -256,6 +325,7 @@ class WebUIAPI(object): @app.route('/api/deck/<string:deck_id>/disable', methods=["POST"]) @xsrf_protect(check=True) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_deck_disable(self, request, deck_id): try: self.director.deck_store.disable(deck_id) @@ -276,6 +346,7 @@ class WebUIAPI(object): @app.route('/api/nettest/<string:test_name>/start', methods=["POST"]) @xsrf_protect(check=True) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_nettest_start(self, request, test_name): try: _ = self.director.netTests[test_name] @@ -321,11 +392,13 @@ class WebUIAPI(object): @app.route('/api/nettest', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_nettest_list(self, request): return self.render_json(self.director.netTests, request) @app.route('/api/input', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_input_list(self, request): input_store_list = self.director.input_store.list() for key, value in input_store_list.items(): @@ -334,6 +407,7 @@ class WebUIAPI(object): @app.route('/api/input/<string:input_id>/content', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_input_content(self, request, input_id): content = self.director.input_store.getContent(input_id) request.setHeader('Content-Type', 'text/plain') @@ -342,6 +416,7 @@ class WebUIAPI(object): @app.route('/api/input/<string:input_id>', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_input_details(self, request, input_id): return self.render_json( self.director.input_store.get(input_id), request @@ -349,12 +424,14 @@ class WebUIAPI(object): @app.route('/api/measurement', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_measurement_list(self, request): measurements = list_measurements() return self.render_json({"measurements": measurements}, request) @app.route('/api/measurement/<string:measurement_id>', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_measurement_summary(self, request, measurement_id): try: measurement = get_measurement(measurement_id) @@ -373,6 +450,7 @@ class WebUIAPI(object): @app.route('/api/measurement/<string:measurement_id>', methods=["DELETE"]) @xsrf_protect(check=True) + @requires_true(attrs=['_is_initialized']) def api_measurement_delete(self, request, measurement_id): try: measurement = get_measurement(measurement_id) @@ -394,6 +472,7 @@ class WebUIAPI(object): @app.route('/api/measurement/<string:measurement_id>/keep', methods=["POST"]) @xsrf_protect(check=True) + @requires_true(attrs=['_is_initialized']) def api_measurement_keep(self, request, measurement_id): try: measurement_dir = self.measurement_path.child(measurement_id) @@ -409,6 +488,7 @@ class WebUIAPI(object): @app.route('/api/measurement/<string:measurement_id>/<int:idx>', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_measurement_view(self, request, measurement_id, idx): try: measurement_dir = self.measurement_path.child(measurement_id) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 35d419d..d672ca8 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -56,7 +56,6 @@ class Storage(dict): for (k, v) in value.items(): self[k] = v - def checkForRoot(): if os.getuid() != 0: raise errors.InsufficientPrivileges

1 0

[ooni-probe/master] ignoreExistingDirectory is a recent twisted thing.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 21e6b839918e88f8260a58b23614194ec1230247 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Aug 5 17:16:50 2016 +0200 ignoreExistingDirectory is a recent twisted thing. Wrap calls to makedirs with proper exception handling. --- ooni/deck/store.py | 13 +++++++++++-- ooni/resources.py | 13 +++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/ooni/deck/store.py b/ooni/deck/store.py index 2d24f29..295f817 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -1,5 +1,6 @@ import csv import json +import errno from copy import deepcopy from twisted.internet import defer @@ -76,8 +77,16 @@ class InputStore(object): self.path = FilePath(config.inputs_directory) self.resources = FilePath(config.resources_directory) - self.path.child("descriptors").makedirs(ignoreExistingDirectory=True) - self.path.child("data").makedirs(ignoreExistingDirectory=True) + try: + self.path.child("descriptors").makedirs() + except OSError as e: + if not e.errno == errno.EEXIST: + raise + try: + self.path.child("data").makedirs() + except OSError as e: + if not e.errno == errno.EEXIST: + raise yield self.update_url_lists(country_code) @defer.inlineCallbacks diff --git a/ooni/resources.py b/ooni/resources.py index 9615c53..f97616c 100644 --- a/ooni/resources.py +++ b/ooni/resources.py @@ -1,4 +1,5 @@ import json +import errno from twisted.python.filepath import FilePath from twisted.internet import defer @@ -107,7 +108,11 @@ def check_for_update(country_code=None): latest_version = yield get_latest_version() resources_dir = FilePath(config.resources_directory) - resources_dir.makedirs(ignoreExistingDirectory=True) + try: + resources_dir.makedirs() + except OSError as e: + if not e.errno == errno.EEXIST: + raise current_manifest = resources_dir.child("manifest.json") if current_manifest.exists(): @@ -149,7 +154,11 @@ def check_for_update(country_code=None): filename = filename[:-3] gzipped = True dst_file = resources_dir.child(pre_path).child(filename) - dst_file.parent().makedirs(ignoreExistingDirectory=True) + try: + dst_file.parent().makedirs() + except OSError as e: + if not e.errno == errno.EEXIST: + raise src_file = dst_file.temporarySibling() src_file.alwaysCreate = 0

1 0

[ooni-probe/master] Use errno instead of integer to detect if file already exists
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 1952ca0f1d55d8b78a1bed22c150bdfaf288b94e Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 1 17:30:45 2016 +0200 Use errno instead of integer to detect if file already exists --- ooni/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/settings.py b/ooni/settings.py index 065252b..a0ef0a5 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -1,6 +1,7 @@ import os import sys import yaml +import errno import getpass from ConfigParser import SafeConfigParser @@ -332,7 +333,7 @@ class OConfig(object): try: os.makedirs(path) except OSError as exc: - if exc.errno != 17: + if exc.errno != errno.EEXIST: raise def create_config_file(self, include_ip=False, include_asn=True,

1 0

[ooni-probe/master] Improvements to start, stop, status workflow of ooniprobe-agent
by art＠torproject.org 19 Sep '16

19 Sep '16

commit e992f35ca6175ec53ec9966ca17966d86c129e8b Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 1 18:34:48 2016 +0200 Improvements to start, stop, status workflow of ooniprobe-agent * Better support for running it across different users --- ooni/scripts/ooniprobe_agent.py | 64 +++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index ecfe7d8..f56f2df 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -2,6 +2,7 @@ from __future__ import print_function import os import time +import errno import signal from twisted.scripts import twistd @@ -55,7 +56,8 @@ def start_agent(options=None): # Since we are starting the logger below ourselves we make twistd log to # a null log observer - twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull'] + twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull', + '--umask', '022'] twistd_config = OoniprobeTwistdConfig() if options is not None: twistd_args.extend(options.twistd_args) @@ -68,9 +70,12 @@ def start_agent(options=None): "StartOoniprobeAgent": StartOoniprobeAgentPlugin() } - if status_agent() == 0: + try: + get_running_pidfile() print("Stop ooniprobe-agent before attempting to start it") return 1 + except NotRunning: + pass print("Starting ooniprobe agent.") WEB_UI_URL = "http://{0}:{1}".format( @@ -80,8 +85,16 @@ def start_agent(options=None): twistd.runApp(twistd_config) return 0 -def status_agent(): - running = False + +class NotRunning(RuntimeError): + pass + +def get_running_pidfile(): + """ + :return: This pid of the running ooniprobe-agent instance. + :raises: NotRunning if it's not running + """ + running_pidfile = None for pidfile in [config.system_pid_path, config.user_pid_path]: if not os.path.exists(pidfile): # Didn't find the pid_file @@ -90,36 +103,51 @@ def status_agent(): pid = int(pid) try: os.kill(pid, signal.SIG_DFL) - running = True - except OSError, oserr: - if oserr.errno == 3: + running_pidfile = pidfile + break + except OSError as ose: + if ose.errno == errno.ESRCH: # Found pid, but isn't running continue - if running is True: + elif ose.errno == errno.EPERM: + # The process is owned by root. We assume it's running + running_pidfile = pidfile + break + if running_pidfile is None: + raise NotRunning + return running_pidfile + +def status_agent(): + try: + get_running_pidfile() print("ooniprobe-agent is running") return 0 - print("ooniprobe-agent is NOT running") - return 1 + except NotRunning: + print("ooniprobe-agent is NOT running") + return 1 def stop_agent(): # This function is borrowed from tahoe - pidfile = os.path.join( - config.running_path, - 'twistd.pid' - ) - if not os.path.exists(pidfile): - print("It seems like ooniprobe-agent is not running") + try: + pidfile = get_running_pidfile() + except NotRunning: + print("ooniprobe-agent is NOT running. Nothing to do.") return 2 + pid = open(pidfile, "r").read() pid = int(pid) try: os.kill(pid, signal.SIGKILL) - except OSError, oserr: - if oserr.errno == 3: + except OSError as ose: + if ose.errno == errno.ESRCH: print("No process was running. Cleaning up.") # the process didn't exist, so wipe the pid file os.remove(pidfile) return 2 + elif ose.errno == errno.EPERM: + # The process is owned by root. We assume it's running + print("ooniprobe-agent is owned by root. We cannot stop it.") + return 3 else: raise try:

1 0

[ooni-probe/master] Fix unittests for director, reporter and oonicli
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 539c491e304a89ef2e57cb49aeeadbddab8d8baa Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Aug 5 17:45:47 2016 +0200 Fix unittests for director, reporter and oonicli --- ooni/director.py | 1 - ooni/templates/httpt.py | 4 ++-- ooni/tests/bases.py | 1 + ooni/tests/test_director.py | 5 +++-- ooni/tests/test_oonicli.py | 10 +++++++--- ooni/tests/test_reporter.py | 4 +++- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 304a14a..ab31bf5 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -373,7 +373,6 @@ class Director(object): Launches a Tor with :param: socks_port :param: control_port :param: tor_binary set in ooniprobe.conf """ - from txtorcon import TorConfig if self._tor_state == 'running': log.debug("Tor is already running") defer.returnValue(self._tor_state) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index cba8702..d1aa1dc 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -110,8 +110,8 @@ class HTTPTest(NetTestCase): self.agent = FixedRedirectAgent(self.agent) self.report['agent'] = 'redirect' except: - log.err("Warning! You are running an old version of twisted"\ - "(<= 10.1). I will not be able to follow redirects."\ + log.err("Warning! You are running an old version of twisted " + "(<= 10.1). I will not be able to follow redirects." "This may make the testing less precise.") if len(self.contentDecoders) > 0: diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py index 40e3b5e..5a7da5f 100644 --- a/ooni/tests/bases.py +++ b/ooni/tests/bases.py @@ -11,6 +11,7 @@ class ConfigTestCase(unittest.TestCase): self.ooni_home_dir = os.path.abspath("ooni_home") self.config = config self.config.initialize_ooni_home(self.ooni_home_dir) + config.is_initialized = lambda: True super(ConfigTestCase, self).setUp() def skipTest(self, reason): diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index 6638adb..e5ca084 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -72,10 +72,11 @@ class TestDirector(ConfigTestCase): def test_start_tor(self): @defer.inlineCallbacks def director_start_tor(): + self.config.advanced.start_tor = True director = Director() yield director.start_tor() - assert config.tor.socks_port == 4242 - assert config.tor.control_port == 4242 + self.assertEqual(config.tor.socks_port, 4242) + self.assertEqual(config.tor.control_port, 4242) return director_start_tor() diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 06a21ff..8ad8a36 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -66,8 +66,6 @@ class TestRunDirector(ConfigTestCase): if not is_internet_connected(): self.skipTest("You must be connected to the internet to run this test") - config.tor.socks_port = 9050 - config.tor.control_port = None self.filenames = ['example-input.txt'] with open('example-input.txt', 'w+') as f: f.write('http://torproject.org/\n') @@ -83,7 +81,7 @@ class TestRunDirector(ConfigTestCase): @defer.inlineCallbacks def run_helper(self, test_name, nettest_args, verify_function, ooni_args=()): - output_file = os.path.abspath('test_report.yamloo') + output_file = os.path.abspath('test_report.njson') self.filenames.append(output_file) oldargv = sys.argv sys.argv = [''] @@ -91,6 +89,12 @@ class TestRunDirector(ConfigTestCase): sys.argv.extend(['-n', '-o', output_file, test_name]) sys.argv.extend(nettest_args) global_options = setupGlobalOptions(False, False, False) + + config.tor.socks_port = 9050 + config.advanced.start_tor = False + config.tor.control_port = None + config.advanced.debug = True + yield runWithDirector(global_options, create_input_store=False) with open(output_file) as f: diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index cbfdaeb..f7a917d 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -3,6 +3,7 @@ import yaml import json import time import shutil +import tempfile from twisted.internet import defer from twisted.trial import unittest @@ -119,8 +120,8 @@ class TestOONIBReportLog(ConfigTestCase): def setUp(self): super(TestOONIBReportLog, self).setUp() - self.report_log = OONIBReportLog() self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy' + self.config.measurements_directory = tempfile.mkdtemp() self.measurement_dir = os.path.join( self.config.measurements_directory, self.measurement_id @@ -128,6 +129,7 @@ class TestOONIBReportLog(ConfigTestCase): self.report_log_path = os.path.join(self.measurement_dir, 'report_log.json') os.mkdir(self.measurement_dir) + self.report_log = OONIBReportLog() def tearDown(self): shutil.rmtree(self.measurement_dir)

1 0

[ooni-probe/master] Check for all the possible pid file locations when checking status of ooniprobe-agent
by art＠torproject.org 19 Sep '16

19 Sep '16

commit bb7bcdd18d6a94097cf4de057c752828cfdf16c6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 1 17:29:23 2016 +0200 Check for all the possible pid file locations when checking status of ooniprobe-agent * This fixes https://github.com/TheTorProject/ooni-probe/issues/597 --- ooni/scripts/ooniprobe_agent.py | 41 ++++++++++++++++++++++++----------------- ooni/settings.py | 8 ++++++++ setup.py | 14 ++++++-------- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index 08fba89..ecfe7d8 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -67,6 +67,11 @@ def start_agent(options=None): twistd_config.loadedPlugins = { "StartOoniprobeAgent": StartOoniprobeAgentPlugin() } + + if status_agent() == 0: + print("Stop ooniprobe-agent before attempting to start it") + return 1 + print("Starting ooniprobe agent.") WEB_UI_URL = "http://{0}:{1}".format( config.advanced.webui_address, config.advanced.webui_port) @@ -76,23 +81,25 @@ def start_agent(options=None): return 0 def status_agent(): - pidfile = os.path.join( - config.running_path, - 'twistd.pid' - ) - if not os.path.exists(pidfile): - print("ooniprobe-agent is NOT running") - return 1 - pid = open(pidfile, "r").read() - pid = int(pid) - try: - os.kill(pid, signal.SIG_DFL) - except OSError, oserr: - if oserr.errno == 3: - print("ooniprobe-agent is NOT running") - return 1 - print("ooniprobe-agent is running") - return 0 + running = False + for pidfile in [config.system_pid_path, config.user_pid_path]: + if not os.path.exists(pidfile): + # Didn't find the pid_file + continue + pid = open(pidfile, "r").read() + pid = int(pid) + try: + os.kill(pid, signal.SIG_DFL) + running = True + except OSError, oserr: + if oserr.errno == 3: + # Found pid, but isn't running + continue + if running is True: + print("ooniprobe-agent is running") + return 0 + print("ooniprobe-agent is NOT running") + return 1 def stop_agent(): # This function is borrowed from tahoe diff --git a/ooni/settings.py b/ooni/settings.py index c36f3f7..065252b 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -235,6 +235,14 @@ class OConfig(object): return self.ooni_home @property + def user_pid_path(self): + return os.path.join(self.ooni_home, "twistd.pid") + + @property + def system_pid_path(self): + return os.path.join(VAR_LIB_PATH, "twistd.pid") + + @property def data_directory_candidates(self): dirs = [ self.ooni_home, diff --git a/setup.py b/setup.py index 7612b39..e5dd836 100644 --- a/setup.py +++ b/setup.py @@ -69,19 +69,17 @@ When you got them run: Using ooniprobe --------------- -To generate a test deck for your country, cd to the directory where you want it -and run: +It is recommended that you start the ooniprobe-agent system daemon that will +expose a localhost only Web UI and automatically run tests for you. -.. code:: bash - - oonideckgen +This can be done with: +.. code:: bash -To setup a daily cronjob run this: + ooniprobe-agent start -.. code:: bash - (crontab -l 2>/dev/null; echo "@daily ooniprobe `oonideckgen | grep -e '^ooniprobe'`") | crontab - +Then connect to the local web interface on http://127.0.0.1:8842/ Have fun! """

1 0

[ooni-probe/master] Add informed consent form to be used in the CLI version of ooniprobe
by art＠torproject.org 19 Sep '16

19 Sep '16

commit c23c47b4a314329b6bf9a778027b421801e55b82 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Aug 4 19:47:15 2016 +0200 Add informed consent form to be used in the CLI version of ooniprobe --- ooni/ui/consent-form.md | 265 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) diff --git a/ooni/ui/consent-form.md b/ooni/ui/consent-form.md new file mode 100644 index 0000000..08de2be --- /dev/null +++ b/ooni/ui/consent-form.md @@ -0,0 +1,265 @@ +The [Open Observatory of Network Interference +(OONI)](https://ooni.torproject.org/) is a free software project, under the [Tor +Project](https://www.torproject.org/), which collects and processes network +measurements with the aim of detecting network anomalies, such as censorship and +traffic manipulation. + +Running OONI may be against the terms of service of your ISP or legally +questionable in your country. By running OONI you will connect to web services +which may be banned, and use web censorship circumvention methods such as Tor. +The OONI project will publish data submitted by probes, possibly including your +IP address or other identifying information. In addition, your use of OONI will +be clear to anybody who has access to your computer, and to anybody who can +monitor your internet connection (such as your employer, ISP or government). + +By running ooniprobe, you are participating as a volunteer in this project. This +form includes information that you should be aware of and consent to *prior* to +running ooniprobe. + +## OONI software tests + +The OONI project has developed multiple free software tests which are designed to: + +* Detect the blocking of websites + +* Detect systems responsible for censorship and traffic manipulation + +* Evaluate the reachability of [Tor bridges](https://bridges.torproject.org/), + proxies, VPNs, and sensitive domains + +Below we provide brief descriptions of how these tests work. + +## Test descriptions + +The recommended set of tests that users run through the +`oonideckgen` command include the following: + +**Web connectivity:** This test examines whether websites are reachable and if +they are not, it attempts to determine whether access to them is blocked through +DNS tampering, TCP connection RST/IP blocking or by having a transparent HTTP +proxy. It does so by identifying the resolver of the user, performing a DNS +lookup, attempting to establish a TCP session and by sending HTTP GET requests +to the servers that are hosting tested websites. + +**HTTP invalid request line:** This test tries to detect the presence of network +components (“middle box”) which could be responsible for censorship and/or +traffic manipulation. Instead of sending a normal HTTP request, this test sends +an invalid HTTP request line - containing an invalid HTTP version number, an +invalid field count and a huge request method – to an echo service listening on +the standard HTTP port. If a middle box is present in the tested network, the +invalid HTTP request line will be intercepted by the middle box and this may +trigger error messages which can help identify the proxy technologies. + +**HTTP header field manipulation:** This test tries to detect the presence of +network components (“middle box”) which could be responsible for censorship +and/or traffic manipulation. It does so by sending HTTP requests which include +valid, but non-canonical HTTP headers to a backend control server which sends +back any data it receives. If we receive the HTTP headers exactly as we sent +them, then we assume that there is no “middle box” in the network. If, +however, such software is present in the network that we are testing, it will +likely normalize the invalid headers that we are sending or add extra headers. + +Another test which attempts to detect traffic manipulation includes **Multi- +protocol traceroute**, which constructs packets in such a way that they perform +a traceroute from multiple protocols and ports simultaneously. Other tests +include **Tor bridge reachability**, **Psiphon**, **Lantern**, **OpenVPN** and +**Meek fronted requests**, which examine whether these services work within a +tested network by attempting to connect to them in an automated way. + +Further test descriptions can be found here. + +## Risks + +Many countries have a lengthy history of subjecting digital rights activists to +various forms of abuse that could make it dangerous for individuals in these +countries to run OONI. The use of OONI might therefore subject users to severe +civil, criminal, or extra-judicial penalties, and such sanctions can potentially +include: + +* Imprisonment + +* Physical assaults + +* Large fines + +* Receiving threats + +* Being placed on government watch lists + +* Targeted for surveillance + +While most countries don't have laws which specifically prohibit the use of +network measurement software, it's important to note that the use of OONI can +*still* potentially be criminalized in certain countries under other, broader +laws if, for example, its use is viewed as an illegal or anti-government +activity. OONI users might also face the risk of being criminalized on the +grounds of *national security* if the data obtained and published by running +OONI is viewed as "jeopardizing" the country's external or internal security. In +extreme cases, any form of active network measurement could be illegal, or even +considered a form of espionage. + +We therefore strongly urge you to consult with lawyers *prior* to running +ooniprobe. You can also reach out to us with specific inquiries at +**legal(a)ooni.nu**. Please note though that we are *not* lawyers, but we might be +able to seek legal advice for you or to put you in touch with lawyers who could +address your questions and/or concerns. + +Some relevant resources include: + +* [Tor Legal FAQ](https://www.eff.org/torchallenge/faq.html) + +* [EFF Know Your Rights](https://www.eff.org/issues/know-your-rights) + +**Note:** The use of OONI is at your *own risk* in accordance to OONI's software +[license](https://github.com/TheTorProject/ooni- probe/blob/master/LICENSE) and +neither the OONI project nor its parent organization, the Tor Project, can be +held liable. + +**Installing ooniprobe** + +As with any other software, the usage of ooniprobe can leave traces. As such, +anybody with physical or remote access to your computer might be able to see +that you have downloaded, installed or run OONI. + +The installation of [Tor](https://www.torproject.org/) software, which is +designed for online anonymity, is a *prerequisite* for using OONI as all +measurements are by default sent to OONI over Tor. Furthermore, one of the +recommended tests that users run through the `oonideckgen` command line (web +connectivity test) is designed to compare HTTP requests over the network of the +user and over the Tor network. Similarly, OONI's Psiphon, Lantern and OpenVPN +tests require the installation of circumvention software. + +We therefore encourage you to consult with a lawyer on the legality of anonymity +software (such as Tor, a VPN or a proxy) *prior* to installing ooniprobe. + +To remove traces of software usage, you can re-install your operating system or +wipe your computer and remove everything (operating system, programs and files) +from your hard drive. + +**Running ooniprobe** + +Third parties (such as your government, ISP and/or employer) monitoring your +internet activity will be able to see all web traffic generated by OONI, +including your IP address, and might be able to link it to you personally. + +Many countries employ sophisticated surveillance measures that allow governments +to track individuals' online activities – even if they are using a VPN or a +proxy server to protect their privacy. In such countries, governments might be +able to identify you as a OONI user regardless of what measures you take to +protect your online privacy. + +OONI's **[HTTP-invalid-request-line](https://github.com/TheTorProject/ooni- +spec/blob/master/test-specs/ts-007-http-invalid-request-line.md)** test (which +is included in oonideckgen) probably presents the *highest risk* as its use +*might* trigger the suspicion of your ISP (and possibly, of your government), +the operators of network components affected by out-of-spec messages might view +them as attacks and this could potentially lead to prosecution under **computer +misuse laws** (or other laws). + +**Testing URLs for censorship** + +When running either oonideckgen (OONI's software package) or OONI's **web +connectivity** test, you will connect to and download data from various websites +which are included in the following two lists: + +* **Country-specific test list:** + https://github.com/citizenlab/test-lists/tree/master/lists + (search for your country's test list based on its country code) + +* **Global test list:** + https://github.com/citizenlab/test-lists/blob/master/lists/global.csv + (including a list of globally accessed websites) + +Many websites included in the above lists will likely be controversial and can +include pornography or hate speech, which might be illegal to access in your +country. We therefore recommend that you examine carefully whether you are +willing to take the risk of accessing and downloading data from such websites +through OONI tests, especially if this could potentially lead to various forms +of retribution. + +If you are uncertain of the potential implications of connecting to and +downloading data from the websites listed in the above lists, you can pass your +*own* test list with the ooniprobe `-f` command line option. + +**Publication of measurements** + +The public (including third parties who view the usage of OONI as illegal or +"suspicious") will be able to see the information collected by OONI once it's +published through: + +* [OONI Explorer](https://explorer.ooni.torproject.org/world/) + +* [OONI's list of measurements](https://measurements.ooni.torproject.org/) + +Unless users **[opt-out](https://github.com/TheTorProject/ooni-spec/blob/master +/informed-consent/data-policy.md#opt-out)**, all measurements that are generated +through OONI tests are by default sent to OONI's measurement collector and +automatically published through the above resources. + +Published data will include your approximate location, the network (ASN) you are +connecting from, and when you ran ooniprobe. Other identifying information, such +as your IP address, is *not* deliberately collected, but might be included in +HTTP headers or other metadata. The full page content downloaded by OONI could +potentially include further information if, for example, a website includes +tracking codes or custom content based on your network location. Such +information could potentially aid third parties in detecting you as an ooniprobe +user. + +## Choices + +We provide you with choices in regards to which tests to run, which data +you would like to be collected and whether you would like to send your +measurements to our collector for publication or not, as outlined below. + +**Tests** + +You can *opt-out* from running all of the tests included in `oonideckgen` by +specifying the test(s) that you want to run and by running it/them manually. You +can view how to run each OONI test through the ooniprobe `-s` command line +option. + +You can run each test included in `oonideckgen` separately through the following: + +* **Web connectivity test:** `ooniprobe blocking/web_connectivity` + +* **HTTP header field manipulation test:** `ooniprobe + manipulation/http_header_field_manipulation` + +* **HTTP invalid request line test:** `ooniprobe + manipulation/http_invalid_request_line` + +**Data collection and publication** + +OONI software users can *opt-out* from sending OONI's measurement collector +specific types of data by [editing the ooniprobe +configuration](https://github.com/TheTorProject/ooni-probe#configuring- +ooniprobe) file inside of `~/.ooni/ooniprobe.conf`. Through this file, users +can opt-out from sending OONI the following types of information: + +* Country code + +* Autonomous System Number (ASN) + +By default, OONI does *not* collect users' IP addresses, but users can choose to +*opt-in* (to provide more accurate information) through the above configuration +file. + +Users can also choose to *opt-out* from sending OONI's measurement collector any +data at all, by running ooniprobe with the `-n` command line option. This option +is quite often chosen by users who prefer to *not* have their measurements +published, due to potential risks that could emerge as a result of such +publication. + +Learn more about how we handle data through our Data Policy. + +## Consent + +My consent means the following: + +I understand the requirements and the risks of running ooniprobe. + +I understand that, unless I opt-out (as explained in the previous section), the +results of the tests that I run will by default be sent to the OONI project and +published by it. + +PRESS q to leave this page

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits September 2016