commit 27a299c5c7d195860e63aa0b1d316f3255e4857d Author: Arturo Filastò arturo@filasto.net Date: Wed Aug 3 13:05:31 2016 +0200
Add support for initialization of ooniprobe --- data/decks/web.yaml | 1 + ooni/agent/scheduler.py | 25 +++--- ooni/scripts/ooniprobe.py | 4 +- ooni/settings.py | 191 ++++++++++++++++++++++++++++++++++++++---- ooni/tests/__init__.py | 7 -- ooni/ui/cli.py | 52 +++++++++++- ooni/ui/web/client/index.html | 2 +- ooni/ui/web/server.py | 82 +++++++++++++++++- ooni/utils/__init__.py | 1 - 9 files changed, 329 insertions(+), 36 deletions(-)
diff --git a/data/decks/web.yaml b/data/decks/web.yaml index a81b8f8..c7b9bdc 100644 --- a/data/decks/web.yaml +++ b/data/decks/web.yaml @@ -2,6 +2,7 @@ name: Web related ooniprobe tests description: This deck runs HTTP Header Field Manipulation, HTTP Invalid Request and the Web Connectivity test +schedule: "@daily" tasks: - name: Runs the HTTP Header Field Manipulation test ooni: diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index a6d689f..1f51bd4 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -22,9 +22,11 @@ class ScheduledTask(object): schedule = None identifier = None
- def __init__(self, schedule=None): + def __init__(self, schedule=None, identifier=None): if schedule is not None: self.schedule = schedule + if identifier is not None: + self.identifier = identifier
assert self.identifier is not None, "self.identifier must be set" assert self.schedule is not None, "self.schedule must be set" @@ -120,23 +122,23 @@ class DeleteOldReports(ScheduledTask): measurement_path.child(measurement['id']).remove()
-class RunDecks(ScheduledTask): +class RunDeck(ScheduledTask): """ This will run the decks that have been configured on the system as the decks to run by default. """ - schedule = '@daily' - identifier = 'run-decks'
- def __init__(self, director, schedule=None): - super(RunDecks, self).__init__(schedule) + def __init__(self, director, deck_id, schedule): + self.deck_id = deck_id self.director = director + identifier = 'run-deck-' + deck_id + super(RunDeck, self).__init__(schedule, identifier)
@defer.inlineCallbacks def task(self): - for deck_id, deck in deck_store.list_enabled(): - yield deck.setup() - yield deck.run(self.director) + deck = deck_store.get(self.deck_id) + yield deck.setup() + yield deck.run(self.director)
class SendHeartBeat(ScheduledTask): """ @@ -215,7 +217,10 @@ class SchedulerService(service.MultiService): self.schedule(UpdateInputsAndResources()) self.schedule(UploadReports()) self.schedule(DeleteOldReports()) - self.schedule(RunDecks(self.director)) + for deck_id, deck in deck_store.list_enabled(): + if deck.schedule is None: + continue + self.schedule(RunDeck(self.director, deck_id, deck.schedule))
self._looping_call.start(self.interval)
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py index f5d5b59..430252a 100644 --- a/ooni/scripts/ooniprobe.py +++ b/ooni/scripts/ooniprobe.py @@ -6,12 +6,14 @@ from twisted.internet import task, defer
def ooniprobe(reactor): from ooni.ui.cli import runWithDaemonDirector, runWithDirector - from ooni.ui.cli import setupGlobalOptions + from ooni.ui.cli import setupGlobalOptions, initializeOoniprobe
global_options = setupGlobalOptions(logging=True, start_tor=True, check_incoherences=True) if global_options['queue']: return runWithDaemonDirector(global_options) + elif global_options['initialize']: + return initializeOoniprobe(global_options) elif global_options['web-ui']: from ooni.scripts.ooniprobe_agent import WEB_UI_URL from ooni.scripts.ooniprobe_agent import status_agent, start_agent diff --git a/ooni/settings.py b/ooni/settings.py index 2161560..8bb3340 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -13,6 +13,125 @@ from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol from ooni.utils import Storage, log, get_ooni_root from ooni import errors
+ +CONFIG_FILE_TEMPLATE = """\ +# This is the configuration file for OONIProbe +# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html +# Keep in mind that indentation matters. + +basic: + # Where OONIProbe should be writing it's log file + logfile: {logfile} + loglevel: WARNING +privacy: + # Should we include the IP address of the probe in the report? + includeip: {include_ip} + # Should we include the ASN of the probe in the report? + includeasn: {include_asn} + # Should we include the country as reported by GeoIP in the report? + includecountry: {include_country} + # Should we collect a full packet capture on the client? + #includepcap: false +reports: + # Should we place a unique ID inside of every report + #unique_id: true + # This is a prefix for each packet capture file (.pcap) per test: + #pcap: null + #collector: null + # Should we be uploading reports to the collector by default? + upload: {should_upload} +advanced: + #debug: false + # enable if auto detection fails + #tor_binary: /usr/sbin/tor + #obfsproxy_binary: /usr/bin/obfsproxy + # For auto detection + # interface: auto + # Of specify a specific interface + # interface: wlan0 + # If you do not specify start_tor, you will have to have Tor running and + # explicitly set the control port and SOCKS port + #start_tor: true + # After how many seconds we should give up on a particular measurement + #measurement_timeout: 120 + # After how many retries we should give up on a measurement + #measurement_retries: 2 + # How many measurements to perform concurrently + #measurement_concurrency: 4 + # After how may seconds we should give up reporting + #reporting_timeout: 360 + # After how many retries to give up on reporting + #reporting_retries: 5 + # How many reports to perform concurrently + #reporting_concurrency: 7 + # If we should support communicating to plaintext backends (via HTTP) + # insecure_backend: false + # The preferred backend type, can be one of onion, https or cloudfront + preferred_backend: {preferred_backend} +tor: + #socks_port: 8801 + #control_port: 8802 + # Specify the absolute path to the Tor bridges to use for testing + #bridges: bridges.list + # Specify path of the tor datadirectory. + # This should be set to something to avoid having Tor download each time + # the descriptors and consensus data. + #data_dir: ~/.tor/ + # + # This is the timeout after which we consider to to not have + # bootstrapped properly. + #timeout: 200 + torrc: + #HTTPProxy: host:port + #HTTPProxyAuthenticator: user:password + #HTTPSProxy: host:port + #HTTPSProxyAuthenticator: user:password + #UseBridges: 1 + #Bridge: + #- "meek_lite 0.0.2.0:1 url=https://meek-reflect.appspot.com/ front=www.google.com" + #- "meek_lite 0.0.2.0:2 url=https://d2zfqthxsdq309.cloudfront.net/ front=a0.awsstatic.com" + #- "meek_lite 0.0.2.0:3 url=https://az786092.vo.msecnd.net/ front=ajax.aspnetcdn.com" + #ClientTransportPlugin: "meek_lite exec /usr/bin/obfs4proxy" +""" + +defaults = { + "basic": { + "loglevel": "WARNING", + "logfile": "ooniprobe.log" + }, + "privacy": { + "includeip": False, + "includeasn": True, + "includecountry": True, + "includepcap": False + }, + "reports": { + "unique_id": True, + "pcap": None, + "collector": None, + "upload": True + }, + "advanced": { + "debug": False, + "tor_binary": None, + "obfsproxy_binary": None, + "interface": "auto", + "start_tor": True, + "measurement_timeout": 120, + "measurement_retries": 2, + "measurement_concurrency": 4, + "reporting_timeout": 360, + "reporting_retries": 5, + "reporting_concurrency": 7, + "insecure_backend": False, + "preferred_backend": "onion" + }, + "tor": { + "timeout": 200, + "torrc": {} + } +} + class OConfig(object): _custom_home = None
@@ -39,6 +158,17 @@ class OConfig(object): return settings.get(category, option) return None
+ def is_initialized(self): + # When this is false it means that the user has not gone + # through the steps of acquiring informed consent and + # initializing this ooniprobe installation. + initialized_path = os.path.join(self.running_path, 'initialized') + return os.path.exists(initialized_path) + + def set_initialized(self): + initialized_path = os.path.join(self.running_path, 'initialized') + with open(initialized_path, 'w+'): pass + @property def var_lib_path(self): if hasattr(sys, 'real_prefix'): @@ -149,7 +279,8 @@ class OConfig(object): config_file = self.global_options['configfile'] self.config_file = expanduser(config_file) else: - self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf') + self.config_file = os.path.join(self.running_path, + 'ooniprobe.conf')
if 'logfile' in self.basic: self.basic.logfile = expanduser( @@ -183,6 +314,33 @@ class OConfig(object): if exc.errno != 17: raise
+ def create_config_file(self, include_ip=False, include_asn=True, + include_country=True, should_upload=True, + preferred_backend="onion"): + def _bool_to_yaml(value): + if value is True: + return 'true' + elif value is False: + return 'false' + else: + return 'null' + # Convert the boolean value to their YAML string representation + include_ip = _bool_to_yaml(include_ip ) + include_asn = _bool_to_yaml(include_asn) + include_country = _bool_to_yaml(include_country) + should_upload = _bool_to_yaml(should_upload) + + logfile = os.path.join(self.running_path, 'ooniprobe.log') + with open(self.config_file, 'w+') as out_file: + out_file.write( + CONFIG_FILE_TEMPLATE.format(logfile=logfile, + include_ip=include_ip, + include_asn=include_asn, + include_country=include_country, + should_upload=should_upload, + preferred_backend=preferred_backend) + ) + self.read_config_file()
def _create_config_file(self): target_config_file = self.config_file @@ -200,19 +358,24 @@ class OConfig(object): w.write(line)
def read_config_file(self, check_incoherences=False): - if not os.path.isfile(self.config_file): - print "Configuration file does not exist." - self._create_config_file() - self.read_config_file() - - with open(self.config_file) as f: - config_file_contents = '\n'.join(f.readlines()) - configuration = yaml.safe_load(config_file_contents) - - for setting in configuration.keys(): - if setting in dir(self) and configuration[setting] is not None: - for k, v in configuration[setting].items(): - getattr(self, setting)[k] = v + #if not os.path.isfile(self.config_file): + # print "Configuration file does not exist." + # self._create_config_file() + # self.read_config_file() + + configuration = {} + if os.path.isfile(self.config_file): + with open(self.config_file) as f: + config_file_contents = '\n'.join(f.readlines()) + configuration = yaml.safe_load(config_file_contents) + + for category in defaults.keys(): + for k, v in defaults[category].items(): + try: + value = configuration.get(category, {})[k] + except KeyError: + value = v + getattr(self, category)[k] = value
self.set_paths() if check_incoherences: diff --git a/ooni/tests/__init__.py b/ooni/tests/__init__.py index e7fd48b..b5dbab4 100644 --- a/ooni/tests/__init__.py +++ b/ooni/tests/__init__.py @@ -1,11 +1,4 @@ import socket -from ooni.settings import config - -config.initialize_ooni_home('ooni_home') -config.read_config_file() -config.logging = False -config.advanced.debug = False -
def is_internet_connected(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 3eccf9a..8cd3358 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -30,7 +30,9 @@ class Options(usage.Options): ["list", "s", "List the currently installed ooniprobe " "nettests"], ["verbose", "v", "Show more verbose information"], - ["web-ui", "w", "Start the web UI"] + ["web-ui", "w", "Start the web UI"], + ["initialize", "z", "Initialize ooniprobe to begin running " + "it"], ]
optParameters = [ @@ -178,10 +180,58 @@ def director_startup_other_failures(failure): log.err("An unhandled exception occurred while starting the director!") log.exception(failure)
+ +def initializeOoniprobe(global_options): + # XXX print here the informed consent documentation. + answer = raw_input('Should we upload measurements to a collector? (Y/n) ') + should_upload = True + if answer.lower().startswith("n"): + should_upload = False + + answer = raw_input('Should we include your IP in measurements? (y/N) ') + include_ip = False + if answer.lower().startswith("y"): + include_ip = True + + answer = raw_input('Should we include your ASN (your network) in ' + 'measurements? (Y/n) ') + include_asn = False + if answer.lower().startswith("n"): + include_asn = True + + answer = raw_input('Should we include your Country in ' + 'measurements? (Y/n) ') + include_country = False + if answer.lower().startswith("n"): + include_country = True + + answer = raw_input('How would you like reports to be uploaded? (onion, ' + 'https, cloudfronted) ') + + preferred_backend = 'onion' + if answer.lower().startswith("https"): + preferred_backend = 'https' + elif answer.lower().startswith("cloudfronted"): + preferred_backend = 'cloudfronted' + + config.create_config_file(include_ip=include_ip, + include_asn=include_asn, + include_country=include_country, + should_upload=should_upload, + preferred_backend=preferred_backend) + config.set_initialized() + def setupGlobalOptions(logging, start_tor, check_incoherences): global_options = parseOptions()
config.global_options = global_options + + if not config.is_initialized(): + log.err("You first need to agree to the informed consent and setup " + "ooniprobe to run it.") + global_options['initialize'] = True + return + config.set_paths() config.initialize_ooni_home() try: diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index e363ba0..6a7c149 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?9d3ccb3bc67af5ed4453"></script></body> + <script type="text/javascript" src="app.bundle.js?9c4ed560c98eaf61a836"></script></body> </html> diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index a03daa4..ed2193e 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -71,6 +71,39 @@ def xsrf_protect(check=True): return deco
+def _requires_value(value, attrs=[]): + + def deco(f): + + @wraps(f) + def wrapper(instance, request, *a, **kw): + for attr in attrs: + attr_value = getattr(instance, attr) + if attr_value is not value: + raise WebUIError(400, "{0} must be {1}".format(attr, + value)) + return f(instance, request, *a, **kw) + + return wrapper + + return deco + +def requires_true(attrs=[]): + """ + This decorator is used to require that a certain set of class attributes are + set to True. + Otherwise it will trigger a WebUIError. + """ + return _requires_value(True, attrs) + +def requires_false(attrs=[]): + """ + This decorator is used to require that a certain set of class attributes are + set to False. + Otherwise it will trigger a WebUIError. + """ + return _requires_value(False, attrs) +
class LongPoller(object): def __init__(self, timeout, _reactor=reactor): @@ -128,6 +161,7 @@ class WebUIAPI(object): for _ in range(30)])
self._director_started = False + self._is_initialized = config.is_initialized()
self.status_poller = LongPoller( self._long_polling_timeout, _reactor) @@ -139,6 +173,10 @@ class WebUIAPI(object): self.status_poller.start()
self.director.subscribe(self.handle_director_event) + if self._is_initialized: + self.start_director() + + def start_director(self): d = self.director.start()
d.addCallback(self.director_started) @@ -151,7 +189,8 @@ class WebUIAPI(object): "software_name": "ooniprobe", "asn": probe_ip.geodata['asn'], "country_code": probe_ip.geodata['countrycode'], - "director_started": self._director_started + "director_started": self._director_started, + "initialized": self._is_initialized }
def handle_director_event(self, event): @@ -208,8 +247,36 @@ class WebUIAPI(object): d.addCallback(got_status_update) return d
+ @app.route('/api/initialize', methods=["POST"]) + @xsrf_protect(check=True) + @requires_false(attrs=['_is_initialized']) + def api_initialize(self, request): + try: + initial_configuration = json.load(request.content) + except ValueError: + raise WebUIError(400, 'Invalid JSON message recevied') + + required_keys = ['include_ip', 'include_asn', 'include_country', + 'should_upload', 'preferred_backend'] + options = {} + for required_key in required_keys: + try: + options[required_key] = initial_configuration[required_key] + except KeyError: + raise WebUIError(400, 'Missing required key {0}'.format( + required_key)) + config.create_config_file(**options) + config.set_initialized() + + self._is_initialized = True + + self.status_poller.notify() + self.start_director() + return self.render_json({"result": "ok"}, request) + @app.route('/api/deck/string:deck_id/start', methods=["POST"]) @xsrf_protect(check=True) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_deck_start(self, request, deck_id): try: deck = self.director.deck_store.get(deck_id) @@ -225,6 +292,7 @@ class WebUIAPI(object):
@app.route('/api/deck', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_deck_list(self, request): deck_list = { 'available': {}, @@ -246,6 +314,7 @@ class WebUIAPI(object):
@app.route('/api/deck/string:deck_id/enable', methods=["POST"]) @xsrf_protect(check=True) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_deck_enable(self, request, deck_id): try: self.director.deck_store.enable(deck_id) @@ -256,6 +325,7 @@ class WebUIAPI(object):
@app.route('/api/deck/string:deck_id/disable', methods=["POST"]) @xsrf_protect(check=True) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_deck_disable(self, request, deck_id): try: self.director.deck_store.disable(deck_id) @@ -276,6 +346,7 @@ class WebUIAPI(object):
@app.route('/api/nettest/string:test_name/start', methods=["POST"]) @xsrf_protect(check=True) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_nettest_start(self, request, test_name): try: _ = self.director.netTests[test_name] @@ -321,11 +392,13 @@ class WebUIAPI(object):
@app.route('/api/nettest', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_director_started', '_is_initialized']) def api_nettest_list(self, request): return self.render_json(self.director.netTests, request)
@app.route('/api/input', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_input_list(self, request): input_store_list = self.director.input_store.list() for key, value in input_store_list.items(): @@ -334,6 +407,7 @@ class WebUIAPI(object):
@app.route('/api/input/string:input_id/content', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_input_content(self, request, input_id): content = self.director.input_store.getContent(input_id) request.setHeader('Content-Type', 'text/plain') @@ -342,6 +416,7 @@ class WebUIAPI(object):
@app.route('/api/input/string:input_id', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_input_details(self, request, input_id): return self.render_json( self.director.input_store.get(input_id), request @@ -349,12 +424,14 @@ class WebUIAPI(object):
@app.route('/api/measurement', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_measurement_list(self, request): measurements = list_measurements() return self.render_json({"measurements": measurements}, request)
@app.route('/api/measurement/string:measurement_id', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_measurement_summary(self, request, measurement_id): try: measurement = get_measurement(measurement_id) @@ -373,6 +450,7 @@ class WebUIAPI(object):
@app.route('/api/measurement/string:measurement_id', methods=["DELETE"]) @xsrf_protect(check=True) + @requires_true(attrs=['_is_initialized']) def api_measurement_delete(self, request, measurement_id): try: measurement = get_measurement(measurement_id) @@ -394,6 +472,7 @@ class WebUIAPI(object):
@app.route('/api/measurement/string:measurement_id/keep', methods=["POST"]) @xsrf_protect(check=True) + @requires_true(attrs=['_is_initialized']) def api_measurement_keep(self, request, measurement_id): try: measurement_dir = self.measurement_path.child(measurement_id) @@ -409,6 +488,7 @@ class WebUIAPI(object): @app.route('/api/measurement/string:measurement_id/int:idx', methods=["GET"]) @xsrf_protect(check=False) + @requires_true(attrs=['_is_initialized']) def api_measurement_view(self, request, measurement_id, idx): try: measurement_dir = self.measurement_path.child(measurement_id) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 35d419d..d672ca8 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -56,7 +56,6 @@ class Storage(dict): for (k, v) in value.items(): self[k] = v
- def checkForRoot(): if os.getuid() != 0: raise errors.InsufficientPrivileges