commit fe64930f836dccf76a87b9e9bcfe655bada49f90 Author: Arturo Filastò arturo@filasto.net Date: Fri Jul 29 16:47:32 2016 +0200

Add commands for managing the lifecycle of the ooniprobe-agent

* Add command to start the web-ui from ooniprobe * Improvements to the measurements API * Add klein dependency --- ooni/agent/agent.py | 4 +- ooni/measurements.py | 93 ++++++++++++++++++++++------ ooni/reporter.py | 13 +++- ooni/scripts/ooniprobe.py | 15 ++++- ooni/scripts/ooniprobe_agent.py | 130 +++++++++++++++++++++++++++++++++++++++- ooni/ui/cli.py | 5 +- ooni/ui/web/client/index.html | 2 +- ooni/ui/web/server.py | 53 +++++----------- requirements.txt | 1 + 9 files changed, 247 insertions(+), 69 deletions(-)

diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py index f5322ed..ff9a2dd 100644 --- a/ooni/agent/agent.py +++ b/ooni/agent/agent.py @@ -7,7 +7,7 @@ from ooni.ui.web.web import WebUIService from ooni.agent.scheduler import SchedulerService

class AgentService(service.MultiService): - def __init__(self): + def __init__(self, web_ui_port): service.MultiService.__init__(self)

director = Director() @@ -15,7 +15,7 @@ class AgentService(service.MultiService): config.initialize_ooni_home() config.read_config_file()

- self.web_ui_service = WebUIService(director) + self.web_ui_service = WebUIService(director, web_ui_port) self.web_ui_service.setServiceParent(self)

self.scheduler_service = SchedulerService(director) diff --git a/ooni/measurements.py b/ooni/measurements.py index f830a99..50efd87 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -1,10 +1,14 @@ +import os import json +import signal + from twisted.python.filepath import FilePath from ooni.settings import config

class Process(): supported_tests = [ - "web_connectivity" + "web_connectivity", + "http_requests" ] @staticmethod def web_connectivity(entry): @@ -15,6 +19,22 @@ class Process(): result['url'] = entry['input'] return result

+ @staticmethod + def http_requests(entry): + result = {} + test_keys = entry['test_keys'] + anomaly = ( + test_keys['body_length_match'] and + test_keys['headers_match'] and + ( + test_keys['control_failure'] != + test_keys['experiment_failure'] + ) + ) + result['anomaly'] = anomaly + result['url'] = entry['input'] + return result + def generate_summary(input_file, output_file): results = {} with open(input_file) as in_file: @@ -34,29 +54,64 @@ def generate_summary(input_file, output_file): with open(output_file, "w") as fw: json.dump(results, fw)

+class MeasurementNotFound(Exception): + pass + +def get_measurement(measurement_id): + measurement_path = FilePath(config.measurements_directory) + measurement = measurement_path.child(measurement_id) + if not measurement.exists(): + raise MeasurementNotFound + + running = False + completed = True + keep = False + if measurement.child("measurement.njson.progress").exists(): + completed = False + # XXX this is done quite often around the code, probably should + # be moved into some utility function. + pid = measurement.child("running.pid").open("r").read() + pid = int(pid) + try: + os.kill(pid, signal.SIG_DFL) + running = True + except OSError: + pass + + if measurement.child("keep").exists(): + keep = True + test_start_time, country_code, asn, test_name = \ + measurement_id.split("-")[:4] + return { + "test_name": test_name, + "country_code": country_code, + "asn": asn, + "test_start_time": test_start_time, + "id": measurement_id, + "completed": completed, + "keep": keep, + "running": running + } + + +def get_summary(measurement_id): + measurement_path = FilePath(config.measurements_directory) + measurement = measurement_path.child(measurement_id) + summary = measurement.child("summary.json") + if not summary.exists(): + generate_summary( + measurement.child("measurement.njson").path, + summary.path + ) + + with summary.open("r") as f: + return json.load(f)

def list_measurements(): measurements = [] measurement_path = FilePath(config.measurements_directory) for measurement_id in measurement_path.listdir(): - measurement = measurement_path.child(measurement_id) - completed = True - keep = False - if measurement.child("measurement.njson.progress").exists(): - completed = False - if measurement.child("keep").exists(): - keep = True - test_start_time, country_code, asn, test_name = \ - measurement_id.split("-")[:4] - measurements.append({ - "test_name": test_name, - "country_code": country_code, - "asn": asn, - "test_start_time": test_start_time, - "id": measurement_id, - "completed": completed, - "keep": keep - }) + measurements.append(get_measurement(measurement_id)) return measurements

if __name__ == "__main__": diff --git a/ooni/reporter.py b/ooni/reporter.py index cf5341d..cfdce08 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -6,7 +6,6 @@ import os from copy import deepcopy

from datetime import datetime -from contextlib import contextmanager

from yaml.representer import SafeRepresenter from yaml.emitter import Emitter @@ -36,6 +35,7 @@ from ooni.utils import generate_filename from ooni.settings import config

from ooni.tasks import ReportEntry +from ooni.measurements import list_measurements

def createPacketReport(packet_list): @@ -413,9 +413,10 @@ class OONIBReportLog(object): @defer.inlineCallbacks def get_report_log_entries(self): entries = [] - for measurement_id in self.measurement_dir.listdir(): + for measurement in list_measurements(): try: - entry = yield self.get_report_log(measurement_id) + entry = yield self.get_report_log(measurement['id']) + entry['completed'] = measurement['completed'] entries.append(entry) except NoReportLog: continue @@ -453,6 +454,9 @@ class OONIBReportLog(object): incomplete_reports = [] all_entries = yield self.get_report_log_entries() for entry in all_entries[:]: + # This means that the measurement itself is incomplete + if entry['completed'] is False: + continue if entry['status'] in ('created',): try: os.kill(entry['pid'], 0) @@ -486,6 +490,9 @@ class OONIBReportLog(object): to_upload_reports = [] all_entries = yield self.get_report_log_entries() for entry in all_entries[:]: + # This means that the measurement itself is incomplete + if entry['completed'] is False: + continue if entry['status'] in ('creation-failed', 'not-created'): to_upload_reports.append( (entry['measurements_path'], entry) diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py index 24493da..f5d5b59 100644 --- a/ooni/scripts/ooniprobe.py +++ b/ooni/scripts/ooniprobe.py @@ -1,5 +1,8 @@ #!/usr/bin/env python -from twisted.internet import task +import webbrowser +from multiprocessing import Process + +from twisted.internet import task, defer

def ooniprobe(reactor): from ooni.ui.cli import runWithDaemonDirector, runWithDirector @@ -9,6 +12,16 @@ def ooniprobe(reactor): check_incoherences=True) if global_options['queue']: return runWithDaemonDirector(global_options) + elif global_options['web-ui']: + from ooni.scripts.ooniprobe_agent import WEB_UI_URL + from ooni.scripts.ooniprobe_agent import status_agent, start_agent + if status_agent() != 0: + p = Process(target=start_agent) + p.start() + p.join() + print("Started ooniprobe-agent") + webbrowser.open_new(WEB_UI_URL) + return defer.succeed(None) else: return runWithDirector(global_options)

diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index 7833308..3f8efc8 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -1,22 +1,58 @@ +from __future__ import print_function + +import os +import time +import signal + from twisted.scripts import twistd from twisted.python import usage

+from ooni.settings import config from ooni.agent.agent import AgentService

+WEB_UI_PORT = 8842 +WEB_UI_URL = "http://127.0.0.1:%7B0%7D%22.format(WEB_UI_PORT) + class StartOoniprobeAgentPlugin: tapname = "ooniprobe"

def makeService(self, so): - return AgentService() + return AgentService(WEB_UI_PORT)

class OoniprobeTwistdConfig(twistd.ServerOptions): subCommands = [ ("StartOoniprobeAgent", None, usage.Options, "ooniprobe agent") ]

-def run(): - twistd_args = ["--nodaemon"] +class StartOptions(usage.Options): + pass + +class StopOptions(usage.Options): + pass + +class StatusOptions(usage.Options): + pass + +class RunOptions(usage.Options): + pass + +class AgentOptions(usage.Options): + subCommands = [ + ['start', None, StartOptions, "Start the ooniprobe-agent in the " + "background"], + ['stop', None, StopOptions, "Stop the ooniprobe-agent"], + ['status', None, StatusOptions, "Show status of the ooniprobe-agent"], + ['run', None, RunOptions, "Run the ooniprobe-agent in the foreground"] + ] + def postOptions(self): + self.twistd_args = [] + +def start_agent(options=None): + os.chdir(config.ooni_home) + twistd_args = [] twistd_config = OoniprobeTwistdConfig() + if options is not None: + twistd_args.extend(options.twistd_args) twistd_args.append("StartOoniprobeAgent") try: twistd_config.parseOptions(twistd_args) @@ -25,8 +61,96 @@ def run(): twistd_config.loadedPlugins = { "StartOoniprobeAgent": StartOoniprobeAgentPlugin() } + print("Starting ooniprobe agent.") + print("To view the GUI go to %s" % WEB_UI_URL) twistd.runApp(twistd_config) return 0

+def status_agent(): + pidfile = os.path.join( + config.ooni_home, + 'twistd.pid' + ) + if not os.path.exists(pidfile): + print("ooniprobe-agent is NOT running") + return 1 + pid = open(pidfile, "r").read() + pid = int(pid) + try: + os.kill(pid, signal.SIG_DFL) + except OSError, oserr: + if oserr.errno == 3: + print("ooniprobe-agent is NOT running") + return 1 + print("ooniprobe-agent is running") + return 0 + +def stop_agent(): + # This function is borrowed from tahoe + pidfile = os.path.join( + config.ooni_home, + 'twistd.pid' + ) + if not os.path.exists(pidfile): + print("It seems like ooniprobe-agent is not running") + return 2 + pid = open(pidfile, "r").read() + pid = int(pid) + try: + os.kill(pid, signal.SIGKILL) + except OSError, oserr: + if oserr.errno == 3: + print("No process was running. Cleaning up.") + # the process didn't exist, so wipe the pid file + os.remove(pidfile) + return 2 + else: + raise + try: + os.remove(pidfile) + except EnvironmentError: + pass + start = time.time() + time.sleep(0.1) + wait = 40 + first_time = True + while True: + # poll once per second until we see the process is no longer running + try: + os.kill(pid, 0) + except OSError: + print("process %d is dead" % pid) + return + wait -= 1 + if wait < 0: + if first_time: + print("It looks like pid %d is still running " + "after %d seconds" % (pid, (time.time() - start))) + print("I will keep watching it until you interrupt me.") + wait = 10 + first_time = False + else: + print("pid %d still running after %d seconds" % \ + (pid, (time.time() - start))) + wait = 10 + time.sleep(1) + # we define rc=1 to mean "I think something is still running, sorry" + return 1 + +def run(): + options = AgentOptions() + options.parseOptions() + + if options.subCommand == "run": + options.twistd_args += ("--nodaemon",) + + if options.subCommand == "stop": + return stop_agent() + + if options.subCommand == "status": + return status_agent() + + return start_agent(options) + if __name__ == "__main__": run() diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 2b5d844..3eccf9a 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -29,7 +29,8 @@ class Options(usage.Options): ["no-geoip", "g", "Disable geoip lookup on start"], ["list", "s", "List the currently installed ooniprobe " "nettests"], - ["verbose", "v", "Show more verbose information"] + ["verbose", "v", "Show more verbose information"], + ["web-ui", "w", "Start the web UI"] ]

optParameters = [ @@ -96,7 +97,7 @@ This will tell you how to run ooniprobe :) sys.exit(0)

def parseArgs(self, *args): - if self['testdeck'] or self['list']: + if self['testdeck'] or self['list'] or self['web-ui']: return try: self['test_file'] = args[0] diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index 7ba33fb..ebed106 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?afba2f26969b4c8f00ec"></script></body> + <script type="text/javascript" src="app.bundle.js?2777836bc218e75c3be5"></script></body> </html> diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 9d98e62..e1a6398 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -20,7 +20,8 @@ from ooni.deck import NGDeck from ooni.settings import config from ooni.utils import log from ooni.director import DirectorEvent -from ooni.measurements import generate_summary +from ooni.measurements import get_summary, get_measurement +from ooni.measurements import list_measurements, MeasurementNotFound from ooni.geoip import probe_ip

config.advanced.debug = True @@ -311,61 +312,40 @@ class WebUIAPI(object): @app.route('/api/measurement', methods=["GET"]) @xsrf_protect(check=False) def api_measurement_list(self, request): - measurements = [] - for measurement_id in self.measurement_path.listdir(): - measurement = self.measurement_path.child(measurement_id) - completed = True - if measurement.child("measurement.njson.progress").exists(): - completed = False - test_start_time, country_code, asn, test_name = \ - measurement_id.split("-")[:4] - measurements.append({ - "test_name": test_name, - "country_code": country_code, - "asn": asn, - "test_start_time": test_start_time, - "id": measurement_id, - "completed": completed - }) + measurements = list_measurements() return self.render_json({"measurements": measurements}, request)

@app.route('/api/measurement/string:measurement_id', methods=["GET"]) @xsrf_protect(check=False) def api_measurement_summary(self, request, measurement_id): try: - measurement_dir = self.measurement_path.child(measurement_id) + measurement = get_measurement(measurement_id) except InsecurePath: raise WebUIError(500, "invalid measurement id") + except MeasurementNotFound: + raise WebUIError(404, "measurement not found")

- if measurement_dir.child("measurements.njson.progress").exists(): - raise WebUIError(400, "measurement in progress") - - if not measurement_dir.child("summary.json").exists(): - # XXX we can perhaps remove this. - generate_summary( - measurement_dir.child("measurements.njson").path, - measurement_dir.child("summary.json").path - ) + if measurement['completed'] is False: raise WebUIError(400, "measurement in progress")

- summary = measurement_dir.child("summary.json") - with summary.open("r") as f: - r = json.load(f) - - return self.render_json(r, request) + summary = get_summary(measurement_id) + return self.render_json(summary, request)

@app.route('/api/measurement/string:measurement_id', methods=["DELETE"]) @xsrf_protect(check=True) def api_measurement_delete(self, request, measurement_id): try: - measurement_dir = self.measurement_path.child(measurement_id) + measurement = get_measurement(measurement_id) except InsecurePath: raise WebUIError(500, "invalid measurement id") + except MeasurementNotFound: + raise WebUIError(404, "measurement not found")

- if measurement_dir.child("measurements.njson.progress").exists(): - raise WebUIError(400, "measurement in progress") + if measurement['running'] is True: + raise WebUIError(400, "Measurement running")

try: + measurement_dir = self.measurement_path.child(measurement_id) measurement_dir.remove() except: raise WebUIError(400, "Failed to delete report") @@ -380,9 +360,6 @@ class WebUIAPI(object): except InsecurePath: raise WebUIError(500, "invalid measurement id")

- if measurement_dir.child("measurements.njson.progress").exists(): - raise WebUIError(400, "measurement in progress") - summary = measurement_dir.child("keep") with summary.open("w+") as f: pass diff --git a/requirements.txt b/requirements.txt index 410566e..c05ebcb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ service-identity pydumbnet zope.interface certifi +klein