[tor-commits] [ooni-probe/master] Add commands for managing the lifecycle of the ooniprobe-agent
art at torproject.org
art at torproject.org
Mon Sep 19 12:14:24 UTC 2016
commit fe64930f836dccf76a87b9e9bcfe655bada49f90
Author: Arturo Filastò <arturo at filasto.net>
Date: Fri Jul 29 16:47:32 2016 +0200
Add commands for managing the lifecycle of the ooniprobe-agent
* Add command to start the web-ui from ooniprobe
* Improvements to the measurements API
* Add klein dependency
---
ooni/agent/agent.py | 4 +-
ooni/measurements.py | 93 ++++++++++++++++++++++------
ooni/reporter.py | 13 +++-
ooni/scripts/ooniprobe.py | 15 ++++-
ooni/scripts/ooniprobe_agent.py | 130 +++++++++++++++++++++++++++++++++++++++-
ooni/ui/cli.py | 5 +-
ooni/ui/web/client/index.html | 2 +-
ooni/ui/web/server.py | 53 +++++-----------
requirements.txt | 1 +
9 files changed, 247 insertions(+), 69 deletions(-)
diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index f5322ed..ff9a2dd 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -7,7 +7,7 @@ from ooni.ui.web.web import WebUIService
from ooni.agent.scheduler import SchedulerService
class AgentService(service.MultiService):
- def __init__(self):
+ def __init__(self, web_ui_port):
service.MultiService.__init__(self)
director = Director()
@@ -15,7 +15,7 @@ class AgentService(service.MultiService):
config.initialize_ooni_home()
config.read_config_file()
- self.web_ui_service = WebUIService(director)
+ self.web_ui_service = WebUIService(director, web_ui_port)
self.web_ui_service.setServiceParent(self)
self.scheduler_service = SchedulerService(director)
diff --git a/ooni/measurements.py b/ooni/measurements.py
index f830a99..50efd87 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -1,10 +1,14 @@
+import os
import json
+import signal
+
from twisted.python.filepath import FilePath
from ooni.settings import config
class Process():
supported_tests = [
- "web_connectivity"
+ "web_connectivity",
+ "http_requests"
]
@staticmethod
def web_connectivity(entry):
@@ -15,6 +19,22 @@ class Process():
result['url'] = entry['input']
return result
+ @staticmethod
+ def http_requests(entry):
+ result = {}
+ test_keys = entry['test_keys']
+ anomaly = (
+ test_keys['body_length_match'] and
+ test_keys['headers_match'] and
+ (
+ test_keys['control_failure'] !=
+ test_keys['experiment_failure']
+ )
+ )
+ result['anomaly'] = anomaly
+ result['url'] = entry['input']
+ return result
+
def generate_summary(input_file, output_file):
results = {}
with open(input_file) as in_file:
@@ -34,29 +54,64 @@ def generate_summary(input_file, output_file):
with open(output_file, "w") as fw:
json.dump(results, fw)
+class MeasurementNotFound(Exception):
+ pass
+
+def get_measurement(measurement_id):
+ measurement_path = FilePath(config.measurements_directory)
+ measurement = measurement_path.child(measurement_id)
+ if not measurement.exists():
+ raise MeasurementNotFound
+
+ running = False
+ completed = True
+ keep = False
+ if measurement.child("measurement.njson.progress").exists():
+ completed = False
+ # XXX this is done quite often around the code, probably should
+ # be moved into some utility function.
+ pid = measurement.child("running.pid").open("r").read()
+ pid = int(pid)
+ try:
+ os.kill(pid, signal.SIG_DFL)
+ running = True
+ except OSError:
+ pass
+
+ if measurement.child("keep").exists():
+ keep = True
+ test_start_time, country_code, asn, test_name = \
+ measurement_id.split("-")[:4]
+ return {
+ "test_name": test_name,
+ "country_code": country_code,
+ "asn": asn,
+ "test_start_time": test_start_time,
+ "id": measurement_id,
+ "completed": completed,
+ "keep": keep,
+ "running": running
+ }
+
+
+def get_summary(measurement_id):
+ measurement_path = FilePath(config.measurements_directory)
+ measurement = measurement_path.child(measurement_id)
+ summary = measurement.child("summary.json")
+ if not summary.exists():
+ generate_summary(
+ measurement.child("measurement.njson").path,
+ summary.path
+ )
+
+ with summary.open("r") as f:
+ return json.load(f)
def list_measurements():
measurements = []
measurement_path = FilePath(config.measurements_directory)
for measurement_id in measurement_path.listdir():
- measurement = measurement_path.child(measurement_id)
- completed = True
- keep = False
- if measurement.child("measurement.njson.progress").exists():
- completed = False
- if measurement.child("keep").exists():
- keep = True
- test_start_time, country_code, asn, test_name = \
- measurement_id.split("-")[:4]
- measurements.append({
- "test_name": test_name,
- "country_code": country_code,
- "asn": asn,
- "test_start_time": test_start_time,
- "id": measurement_id,
- "completed": completed,
- "keep": keep
- })
+ measurements.append(get_measurement(measurement_id))
return measurements
if __name__ == "__main__":
diff --git a/ooni/reporter.py b/ooni/reporter.py
index cf5341d..cfdce08 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -6,7 +6,6 @@ import os
from copy import deepcopy
from datetime import datetime
-from contextlib import contextmanager
from yaml.representer import SafeRepresenter
from yaml.emitter import Emitter
@@ -36,6 +35,7 @@ from ooni.utils import generate_filename
from ooni.settings import config
from ooni.tasks import ReportEntry
+from ooni.measurements import list_measurements
def createPacketReport(packet_list):
@@ -413,9 +413,10 @@ class OONIBReportLog(object):
@defer.inlineCallbacks
def get_report_log_entries(self):
entries = []
- for measurement_id in self.measurement_dir.listdir():
+ for measurement in list_measurements():
try:
- entry = yield self.get_report_log(measurement_id)
+ entry = yield self.get_report_log(measurement['id'])
+ entry['completed'] = measurement['completed']
entries.append(entry)
except NoReportLog:
continue
@@ -453,6 +454,9 @@ class OONIBReportLog(object):
incomplete_reports = []
all_entries = yield self.get_report_log_entries()
for entry in all_entries[:]:
+ # This means that the measurement itself is incomplete
+ if entry['completed'] is False:
+ continue
if entry['status'] in ('created',):
try:
os.kill(entry['pid'], 0)
@@ -486,6 +490,9 @@ class OONIBReportLog(object):
to_upload_reports = []
all_entries = yield self.get_report_log_entries()
for entry in all_entries[:]:
+ # This means that the measurement itself is incomplete
+ if entry['completed'] is False:
+ continue
if entry['status'] in ('creation-failed', 'not-created'):
to_upload_reports.append(
(entry['measurements_path'], entry)
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
index 24493da..f5d5b59 100644
--- a/ooni/scripts/ooniprobe.py
+++ b/ooni/scripts/ooniprobe.py
@@ -1,5 +1,8 @@
#!/usr/bin/env python
-from twisted.internet import task
+import webbrowser
+from multiprocessing import Process
+
+from twisted.internet import task, defer
def ooniprobe(reactor):
from ooni.ui.cli import runWithDaemonDirector, runWithDirector
@@ -9,6 +12,16 @@ def ooniprobe(reactor):
check_incoherences=True)
if global_options['queue']:
return runWithDaemonDirector(global_options)
+ elif global_options['web-ui']:
+ from ooni.scripts.ooniprobe_agent import WEB_UI_URL
+ from ooni.scripts.ooniprobe_agent import status_agent, start_agent
+ if status_agent() != 0:
+ p = Process(target=start_agent)
+ p.start()
+ p.join()
+ print("Started ooniprobe-agent")
+ webbrowser.open_new(WEB_UI_URL)
+ return defer.succeed(None)
else:
return runWithDirector(global_options)
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 7833308..3f8efc8 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -1,22 +1,58 @@
+from __future__ import print_function
+
+import os
+import time
+import signal
+
from twisted.scripts import twistd
from twisted.python import usage
+from ooni.settings import config
from ooni.agent.agent import AgentService
+WEB_UI_PORT = 8842
+WEB_UI_URL = "http://127.0.0.1:{0}".format(WEB_UI_PORT)
+
class StartOoniprobeAgentPlugin:
tapname = "ooniprobe"
def makeService(self, so):
- return AgentService()
+ return AgentService(WEB_UI_PORT)
class OoniprobeTwistdConfig(twistd.ServerOptions):
subCommands = [
("StartOoniprobeAgent", None, usage.Options, "ooniprobe agent")
]
-def run():
- twistd_args = ["--nodaemon"]
+class StartOptions(usage.Options):
+ pass
+
+class StopOptions(usage.Options):
+ pass
+
+class StatusOptions(usage.Options):
+ pass
+
+class RunOptions(usage.Options):
+ pass
+
+class AgentOptions(usage.Options):
+ subCommands = [
+ ['start', None, StartOptions, "Start the ooniprobe-agent in the "
+ "background"],
+ ['stop', None, StopOptions, "Stop the ooniprobe-agent"],
+ ['status', None, StatusOptions, "Show status of the ooniprobe-agent"],
+ ['run', None, RunOptions, "Run the ooniprobe-agent in the foreground"]
+ ]
+ def postOptions(self):
+ self.twistd_args = []
+
+def start_agent(options=None):
+ os.chdir(config.ooni_home)
+ twistd_args = []
twistd_config = OoniprobeTwistdConfig()
+ if options is not None:
+ twistd_args.extend(options.twistd_args)
twistd_args.append("StartOoniprobeAgent")
try:
twistd_config.parseOptions(twistd_args)
@@ -25,8 +61,96 @@ def run():
twistd_config.loadedPlugins = {
"StartOoniprobeAgent": StartOoniprobeAgentPlugin()
}
+ print("Starting ooniprobe agent.")
+ print("To view the GUI go to %s" % WEB_UI_URL)
twistd.runApp(twistd_config)
return 0
+def status_agent():
+ pidfile = os.path.join(
+ config.ooni_home,
+ 'twistd.pid'
+ )
+ if not os.path.exists(pidfile):
+ print("ooniprobe-agent is NOT running")
+ return 1
+ pid = open(pidfile, "r").read()
+ pid = int(pid)
+ try:
+ os.kill(pid, signal.SIG_DFL)
+ except OSError, oserr:
+ if oserr.errno == 3:
+ print("ooniprobe-agent is NOT running")
+ return 1
+ print("ooniprobe-agent is running")
+ return 0
+
+def stop_agent():
+ # This function is borrowed from tahoe
+ pidfile = os.path.join(
+ config.ooni_home,
+ 'twistd.pid'
+ )
+ if not os.path.exists(pidfile):
+ print("It seems like ooniprobe-agent is not running")
+ return 2
+ pid = open(pidfile, "r").read()
+ pid = int(pid)
+ try:
+ os.kill(pid, signal.SIGKILL)
+ except OSError, oserr:
+ if oserr.errno == 3:
+ print("No process was running. Cleaning up.")
+ # the process didn't exist, so wipe the pid file
+ os.remove(pidfile)
+ return 2
+ else:
+ raise
+ try:
+ os.remove(pidfile)
+ except EnvironmentError:
+ pass
+ start = time.time()
+ time.sleep(0.1)
+ wait = 40
+ first_time = True
+ while True:
+ # poll once per second until we see the process is no longer running
+ try:
+ os.kill(pid, 0)
+ except OSError:
+ print("process %d is dead" % pid)
+ return
+ wait -= 1
+ if wait < 0:
+ if first_time:
+ print("It looks like pid %d is still running "
+ "after %d seconds" % (pid, (time.time() - start)))
+ print("I will keep watching it until you interrupt me.")
+ wait = 10
+ first_time = False
+ else:
+ print("pid %d still running after %d seconds" % \
+ (pid, (time.time() - start)))
+ wait = 10
+ time.sleep(1)
+ # we define rc=1 to mean "I think something is still running, sorry"
+ return 1
+
+def run():
+ options = AgentOptions()
+ options.parseOptions()
+
+ if options.subCommand == "run":
+ options.twistd_args += ("--nodaemon",)
+
+ if options.subCommand == "stop":
+ return stop_agent()
+
+ if options.subCommand == "status":
+ return status_agent()
+
+ return start_agent(options)
+
if __name__ == "__main__":
run()
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 2b5d844..3eccf9a 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -29,7 +29,8 @@ class Options(usage.Options):
["no-geoip", "g", "Disable geoip lookup on start"],
["list", "s", "List the currently installed ooniprobe "
"nettests"],
- ["verbose", "v", "Show more verbose information"]
+ ["verbose", "v", "Show more verbose information"],
+ ["web-ui", "w", "Start the web UI"]
]
optParameters = [
@@ -96,7 +97,7 @@ This will tell you how to run ooniprobe :)
sys.exit(0)
def parseArgs(self, *args):
- if self['testdeck'] or self['list']:
+ if self['testdeck'] or self['list'] or self['web-ui']:
return
try:
self['test_file'] = args[0]
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index 7ba33fb..ebed106 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
<app>
Loading...
</app>
- <script type="text/javascript" src="app.bundle.js?afba2f26969b4c8f00ec"></script></body>
+ <script type="text/javascript" src="app.bundle.js?2777836bc218e75c3be5"></script></body>
</html>
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 9d98e62..e1a6398 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -20,7 +20,8 @@ from ooni.deck import NGDeck
from ooni.settings import config
from ooni.utils import log
from ooni.director import DirectorEvent
-from ooni.measurements import generate_summary
+from ooni.measurements import get_summary, get_measurement
+from ooni.measurements import list_measurements, MeasurementNotFound
from ooni.geoip import probe_ip
config.advanced.debug = True
@@ -311,61 +312,40 @@ class WebUIAPI(object):
@app.route('/api/measurement', methods=["GET"])
@xsrf_protect(check=False)
def api_measurement_list(self, request):
- measurements = []
- for measurement_id in self.measurement_path.listdir():
- measurement = self.measurement_path.child(measurement_id)
- completed = True
- if measurement.child("measurement.njson.progress").exists():
- completed = False
- test_start_time, country_code, asn, test_name = \
- measurement_id.split("-")[:4]
- measurements.append({
- "test_name": test_name,
- "country_code": country_code,
- "asn": asn,
- "test_start_time": test_start_time,
- "id": measurement_id,
- "completed": completed
- })
+ measurements = list_measurements()
return self.render_json({"measurements": measurements}, request)
@app.route('/api/measurement/<string:measurement_id>', methods=["GET"])
@xsrf_protect(check=False)
def api_measurement_summary(self, request, measurement_id):
try:
- measurement_dir = self.measurement_path.child(measurement_id)
+ measurement = get_measurement(measurement_id)
except InsecurePath:
raise WebUIError(500, "invalid measurement id")
+ except MeasurementNotFound:
+ raise WebUIError(404, "measurement not found")
- if measurement_dir.child("measurements.njson.progress").exists():
- raise WebUIError(400, "measurement in progress")
-
- if not measurement_dir.child("summary.json").exists():
- # XXX we can perhaps remove this.
- generate_summary(
- measurement_dir.child("measurements.njson").path,
- measurement_dir.child("summary.json").path
- )
+ if measurement['completed'] is False:
raise WebUIError(400, "measurement in progress")
- summary = measurement_dir.child("summary.json")
- with summary.open("r") as f:
- r = json.load(f)
-
- return self.render_json(r, request)
+ summary = get_summary(measurement_id)
+ return self.render_json(summary, request)
@app.route('/api/measurement/<string:measurement_id>', methods=["DELETE"])
@xsrf_protect(check=True)
def api_measurement_delete(self, request, measurement_id):
try:
- measurement_dir = self.measurement_path.child(measurement_id)
+ measurement = get_measurement(measurement_id)
except InsecurePath:
raise WebUIError(500, "invalid measurement id")
+ except MeasurementNotFound:
+ raise WebUIError(404, "measurement not found")
- if measurement_dir.child("measurements.njson.progress").exists():
- raise WebUIError(400, "measurement in progress")
+ if measurement['running'] is True:
+ raise WebUIError(400, "Measurement running")
try:
+ measurement_dir = self.measurement_path.child(measurement_id)
measurement_dir.remove()
except:
raise WebUIError(400, "Failed to delete report")
@@ -380,9 +360,6 @@ class WebUIAPI(object):
except InsecurePath:
raise WebUIError(500, "invalid measurement id")
- if measurement_dir.child("measurements.njson.progress").exists():
- raise WebUIError(400, "measurement in progress")
-
summary = measurement_dir.child("keep")
with summary.open("w+") as f:
pass
diff --git a/requirements.txt b/requirements.txt
index 410566e..c05ebcb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,3 +17,4 @@ service-identity
pydumbnet
zope.interface
certifi
+klein
More information about the tor-commits
mailing list