commit 1ec88b611d77048b8281d3358b20883388bd8283
Author: Arturo Filastò <arturo(a)filasto.net>
Date: Thu Jul 14 21:30:01 2016 +0200
Write ooniprobe reports in JSON format on disk
* Implement various API endpoints
---
ooni/measurements.py | 19 +++----
ooni/reporter.py | 116 ++++++++++++++++++++++++++++------------
ooni/settings.py | 3 +-
ooni/ui/web/server.py | 142 +++++++++++++++++++++++++++----------------------
ooni/ui/web/web.py | 7 ++-
ooni/utils/__init__.py | 6 ++-
6 files changed, 179 insertions(+), 114 deletions(-)
diff --git a/ooni/measurements.py b/ooni/measurements.py
index 5244ea4..976b125 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -9,26 +9,27 @@ class GenerateResults(object):
self.input_file = input_file
def process_web_connectivity(self, entry):
- anomaly = {}
- anomaly['result'] = False
+ result = {}
+ result['anomaly'] = False
if entry['test_keys']['blocking'] is not False:
- anomaly['result'] = True
- anomaly['url'] = entry['input']
- return anomaly
+ result['anomaly'] = True
+ result['url'] = entry['input']
+ return result
def output(self, output_file):
results = {}
with open(self.input_file) as in_file:
- for line in in_file:
+ for idx, line in enumerate(in_file):
entry = json.loads(line.strip())
if entry['test_name'] not in self.supported_tests:
raise Exception("Unsupported test")
- anomaly = getattr(self, 'process_'+entry['test_name'])(entry)
+ result = getattr(self, 'process_'+entry['test_name'])(entry)
+ result['idx'] = idx
results['test_name'] = entry['test_name']
results['country_code'] = entry['probe_cc']
results['asn'] = entry['probe_asn']
- results['anomalies'] = results.get('anomalies', [])
- results['anomalies'].append(anomaly)
+ results['results'] = results.get('results', [])
+ results['results'].append(result)
with open(output_file, "w") as fw:
json.dump(results, fw)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index f76fada..f07b3cf 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -1,5 +1,6 @@
import uuid
import yaml
+import json
import os
from copy import deepcopy
@@ -206,6 +207,56 @@ class YAMLReporter(OReporter):
def finish(self):
self._stream.close()
+class NJSONReporter(OReporter):
+
+ """
+ report_destination:
+ the destination directory of the report
+
+ """
+
+ def __init__(self, test_details, report_filename):
+ self.report_path = report_filename
+ OReporter.__init__(self, test_details)
+
+ def _writeln(self, line):
+ self._write("%s\n" % line)
+
+ def _write(self, data):
+ if not self._stream:
+ raise errors.ReportNotCreated
+ if self._stream.closed:
+ raise errors.ReportAlreadyClosed
+ s = str(data)
+ assert isinstance(s, type(''))
+ self._stream.write(s)
+ untilConcludes(self._stream.flush)
+
+ def writeReportEntry(self, entry):
+ if isinstance(entry, Measurement):
+ e = deepcopy(entry.testInstance.report)
+ elif isinstance(entry, dict):
+ e = deepcopy(entry)
+ else:
+ raise Exception("Failed to serialise entry")
+ report_entry = {
+ 'input': e.pop('input', None),
+ 'id': str(uuid.uuid4()),
+ 'test_start_time': e.pop('test_start_time', None),
+ 'measurement_start_time': e.pop('measurement_start_time', None),
+ 'test_runtime': e.pop('test_runtime', None),
+ 'test_keys': e
+ }
+ report_entry.update(self.testDetails)
+ self._write(json.dumps(report_entry))
+ self._write("\n")
+
+ def createReport(self):
+ self._stream = open(self.report_path, 'w+')
+
+ def finish(self):
+ self._stream.close()
+
class OONIBReporter(OReporter):
@@ -219,25 +270,20 @@ class OONIBReporter(OReporter):
def serializeEntry(self, entry, serialisation_format="yaml"):
if serialisation_format == "json":
if isinstance(entry, Measurement):
- report_entry = {
- 'input': entry.testInstance.report.pop('input', None),
- 'id': str(uuid.uuid4()),
- 'test_start_time': entry.testInstance.report.pop('test_start_time', None),
- 'measurement_start_time': entry.testInstance.report.pop('measurement_start_time', None),
- 'test_runtime': entry.testInstance.report.pop('test_runtime', None),
- 'test_keys': entry.testInstance.report
- }
+ e = deepcopy(entry.testInstance.report)
+
elif isinstance(entry, dict):
- report_entry = {
- 'input': entry.pop('input', None),
- 'id': str(uuid.uuid4()),
- 'test_start_time': entry.pop('test_start_time', None),
- 'measurement_start_time': entry.pop('measurement_start_time', None),
- 'test_runtime': entry.pop('test_runtime', None),
- 'test_keys': entry
- }
+ e = deepcopy(entry)
else:
raise Exception("Failed to serialise entry")
+ report_entry = {
+ 'input': e.pop('input', None),
+ 'id': str(uuid.uuid4()),
+ 'test_start_time': e.pop('test_start_time', None),
+ 'measurement_start_time': e.pop('measurement_start_time', None),
+ 'test_runtime': e.pop('test_runtime', None),
+ 'test_keys': e
+ }
report_entry.update(self.testDetails)
return report_entry
else:
@@ -468,7 +514,7 @@ class Report(object):
def __init__(self, test_details, report_filename,
reportEntryManager, collector_client=None,
- no_yamloo=False):
+ no_njson=False):
"""
This is an abstraction layer on top of all the configured reporters.
@@ -499,9 +545,9 @@ class Report(object):
self.report_log = OONIBReportLog()
- self.yaml_reporter = None
+ self.njson_reporter = None
self.oonib_reporter = None
- self.no_yamloo = no_yamloo
+ self.no_njson = no_njson
self.done = defer.Deferred()
self.reportEntryManager = reportEntryManager
@@ -509,7 +555,7 @@ class Report(object):
def generateReportFilename(self):
report_filename = generate_filename(self.test_details,
prefix='report',
- extension='yamloo')
+ extension='njson')
report_path = os.path.join('.', report_filename)
return os.path.abspath(report_path)
@@ -543,12 +589,12 @@ class Report(object):
self.collector_client)
self.test_details['report_id'] = yield self.open_oonib_reporter()
- if not self.no_yamloo:
- self.yaml_reporter = YAMLReporter(self.test_details,
- self.report_filename)
+ if not self.no_njson:
+ self.njson_reporter = NJSONReporter(self.test_details,
+ self.report_filename)
if not self.oonib_reporter:
yield self.report_log.not_created(self.report_filename)
- yield defer.maybeDeferred(self.yaml_reporter.createReport)
+ yield defer.maybeDeferred(self.njson_reporter.createReport)
defer.returnValue(self.reportId)
@@ -570,7 +616,7 @@ class Report(object):
d = defer.Deferred()
deferreds = []
- def yaml_report_failed(failure):
+ def njson_report_failed(failure):
d.errback(failure)
def oonib_report_failed(failure):
@@ -580,11 +626,11 @@ class Report(object):
if not d.called:
d.callback(None)
- if self.yaml_reporter:
- write_yaml_report = ReportEntry(self.yaml_reporter, measurement)
- self.reportEntryManager.schedule(write_yaml_report)
- write_yaml_report.done.addErrback(yaml_report_failed)
- deferreds.append(write_yaml_report.done)
+ if self.njson_reporter:
+ write_njson_report = ReportEntry(self.njson_reporter, measurement)
+ self.reportEntryManager.schedule(write_njson_report)
+ write_njson_report.done.addErrback(njson_report_failed)
+ deferreds.append(write_njson_report.done)
if self.oonib_reporter:
write_oonib_report = ReportEntry(self.oonib_reporter, measurement)
@@ -609,7 +655,7 @@ class Report(object):
d = defer.Deferred()
deferreds = []
- def yaml_report_failed(failure):
+ def njson_report_failed(failure):
d.errback(failure)
def oonib_report_closed(result):
@@ -623,10 +669,10 @@ class Report(object):
if not d.called:
d.callback(None)
- if self.yaml_reporter:
- close_yaml = defer.maybeDeferred(self.yaml_reporter.finish)
- close_yaml.addErrback(yaml_report_failed)
- deferreds.append(close_yaml)
+ if self.njson_reporter:
+ close_njson = defer.maybeDeferred(self.njson_reporter.finish)
+ close_njson.addErrback(njson_report_failed)
+ deferreds.append(close_njson)
if self.oonib_reporter:
close_oonib = self.oonib_reporter.finish()
diff --git a/ooni/settings.py b/ooni/settings.py
index ffbf68e..5245451 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -106,7 +106,8 @@ class OConfig(object):
else:
self.decks_directory = os.path.join(self.ooni_home, 'decks')
- self.reports_directory = os.path.join(self.ooni_home, 'reports')
+ self.measurements_directory = os.path.join(self.ooni_home,
+ 'measurements')
self.resources_directory = os.path.join(self.data_directory,
"resources")
if self.advanced.report_log_file:
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 5be581c..ccc5d87 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -3,34 +3,20 @@ from __future__ import print_function
import os
import json
+from twisted.internet import defer
from twisted.python import usage
from twisted.python.filepath import FilePath, InsecurePath
from twisted.web import static
from klein import Klein
+from werkzeug.exceptions import NotFound
-from ooni.settings import config
from ooni import errors
+from ooni.deck import Deck
+from ooni.settings import config
from ooni.nettest import NetTestLoader
from ooni.measurements import GenerateResults
-
-class RouteNotFound(Exception):
- def __init__(self, path, method):
- self._path = path
- self._method = method
-
- def __repr__(self):
- return "<RouteNotFound {0} {1}>".format(self._path,
- self._method)
-
-def _resolvePath(request):
- path = b''
- if request.postpath:
- path = b'/'.join(request.postpath)
-
- if not path.startswith(b'/'):
- path = b'/' + path
- return path
+from ooni.utils import generate_filename
def rpath(*path):
context = os.path.abspath(os.path.dirname(__file__))
@@ -48,6 +34,9 @@ def getNetTestLoader(test_options, test_file):
"""
options = []
for k, v in test_options.items():
+ if v is None:
+ print("Skipping %s because none" % k)
+ continue
options.append('--'+k)
options.append(v)
@@ -61,6 +50,11 @@ class WebUIAPI(object):
def __init__(self, config, director):
self.director = director
self.config = config
+ self.active_measurements = {}
+
+ @app.handle_errors(NotFound)
+ def not_found(self, request, _):
+ request.redirect('/client/')
def render_json(self, obj, request):
json_string = json.dumps(obj) + "\n"
@@ -68,28 +62,43 @@ class WebUIAPI(object):
request.setHeader('Content-Length', len(json_string))
return json_string
- @app.route('/api/decks/generate', methods=["GET"])
- def generate_decks(self, request):
+ @app.route('/api/deck/generate', methods=["GET"])
+ def api_deck_generate(self, request):
return self.render_json({"generate": "deck"}, request)
- @app.route('/api/decks/<string:deck_name>/start', methods=["POST"])
- def start_deck(self, request, deck_name):
+ @app.route('/api/deck/<string:deck_name>/start', methods=["POST"])
+ def api_deck_start(self, request, deck_name):
return self.render_json({"start": deck_name}, request)
- @app.route('/api/decks/<string:deck_name>/stop', methods=["POST"])
- def stop_deck(self, request, deck_name):
- return self.render_json({"stop": deck_name}, request)
-
- @app.route('/api/decks/<string:deck_name>', methods=["GET"])
- def deck_status(self, request, deck_name):
- return self.render_json({"status": deck_name}, request)
-
- @app.route('/api/decks', methods=["GET"])
- def deck_list(self, request):
+ @app.route('/api/deck', methods=["GET"])
+ def api_deck_list(self, request):
return self.render_json({"command": "deck-list"}, request)
- @app.route('/api/net-tests/<string:test_name>/start', methods=["POST"])
- def test_start(self, request, test_name):
+ @defer.inlineCallbacks
+ def run_deck(self, deck):
+ yield deck.setup()
+ measurement_ids = []
+ for net_test_loader in deck.netTestLoaders:
+ # XXX synchronize this with startNetTest
+ test_details = net_test_loader.getTestDetails()
+ measurement_id = generate_filename(test_details)
+
+ measurement_dir = os.path.join(
+ config.measurements_directory,
+ measurement_id
+ )
+ os.mkdir(measurement_dir)
+ report_filename = os.path.join(measurement_dir,
+ "measurements.njson")
+ measurement_ids.append(measurement_id)
+ self.active_measurements[measurement_id] = {
+ 'test_name': test_details['test_name'],
+ 'test_start_time': test_details['test_start_time']
+ }
+ self.director.startNetTest(net_test_loader, report_filename)
+
+ @app.route('/api/nettest/<string:test_name>/start', methods=["POST"])
+ def api_nettest_start(self, request, test_name):
try:
net_test = self.director.netTests[test_name]
except KeyError:
@@ -99,21 +108,19 @@ class WebUIAPI(object):
'error_message': 'Could not find the specified test'
}, request)
try:
- test_options = json.load(request.content.read())
+ test_options = json.load(request.content)
except ValueError:
return self.render_json({
'error_code': 500,
'error_message': 'Invalid JSON message recevied'
}, request)
+ deck = Deck(no_collector=True) # XXX remove no_collector
net_test_loader = getNetTestLoader(test_options, net_test['path'])
try:
- net_test_loader.checkOptions()
- # XXX we actually want to generate the report_filename in a smart
- # way so that we can know where it is located and learn the results
- # of the measurement.
- report_filename = None
- self.director.startNetTest(net_test_loader, report_filename)
+ deck.insert(net_test_loader)
+ self.run_deck(deck)
+
except errors.MissingRequiredOption, option_name:
request.setResponseCode(500)
return self.render_json({
@@ -134,25 +141,18 @@ class WebUIAPI(object):
'error_message': 'Insufficient priviledges'
}, request)
- return self.render_json({"deck": "list"}, request)
-
- @app.route('/api/net-tests/<string:test_name>/start', methods=["POST"])
- def test_stop(self, request, test_name):
- return self.render_json({
- "command": "test-stop",
- "test-name": test_name
- }, request)
-
- @app.route('/api/net-tests/<string:test_name>', methods=["GET"])
- def test_status(self, request, test_name):
- return self.render_json({"command": "test-stop"}, request)
+ return self.render_json({"status": "started"}, request)
- @app.route('/api/net-tests', methods=["GET"])
- def test_list(self, request):
+ @app.route('/api/nettest', methods=["GET"])
+ def api_nettest_list(self, request):
return self.render_json(self.director.netTests, request)
+ @app.route('/api/status', methods=["GET"])
+ def api_status(self):
+ return self.render_json()
+
@app.route('/api/measurement', methods=["GET"])
- def measurement_list(self, request):
+ def api_measurement_list(self, request):
measurement_ids = os.listdir(os.path.join(config.ooni_home,
"measurements"))
measurements = []
@@ -169,8 +169,8 @@ class WebUIAPI(object):
return self.render_json({"measurements": measurements}, request)
@app.route('/api/measurement/<string:measurement_id>', methods=["GET"])
- def measurement_summary(self, request, measurement_id):
- measurement_path = FilePath(config.ooni_home).child("measurements")
+ def api_measurement_summary(self, request, measurement_id):
+ measurement_path = FilePath(config.measurements_directory)
try:
measurement_dir = measurement_path.child(measurement_id)
except InsecurePath:
@@ -189,13 +189,29 @@ class WebUIAPI(object):
@app.route('/api/measurement/<string:measurement_id>/<int:idx>',
methods=["GET"])
- def measurement_open(self, request, measurement_id, idx):
- return self.render_json({"command": "results"}, request)
+ def api_measurement_view(self, request, measurement_id, idx):
+ measurement_path = FilePath(config.measurements_directory)
+ try:
+ measurement_dir = measurement_path.child(measurement_id)
+ except InsecurePath:
+ return self.render_json({"error": "invalid measurement id"})
+ measurements = measurement_dir.child("measurements.njson")
+
+ # XXX maybe implement some caching here
+ with measurements.open("r") as f:
+ r = None
+ for f_idx, line in enumerate(f):
+ if f_idx == idx:
+ r = json.loads(line)
+ break
+ if r is None:
+ return self.render_json({"error": "Could not find measurement "
+ "with this idx"}, request)
+ return self.render_json(r, request)
@app.route('/client/', branch=True)
def static(self, request):
- path = rpath("build")
- print(path)
+ path = rpath("client")
return static.File(path)
<<<<<<< acda284b56fa3a75acbe7d000fbdefb643839948
diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py
index f709c18..6c6971c 100644
--- a/ooni/ui/web/web.py
+++ b/ooni/ui/web/web.py
@@ -24,10 +24,9 @@ class WebUIService(service.MultiService):
root = server.Site(WebUIAPI(config, director).app.resource())
self._port = reactor.listenTCP(self.portNum, root)
director = Director()
- #d = director.start()
- #d.addCallback(_started)
- #d.addErrback(self._startupFailed)
- _started(None)
+ d = director.start()
+ d.addCallback(_started)
+ d.addErrback(self._startupFailed)
def _startupFailed(self, err):
log.err("Failed to start the director")
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index b28aadb..1aaac7b 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -97,18 +97,20 @@ def generate_filename(test_details, prefix=None, extension=None):
extension.
"""
LONG_DATE = "%Y-%m-%d %H:%M:%S"
- SHORT_DATE = "%Y-%m-%dT%H%M%SZ"
+ SHORT_DATE = "%Y%m%dT%H%M%SZ"
kwargs = {}
filename_format = ""
if prefix is not None:
kwargs["prefix"] = prefix
filename_format += "{prefix}-"
- filename_format += "{test_name}-{timestamp}"
+ filename_format += "{timestamp}-{probe_cc}-{probe_asn}-{test_name}"
if extension is not None:
kwargs["extension"] = extension
filename_format += ".{extension}"
kwargs['test_name'] = test_details['test_name']
+ kwargs['probe_cc'] = test_details['probe_cc']
+ kwargs['probe_asn'] = test_details['probe_asn']
kwargs['timestamp'] = datetime.strptime(test_details['test_start_time'],
LONG_DATE).strftime(SHORT_DATE)
return filename_format.format(**kwargs)
