commit 80dc4719da78527458a97ab33b97b1156e67f113 Author: Arturo Filastò <art@fuffa.org> Date: Wed Jun 25 13:18:05 2014 +0200 Add tool for viewing the status of report submission and allow users to upload reports. --- bin/oonireport | 26 +++++++++++++++++ ooni/report/__init__.py | 1 + ooni/report/cli.py | 57 ++++++++++++++++++++++++++++++++++++ ooni/report/parser.py | 35 +++++++++++++++++++++++ ooni/report/tool.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 192 insertions(+) diff --git a/bin/oonireport b/bin/oonireport new file mode 100755 index 0000000..4e6d48a --- /dev/null +++ b/bin/oonireport @@ -0,0 +1,26 @@ +#!/usr/bin/env python +import os +import sys + +sys.path[:] = map(os.path.abspath, sys.path) +sys.path.insert(0, os.path.abspath(os.getcwd())) + +from twisted.internet import defer, reactor + +from ooni.report import cli + +def failed(failure): + log.err("Failed to run ooni-report") + log.exception(failure) + reactor.stop() + +def done(result): + reactor.stop() + +def start(): + d = defer.maybeDeferred(cli.run) + d.addCallback(done) + d.addErrback(done) + +reactor.callWhenRunning(start) +reactor.run() diff --git a/ooni/report/__init__.py b/ooni/report/__init__.py new file mode 100644 index 0000000..a4e2017 --- /dev/null +++ b/ooni/report/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1" diff --git a/ooni/report/cli.py b/ooni/report/cli.py new file mode 100644 index 0000000..eee5c53 --- /dev/null +++ b/ooni/report/cli.py @@ -0,0 +1,57 @@ +import os +import sys + +from ooni.report import __version__ +from ooni.report import tool +from ooni.settings import config + +from twisted.python import usage + + +class Options(usage.Options): + + synopsis = """%s [options] upload | status +""" % (os.path.basename(sys.argv[0]),) + + optParameters = [ + ["collector", "c", None, + "Specify the collector to upload the result to."], + ["bouncer", "b", None, + "Specify the bouncer to query for a collector."] + ] + + def opt_version(self): + print "oonireport version:", __version__ + sys.exit(0) + + def parseArgs(self, *args): + self['command'] = args[0] + if self['command'] not in ("upload", "status"): + raise usage.UsageError( + "Must specify either command upload or status" + ) + if self['command'] == "upload": + try: + self['report_file'] = args[1] + except KeyError: + self['report_file'] = None + + +def parse_options(): + options = Options() + options.parseOptions() + return dict(options) + + +def run(): + config.read_config_file() + options = parse_options() + if options['command'] == "upload" and options['report_file']: + return tool.upload(options['report_file'], + options['collector'], + options['bouncer']) + elif options['command'] == "upload": + return tool.upload_all(options['collector'], + options['bouncer']) + elif options['command'] == "status": + return tool.status() diff --git a/ooni/report/parser.py b/ooni/report/parser.py new file mode 100644 index 0000000..a986522 --- /dev/null +++ b/ooni/report/parser.py @@ -0,0 +1,35 @@ +import yaml + + +class ReportLoader(object): + _header_keys = ( + 'probe_asn', + 'probe_cc', + 'probe_ip', + 'start_time', + 'test_name', + 'test_version', + 'options', + 'input_hashes', + 'software_name', + 'software_version' + ) + + def __init__(self, report_filename): + self._fp = open(report_filename) + self._yfp = yaml.safe_load_all(self._fp) + + self.header = self._yfp.next() + + def __iter__(self): + return self + + def next(self): + try: + self._yfp.next() + except StopIteration: + self.close() + raise StopIteration + + def close(self): + self._fp.close() diff --git a/ooni/report/tool.py b/ooni/report/tool.py new file mode 100644 index 0000000..f3dd430 --- /dev/null +++ b/ooni/report/tool.py @@ -0,0 +1,73 @@ +import yaml + +from twisted.internet import defer + +from ooni.reporter import OONIBReporter, OONIBReportLog + +from ooni.utils import log +from ooni.report import parser +from ooni.settings import config +from ooni.oonibclient import OONIBClient + + +oonib_report_log = OONIBReportLog() + + +@defer.inlineCallbacks +def upload(report_file, collector=None, bouncer=None): + with open(config.report_log_file) as f: + report_log = yaml.safe_load(f) + + report = parser.ReportLoader(report_file) + if bouncer: + oonib_client = OONIBClient(bouncer) + collector = yield oonib_client.lookupTestCollector( + report.header['test_name'] + ) + + if collector is None: + try: + collector = report_log[report_file]['collector'] + except KeyError: + raise Exception( + "No collector or bouncer specified and report file not in log." + ) + + oonib_reporter = OONIBReporter(report.header, collector) + log.msg("Creating report for %s with %s" % (report_file, collector)) + report_id = yield oonib_reporter.createReport() + yield oonib_report_log.created(report_file, collector, report_id) + for entry in report: + print "Writing entry" + yield oonib_reporter.writeReportEntry(entry) + log.msg("Closing report.") + yield oonib_reporter.finish() + yield oonib_report_log.closed(report_file) + + +@defer.inlineCallbacks +def upload_all(collector=None, bouncer=None): + for report_file, value in oonib_report_log.reports_to_upload: + yield upload(report_file, collector, bouncer) + + +def print_report(report_file, value): + print "* %s" % report_file + print " %s" % value['created_at'] + + +def status(): + print "Reports to be uploaded" + print "----------------------" + for report_file, value in oonib_report_log.reports_to_upload: + print_report(report_file, value) + + print "Reports in progress" + print "-------------------" + for report_file, value in oonib_report_log.reports_in_progress: + print_report(report_file, value) + + print "Incomplete reports" + print "------------------" + for report_file, value in oonib_report_log.reports_incomplete: + print_report(report_file, value)