[tor-commits] [ooni-probe/master] Add tool for viewing the status of report submission and allow users to upload reports.

[art at torproject.org]

commit 80dc4719da78527458a97ab33b97b1156e67f113
Author: Arturo Filastò <art at fuffa.org>
Date:   Wed Jun 25 13:18:05 2014 +0200

    Add tool for viewing the status of report submission and allow users to upload reports.
---
 bin/oonireport          |   26 +++++++++++++++++
 ooni/report/__init__.py |    1 +
 ooni/report/cli.py      |   57 ++++++++++++++++++++++++++++++++++++
 ooni/report/parser.py   |   35 +++++++++++++++++++++++
 ooni/report/tool.py     |   73 +++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 192 insertions(+)

diff --git a/bin/oonireport b/bin/oonireport
new file mode 100755
index 0000000..4e6d48a
--- /dev/null
+++ b/bin/oonireport
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+import os
+import sys
+
+sys.path[:] = map(os.path.abspath, sys.path)
+sys.path.insert(0, os.path.abspath(os.getcwd()))
+
+from twisted.internet import defer, reactor
+
+from ooni.report import cli
+
+def failed(failure):
+    log.err("Failed to run ooni-report")
+    log.exception(failure)
+    reactor.stop()
+
+def done(result):
+    reactor.stop()
+
+def start():
+    d = defer.maybeDeferred(cli.run)
+    d.addCallback(done)
+    d.addErrback(done)
+
+reactor.callWhenRunning(start)
+reactor.run()
diff --git a/ooni/report/__init__.py b/ooni/report/__init__.py
new file mode 100644
index 0000000..a4e2017
--- /dev/null
+++ b/ooni/report/__init__.py
@@ -0,0 +1 @@
+__version__ = "0.1"
diff --git a/ooni/report/cli.py b/ooni/report/cli.py
new file mode 100644
index 0000000..eee5c53
--- /dev/null
+++ b/ooni/report/cli.py
@@ -0,0 +1,57 @@
+import os
+import sys
+
+from ooni.report import __version__
+from ooni.report import tool
+from ooni.settings import config
+
+from twisted.python import usage
+
+
+class Options(usage.Options):
+
+    synopsis = """%s [options] upload | status
+""" % (os.path.basename(sys.argv[0]),)
+
+    optParameters = [
+        ["collector", "c", None,
+         "Specify the collector to upload the result to."],
+        ["bouncer", "b", None,
+         "Specify the bouncer to query for a collector."]
+    ]
+
+    def opt_version(self):
+        print "oonireport version:", __version__
+        sys.exit(0)
+
+    def parseArgs(self, *args):
+        self['command'] = args[0]
+        if self['command'] not in ("upload", "status"):
+            raise usage.UsageError(
+                "Must specify either command upload or status"
+            )
+        if self['command'] == "upload":
+            try:
+                self['report_file'] = args[1]
+            except KeyError:
+                self['report_file'] = None
+
+
+def parse_options():
+    options = Options()
+    options.parseOptions()
+    return dict(options)
+
+
+def run():
+    config.read_config_file()
+    options = parse_options()
+    if options['command'] == "upload" and options['report_file']:
+        return tool.upload(options['report_file'],
+                           options['collector'],
+                           options['bouncer'])
+    elif options['command'] == "upload":
+        return tool.upload_all(options['collector'],
+                               options['bouncer'])
+    elif options['command'] == "status":
+        return tool.status()
diff --git a/ooni/report/parser.py b/ooni/report/parser.py
new file mode 100644
index 0000000..a986522
--- /dev/null
+++ b/ooni/report/parser.py
@@ -0,0 +1,35 @@
+import yaml
+
+
+class ReportLoader(object):
+    _header_keys = (
+        'probe_asn',
+        'probe_cc',
+        'probe_ip',
+        'start_time',
+        'test_name',
+        'test_version',
+        'options',
+        'input_hashes',
+        'software_name',
+        'software_version'
+    )
+
+    def __init__(self, report_filename):
+        self._fp = open(report_filename)
+        self._yfp = yaml.safe_load_all(self._fp)
+
+        self.header = self._yfp.next()
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        try:
+            self._yfp.next()
+        except StopIteration:
+            self.close()
+            raise StopIteration
+
+    def close(self):
+        self._fp.close()
diff --git a/ooni/report/tool.py b/ooni/report/tool.py
new file mode 100644
index 0000000..f3dd430
--- /dev/null
+++ b/ooni/report/tool.py
@@ -0,0 +1,73 @@
+import yaml
+
+from twisted.internet import defer
+
+from ooni.reporter import OONIBReporter, OONIBReportLog
+
+from ooni.utils import log
+from ooni.report import parser
+from ooni.settings import config
+from ooni.oonibclient import OONIBClient
+
+
+oonib_report_log = OONIBReportLog()
+
+
+ at defer.inlineCallbacks
+def upload(report_file, collector=None, bouncer=None):
+    with open(config.report_log_file) as f:
+        report_log = yaml.safe_load(f)
+
+    report = parser.ReportLoader(report_file)
+    if bouncer:
+        oonib_client = OONIBClient(bouncer)
+        collector = yield oonib_client.lookupTestCollector(
+            report.header['test_name']
+        )
+
+    if collector is None:
+        try:
+            collector = report_log[report_file]['collector']
+        except KeyError:
+            raise Exception(
+                "No collector or bouncer specified and report file not in log."
+            )
+
+    oonib_reporter = OONIBReporter(report.header, collector)
+    log.msg("Creating report for %s with %s" % (report_file, collector))
+    report_id = yield oonib_reporter.createReport()
+    yield oonib_report_log.created(report_file, collector, report_id)
+    for entry in report:
+        print "Writing entry"
+        yield oonib_reporter.writeReportEntry(entry)
+    log.msg("Closing report.")
+    yield oonib_reporter.finish()
+    yield oonib_report_log.closed(report_file)
+
+
+ at defer.inlineCallbacks
+def upload_all(collector=None, bouncer=None):
+    for report_file, value in oonib_report_log.reports_to_upload:
+        yield upload(report_file, collector, bouncer)
+
+
+def print_report(report_file, value):
+    print "* %s" % report_file
+    print "  %s" % value['created_at']
+
+
+def status():
+    print "Reports to be uploaded"
+    print "----------------------"
+    for report_file, value in oonib_report_log.reports_to_upload:
+        print_report(report_file, value)
+
+    print "Reports in progress"
+    print "-------------------"
+    for report_file, value in oonib_report_log.reports_in_progress:
+        print_report(report_file, value)
+
+    print "Incomplete reports"
+    print "------------------"
+    for report_file, value in oonib_report_log.reports_incomplete:
+        print_report(report_file, value)