[tor-commits] [ooni-probe/master] Fix report uploader to work with JSON

[art at torproject.org]

commit 061af8e36784ced0f96965eb992b340d63ce027f
Author: Arturo Filastò <arturo at filasto.net>
Date:   Thu Feb 4 16:39:14 2016 +0100

    Fix report uploader to work with JSON
    
    * Add better exception handling to deckgen
    * We should actually be using POST since PUT is indempotent
    * Debugging of backward and forward data format compatibility
    * Better integrity checks in web_consistency
---
 ooni/deckgen/processors/citizenlab_test_lists.py |  5 +++++
 ooni/nettest.py                                  |  3 ++-
 ooni/report/tool.py                              |  4 ++--
 ooni/reporter.py                                 | 14 +++++++++++---
 4 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py
index 0a56ec0..086b550 100644
--- a/ooni/deckgen/processors/citizenlab_test_lists.py
+++ b/ooni/deckgen/processors/citizenlab_test_lists.py
@@ -47,6 +47,11 @@ def generate_global_input(dst):
                                            "test-lists-master/lists/"
                                            "global.csv")
 
+    if not input_list:
+        print("Could not find the global input list")
+        print("Perhaps you should run ooniresources --update-inputs")
+        raise Exception("Could not find the global input list")
+
     load_input(input_list, filename)
 
     return filename
diff --git a/ooni/nettest.py b/ooni/nettest.py
index e347d55..9bdc6d9 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -220,7 +220,8 @@ class NetTestLoader(object):
             'input_hashes': input_file_hashes,
             'report_id': self.reportID,
             'test_helpers': self.testHelpers,
-            'annotations': self.annotations
+            'annotations': self.annotations,
+            'data_format_version': '0.2.0'
         }
         return test_details
 
diff --git a/ooni/report/tool.py b/ooni/report/tool.py
index a16794a..5946c42 100644
--- a/ooni/report/tool.py
+++ b/ooni/report/tool.py
@@ -14,7 +14,7 @@ from ooni.oonibclient import OONIBClient
 def upload(report_file, collector=None, bouncer=None):
     oonib_report_log = OONIBReportLog()
 
-    print "Attempting to upload %s" % report_file
+    log.msg("Attempting to upload %s" % report_file)
 
     with open(config.report_log_file) as f:
         report_log = yaml.safe_load(f)
@@ -47,9 +47,9 @@ def upload(report_file, collector=None, bouncer=None):
     oonib_reporter = OONIBReporter(report.header, collector)
     log.msg("Creating report for %s with %s" % (report_file, collector))
     report_id = yield oonib_reporter.createReport()
+    report.header['report_id'] = report_id
     yield oonib_report_log.created(report_file, collector, report_id)
     for entry in report:
-        print "Writing entry"
         yield oonib_reporter.writeReportEntry(entry)
     log.msg("Closing report.")
     yield oonib_reporter.finish()
diff --git a/ooni/reporter.py b/ooni/reporter.py
index b725540..d9d147c 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -1,3 +1,4 @@
+import uuid
 import yaml
 import json
 import os
@@ -198,7 +199,6 @@ class YAMLReporter(OReporter):
             raise Exception("Failed to serialise entry")
         content += safe_dump(report_entry)
         content += '...\n'
-        report_entry.update(self.testDetails)
         self._write(content)
 
     def createReport(self):
@@ -265,11 +265,17 @@ class OONIBReporter(OReporter):
             if isinstance(entry, Measurement):
                 report_entry = {
                     'input': entry.testInstance.report.pop('input', None),
+                    'id': str(uuid.uuid4()),
+                    'test_start_time': entry.testInstance.report.pop('test_start_time', None),
+                    'test_runtime': entry.testInstance.report.pop('test_runtime', None),
                     'test_keys': entry.testInstance.report
                 }
             elif isinstance(entry, dict):
                 report_entry = {
                     'input': entry.pop('input', None),
+                    'id': str(uuid.uuid4()),
+                    'test_start_time': entry.pop('test_start_time', None),
+                    'test_runtime': entry.pop('test_runtime', None),
                     'test_keys': entry
                 }
             else:
@@ -284,7 +290,6 @@ class OONIBReporter(OReporter):
                 report_entry = entry
             else:
                 raise Exception("Failed to serialise entry")
-            report_entry.update(self.testDetails)
             content += safe_dump(report_entry)
             content += '...\n'
             return content
@@ -312,7 +317,7 @@ class OONIBReporter(OReporter):
         bodyProducer = StringProducer(request_json)
 
         try:
-            yield self.agent.request("PUT", str(url),
+            yield self.agent.request("POST", str(url),
                                      bodyProducer=bodyProducer)
         except Exception as exc:
             log.err("Error in writing report entry")
@@ -349,6 +354,7 @@ class OONIBReporter(OReporter):
             'test_version': self.testDetails['test_version'],
             'start_time': self.testDetails['start_time'],
             'input_hashes': self.testDetails['input_hashes'],
+            'data_format_version': self.testDetails['data_format_version'],
             'format': 'json'
         }
         # import values from the environment
@@ -618,6 +624,8 @@ class Report(object):
                                                    self.collector_address)
 
         def created(report_id):
+            self.reportID = report_id
+            self.test_details['report_id'] = report_id
             if not self.oonib_reporter:
                 return
             return self.report_log.created(self.report_filename,