commit 061af8e36784ced0f96965eb992b340d63ce027f Author: Arturo Filastò arturo@filasto.net Date: Thu Feb 4 16:39:14 2016 +0100

Fix report uploader to work with JSON

* Add better exception handling to deckgen * We should actually be using POST since PUT is indempotent * Debugging of backward and forward data format compatibility * Better integrity checks in web_consistency --- ooni/deckgen/processors/citizenlab_test_lists.py | 5 +++++ ooni/nettest.py | 3 ++- ooni/report/tool.py | 4 ++-- ooni/reporter.py | 14 +++++++++++--- 4 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py index 0a56ec0..086b550 100644 --- a/ooni/deckgen/processors/citizenlab_test_lists.py +++ b/ooni/deckgen/processors/citizenlab_test_lists.py @@ -47,6 +47,11 @@ def generate_global_input(dst): "test-lists-master/lists/" "global.csv")

+ if not input_list: + print("Could not find the global input list") + print("Perhaps you should run ooniresources --update-inputs") + raise Exception("Could not find the global input list") + load_input(input_list, filename)

return filename diff --git a/ooni/nettest.py b/ooni/nettest.py index e347d55..9bdc6d9 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -220,7 +220,8 @@ class NetTestLoader(object): 'input_hashes': input_file_hashes, 'report_id': self.reportID, 'test_helpers': self.testHelpers, - 'annotations': self.annotations + 'annotations': self.annotations, + 'data_format_version': '0.2.0' } return test_details

diff --git a/ooni/report/tool.py b/ooni/report/tool.py index a16794a..5946c42 100644 --- a/ooni/report/tool.py +++ b/ooni/report/tool.py @@ -14,7 +14,7 @@ from ooni.oonibclient import OONIBClient def upload(report_file, collector=None, bouncer=None): oonib_report_log = OONIBReportLog()

- print "Attempting to upload %s" % report_file + log.msg("Attempting to upload %s" % report_file)

with open(config.report_log_file) as f: report_log = yaml.safe_load(f) @@ -47,9 +47,9 @@ def upload(report_file, collector=None, bouncer=None): oonib_reporter = OONIBReporter(report.header, collector) log.msg("Creating report for %s with %s" % (report_file, collector)) report_id = yield oonib_reporter.createReport() + report.header['report_id'] = report_id yield oonib_report_log.created(report_file, collector, report_id) for entry in report: - print "Writing entry" yield oonib_reporter.writeReportEntry(entry) log.msg("Closing report.") yield oonib_reporter.finish() diff --git a/ooni/reporter.py b/ooni/reporter.py index b725540..d9d147c 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -1,3 +1,4 @@ +import uuid import yaml import json import os @@ -198,7 +199,6 @@ class YAMLReporter(OReporter): raise Exception("Failed to serialise entry") content += safe_dump(report_entry) content += '...\n' - report_entry.update(self.testDetails) self._write(content)

def createReport(self): @@ -265,11 +265,17 @@ class OONIBReporter(OReporter): if isinstance(entry, Measurement): report_entry = { 'input': entry.testInstance.report.pop('input', None), + 'id': str(uuid.uuid4()), + 'test_start_time': entry.testInstance.report.pop('test_start_time', None), + 'test_runtime': entry.testInstance.report.pop('test_runtime', None), 'test_keys': entry.testInstance.report } elif isinstance(entry, dict): report_entry = { 'input': entry.pop('input', None), + 'id': str(uuid.uuid4()), + 'test_start_time': entry.pop('test_start_time', None), + 'test_runtime': entry.pop('test_runtime', None), 'test_keys': entry } else: @@ -284,7 +290,6 @@ class OONIBReporter(OReporter): report_entry = entry else: raise Exception("Failed to serialise entry") - report_entry.update(self.testDetails) content += safe_dump(report_entry) content += '...\n' return content @@ -312,7 +317,7 @@ class OONIBReporter(OReporter): bodyProducer = StringProducer(request_json)

try: - yield self.agent.request("PUT", str(url), + yield self.agent.request("POST", str(url), bodyProducer=bodyProducer) except Exception as exc: log.err("Error in writing report entry") @@ -349,6 +354,7 @@ class OONIBReporter(OReporter): 'test_version': self.testDetails['test_version'], 'start_time': self.testDetails['start_time'], 'input_hashes': self.testDetails['input_hashes'], + 'data_format_version': self.testDetails['data_format_version'], 'format': 'json' } # import values from the environment @@ -618,6 +624,8 @@ class Report(object): self.collector_address)

def created(report_id): + self.reportID = report_id + self.test_details['report_id'] = report_id if not self.oonib_reporter: return return self.report_log.created(self.report_filename,