commit 00ff99fe73158538b875c76e13e0ffc0249a86e9 Author: Arturo Filastò art@fuffa.org Date: Wed Jun 18 12:25:03 2014 +0200

Start implementing report log for keeping track of which reports have been created and which have failed.

Related to: https://trac.torproject.org/projects/tor/ticket/11860 --- ooni/errors.py | 12 +++---- ooni/reporter.py | 57 ++++++++++++++++++++++++++++- ooni/settings.py | 2 ++ ooni/tests/test_reporter.py | 84 +++++++++++++++++++++++++++++++++++++------ 4 files changed, 135 insertions(+), 20 deletions(-)

diff --git a/ooni/errors.py b/ooni/errors.py index 9f9ae75..fe6350b 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -190,10 +190,6 @@ class TorControlPortNotFound(Exception): pass

-class ReportNotCreated(Exception): - pass - - class InsufficientPrivileges(Exception): pass

@@ -202,10 +198,6 @@ class ProbeIPUnknown(Exception): pass

-class GeoIPDataFilesNotFound(Exception): - pass - - class NoMoreReporters(Exception): pass

@@ -282,6 +274,10 @@ class InvalidDestination(ReporterException): pass

+class ReportLogExists(Exception): + pass + + def get_error(error_key): if error_key == 'test-helpers-key-missing': return CouldNotFindTestHelper diff --git a/ooni/reporter.py b/ooni/reporter.py index c5d07ae..7d7d709 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -401,7 +401,62 @@ class OONIBReporter(OReporter): def finish(self): url = self.collectorAddress + '/report/' + self.reportID + '/close' log.debug("Closing the report %s" % url) - response = yield self.agent.request("POST", str(url)) + yield self.agent.request("POST", str(url)) + + +class OONIBReportLog(object): + + def __init__(self, file_name=config.report_log_file): + self._lock = defer.DeferredLock() + self.file_name = file_name + + def create_report_log(self): + if os.path.exists(self.file_name): + raise errors.ReportLogExists + with open(self.file_name, 'w+') as f: + f.write(yaml.safe_dump({})) + + @contextmanager + def edit_report_log(self): + with open(self.file_name) as rfp: + report = yaml.safe_load(rfp) + with open(self.file_name, 'w+') as wfp: + yield report + wfp.write(yaml.safe_dump(report)) + + def _report_created(self, report_file, collector_address, report_id): + with self.edit_report_log() as report: + report[report_file] = { + 'created_at': datetime.now(), + 'status': 'created', + 'collector': collector_address, + 'report_id': report_id + } + + def report_created(self, report_file, collector_address, report_id): + return self._lock.run(self._report_created, report_file, + collector_address, report_id) + + def _report_creation_failed(self, report_file, collector_address): + with self.edit_report_log() as report: + report[report_file] = { + 'created_at': datetime.now(), + 'status': 'creation-failed', + 'collector': collector_address + } + + def report_creation_failed(self, report_file, collector_address): + return self._lock.run(self._report_creation_failed, report_file, + collector_address) + + def _report_closed(self, report_file): + with self.edit_report_log() as report: + if report[report_file]['status'] != "created": + raise errors.ReportNotCreated() + del report[report_file] + + def report_closed(self, report_file): + return self._lock.run(self._report_closed, report_file)

class Report(object): diff --git a/ooni/settings.py b/ooni/settings.py index cd4e04b..0d1275e 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -8,6 +8,7 @@ from os.path import abspath, expanduser from ooni import otime, geoip from ooni.utils import Storage

+ class OConfig(object): _custom_home = None

@@ -49,6 +50,7 @@ class OConfig(object): self.inputs_directory = os.path.join(self.ooni_home, 'inputs') self.decks_directory = os.path.join(self.ooni_home, 'decks') self.reports_directory = os.path.join(self.ooni_home, 'reports') + self.report_log_file = os.path.join(self.ooni_home, 'reporting.yml')

if self.global_options.get('configfile'): config_file = self.global_options['configfile'] diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 23c1190..5ca7bfa 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -1,3 +1,4 @@ +import os import yaml import json import time @@ -6,9 +7,9 @@ from mock import MagicMock from twisted.internet import defer from twisted.trial import unittest

-from ooni.utils.net import StringProducer from ooni import errors as e -from ooni.reporter import YAMLReporter, OONIBReporter +from ooni.reporter import YAMLReporter, OONIBReporter, OONIBReportLog +

class MockTest(object): _start_time = time.time() @@ -33,7 +34,9 @@ oonib_generic_error_message = { 'error': 'generic-error' }

+ class TestYAMLReporter(unittest.TestCase): + def setUp(self): pass

@@ -51,37 +54,44 @@ class TestYAMLReporter(unittest.TestCase):

entry = report_entries.next() # Check for first entry of report - assert all(x in entry \ - for x in ['report_content', 'input', \ - 'test_name', 'test_started', \ + assert all(x in entry + for x in ['report_content', 'input', + 'test_name', 'test_started', 'test_runtime'])

+ class TestOONIBReporter(unittest.TestCase): - + def setUp(self): self.mock_response = {} self.collector_address = 'http://example.com'

- self.oonib_reporter = OONIBReporter(test_details, self.collector_address) + self.oonib_reporter = OONIBReporter( + test_details, + self.collector_address) self.oonib_reporter.agent = MagicMock() self.mock_agent_response = MagicMock() + def deliverBody(body_receiver): body_receiver.dataReceived(json.dumps(self.mock_response)) body_receiver.connectionLost(None) self.mock_agent_response.deliverBody = deliverBody - self.oonib_reporter.agent.request.return_value = defer.succeed(self.mock_agent_response) - + self.oonib_reporter.agent.request.return_value = defer.succeed( + self.mock_agent_response) + @defer.inlineCallbacks def test_create_report(self): self.mock_response = oonib_new_report_message yield self.oonib_reporter.createReport() - assert self.oonib_reporter.reportID == oonib_new_report_message['report_id'] + assert self.oonib_reporter.reportID == oonib_new_report_message[ + 'report_id']

@defer.inlineCallbacks def test_create_report_failure(self): self.mock_response = oonib_generic_error_message self.mock_agent_response.code = 406 - yield self.assertFailure(self.oonib_reporter.createReport(), e.OONIBReportCreationError) + yield self.assertFailure(self.oonib_reporter.createReport(), + e.OONIBReportCreationError)

@defer.inlineCallbacks def test_write_report_entry(self): @@ -89,3 +99,55 @@ class TestOONIBReporter(unittest.TestCase): yield self.oonib_reporter.writeReportEntry(req) assert self.oonib_reporter.agent.request.called

+ +class TestOONIBReportLog(unittest.TestCase): + + def setUp(self): + self.report_log = OONIBReportLog('report_log') + self.report_log.create_report_log() + + def tearDown(self): + os.remove(self.report_log.file_name) + + @defer.inlineCallbacks + def test_report_created(self): + yield self.report_log.report_created("path_to_my_report.yaml", + 'httpo://foo.onion', + 'someid') + with open(self.report_log.file_name) as f: + report = yaml.safe_load(f) + assert "path_to_my_report.yaml" in report + + @defer.inlineCallbacks + def test_concurrent_edit(self): + d1 = self.report_log.report_created("path_to_my_report1.yaml", + 'httpo://foo.onion', + 'someid1') + d2 = self.report_log.report_created("path_to_my_report2.yaml", + 'httpo://foo.onion', + 'someid2') + yield defer.DeferredList([d1, d2]) + with open(self.report_log.file_name) as f: + report = yaml.safe_load(f) + assert "path_to_my_report1.yaml" in report + assert "path_to_my_report2.yaml" in report + + @defer.inlineCallbacks + def test_report_closed(self): + yield self.report_log.report_created("path_to_my_report.yaml", + 'httpo://foo.onion', + 'someid') + yield self.report_log.report_closed("path_to_my_report.yaml") + + with open(self.report_log.file_name) as f: + report = yaml.safe_load(f) + assert "path_to_my_report.yaml" not in report + + @defer.inlineCallbacks + def test_report_creation_failed(self): + yield self.report_log.report_creation_failed("path_to_my_report.yaml", + 'httpo://foo.onion') + with open(self.report_log.file_name) as f: + report = yaml.safe_load(f) + assert "path_to_my_report.yaml" in report + assert report["path_to_my_report.yaml"]["status"] == "creation-failed"