[tor-commits] [ooni-probe/master] Start implementing report log for keeping track of which reports have been created and which have failed.
art at torproject.org
art at torproject.org
Thu Jun 26 13:58:11 UTC 2014
commit 00ff99fe73158538b875c76e13e0ffc0249a86e9
Author: Arturo Filastò <art at fuffa.org>
Date: Wed Jun 18 12:25:03 2014 +0200
Start implementing report log for keeping track of which reports have been created and which have failed.
Related to:
https://trac.torproject.org/projects/tor/ticket/11860
---
ooni/errors.py | 12 +++----
ooni/reporter.py | 57 ++++++++++++++++++++++++++++-
ooni/settings.py | 2 ++
ooni/tests/test_reporter.py | 84 +++++++++++++++++++++++++++++++++++++------
4 files changed, 135 insertions(+), 20 deletions(-)
diff --git a/ooni/errors.py b/ooni/errors.py
index 9f9ae75..fe6350b 100644
--- a/ooni/errors.py
+++ b/ooni/errors.py
@@ -190,10 +190,6 @@ class TorControlPortNotFound(Exception):
pass
-class ReportNotCreated(Exception):
- pass
-
-
class InsufficientPrivileges(Exception):
pass
@@ -202,10 +198,6 @@ class ProbeIPUnknown(Exception):
pass
-class GeoIPDataFilesNotFound(Exception):
- pass
-
-
class NoMoreReporters(Exception):
pass
@@ -282,6 +274,10 @@ class InvalidDestination(ReporterException):
pass
+class ReportLogExists(Exception):
+ pass
+
+
def get_error(error_key):
if error_key == 'test-helpers-key-missing':
return CouldNotFindTestHelper
diff --git a/ooni/reporter.py b/ooni/reporter.py
index c5d07ae..7d7d709 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -401,7 +401,62 @@ class OONIBReporter(OReporter):
def finish(self):
url = self.collectorAddress + '/report/' + self.reportID + '/close'
log.debug("Closing the report %s" % url)
- response = yield self.agent.request("POST", str(url))
+ yield self.agent.request("POST", str(url))
+
+
+class OONIBReportLog(object):
+
+ def __init__(self, file_name=config.report_log_file):
+ self._lock = defer.DeferredLock()
+ self.file_name = file_name
+
+ def create_report_log(self):
+ if os.path.exists(self.file_name):
+ raise errors.ReportLogExists
+ with open(self.file_name, 'w+') as f:
+ f.write(yaml.safe_dump({}))
+
+ @contextmanager
+ def edit_report_log(self):
+ with open(self.file_name) as rfp:
+ report = yaml.safe_load(rfp)
+ with open(self.file_name, 'w+') as wfp:
+ yield report
+ wfp.write(yaml.safe_dump(report))
+
+ def _report_created(self, report_file, collector_address, report_id):
+ with self.edit_report_log() as report:
+ report[report_file] = {
+ 'created_at': datetime.now(),
+ 'status': 'created',
+ 'collector': collector_address,
+ 'report_id': report_id
+ }
+
+ def report_created(self, report_file, collector_address, report_id):
+ return self._lock.run(self._report_created, report_file,
+ collector_address, report_id)
+
+ def _report_creation_failed(self, report_file, collector_address):
+ with self.edit_report_log() as report:
+ report[report_file] = {
+ 'created_at': datetime.now(),
+ 'status': 'creation-failed',
+ 'collector': collector_address
+ }
+
+ def report_creation_failed(self, report_file, collector_address):
+ return self._lock.run(self._report_creation_failed, report_file,
+ collector_address)
+
+ def _report_closed(self, report_file):
+ with self.edit_report_log() as report:
+ if report[report_file]['status'] != "created":
+ raise errors.ReportNotCreated()
+ del report[report_file]
+
+ def report_closed(self, report_file):
+ return self._lock.run(self._report_closed, report_file)
class Report(object):
diff --git a/ooni/settings.py b/ooni/settings.py
index cd4e04b..0d1275e 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -8,6 +8,7 @@ from os.path import abspath, expanduser
from ooni import otime, geoip
from ooni.utils import Storage
+
class OConfig(object):
_custom_home = None
@@ -49,6 +50,7 @@ class OConfig(object):
self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
self.decks_directory = os.path.join(self.ooni_home, 'decks')
self.reports_directory = os.path.join(self.ooni_home, 'reports')
+ self.report_log_file = os.path.join(self.ooni_home, 'reporting.yml')
if self.global_options.get('configfile'):
config_file = self.global_options['configfile']
diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py
index 23c1190..5ca7bfa 100644
--- a/ooni/tests/test_reporter.py
+++ b/ooni/tests/test_reporter.py
@@ -1,3 +1,4 @@
+import os
import yaml
import json
import time
@@ -6,9 +7,9 @@ from mock import MagicMock
from twisted.internet import defer
from twisted.trial import unittest
-from ooni.utils.net import StringProducer
from ooni import errors as e
-from ooni.reporter import YAMLReporter, OONIBReporter
+from ooni.reporter import YAMLReporter, OONIBReporter, OONIBReportLog
+
class MockTest(object):
_start_time = time.time()
@@ -33,7 +34,9 @@ oonib_generic_error_message = {
'error': 'generic-error'
}
+
class TestYAMLReporter(unittest.TestCase):
+
def setUp(self):
pass
@@ -51,37 +54,44 @@ class TestYAMLReporter(unittest.TestCase):
entry = report_entries.next()
# Check for first entry of report
- assert all(x in entry \
- for x in ['report_content', 'input', \
- 'test_name', 'test_started', \
+ assert all(x in entry
+ for x in ['report_content', 'input',
+ 'test_name', 'test_started',
'test_runtime'])
+
class TestOONIBReporter(unittest.TestCase):
-
+
def setUp(self):
self.mock_response = {}
self.collector_address = 'http://example.com'
- self.oonib_reporter = OONIBReporter(test_details, self.collector_address)
+ self.oonib_reporter = OONIBReporter(
+ test_details,
+ self.collector_address)
self.oonib_reporter.agent = MagicMock()
self.mock_agent_response = MagicMock()
+
def deliverBody(body_receiver):
body_receiver.dataReceived(json.dumps(self.mock_response))
body_receiver.connectionLost(None)
self.mock_agent_response.deliverBody = deliverBody
- self.oonib_reporter.agent.request.return_value = defer.succeed(self.mock_agent_response)
-
+ self.oonib_reporter.agent.request.return_value = defer.succeed(
+ self.mock_agent_response)
+
@defer.inlineCallbacks
def test_create_report(self):
self.mock_response = oonib_new_report_message
yield self.oonib_reporter.createReport()
- assert self.oonib_reporter.reportID == oonib_new_report_message['report_id']
+ assert self.oonib_reporter.reportID == oonib_new_report_message[
+ 'report_id']
@defer.inlineCallbacks
def test_create_report_failure(self):
self.mock_response = oonib_generic_error_message
self.mock_agent_response.code = 406
- yield self.assertFailure(self.oonib_reporter.createReport(), e.OONIBReportCreationError)
+ yield self.assertFailure(self.oonib_reporter.createReport(),
+ e.OONIBReportCreationError)
@defer.inlineCallbacks
def test_write_report_entry(self):
@@ -89,3 +99,55 @@ class TestOONIBReporter(unittest.TestCase):
yield self.oonib_reporter.writeReportEntry(req)
assert self.oonib_reporter.agent.request.called
+
+class TestOONIBReportLog(unittest.TestCase):
+
+ def setUp(self):
+ self.report_log = OONIBReportLog('report_log')
+ self.report_log.create_report_log()
+
+ def tearDown(self):
+ os.remove(self.report_log.file_name)
+
+ @defer.inlineCallbacks
+ def test_report_created(self):
+ yield self.report_log.report_created("path_to_my_report.yaml",
+ 'httpo://foo.onion',
+ 'someid')
+ with open(self.report_log.file_name) as f:
+ report = yaml.safe_load(f)
+ assert "path_to_my_report.yaml" in report
+
+ @defer.inlineCallbacks
+ def test_concurrent_edit(self):
+ d1 = self.report_log.report_created("path_to_my_report1.yaml",
+ 'httpo://foo.onion',
+ 'someid1')
+ d2 = self.report_log.report_created("path_to_my_report2.yaml",
+ 'httpo://foo.onion',
+ 'someid2')
+ yield defer.DeferredList([d1, d2])
+ with open(self.report_log.file_name) as f:
+ report = yaml.safe_load(f)
+ assert "path_to_my_report1.yaml" in report
+ assert "path_to_my_report2.yaml" in report
+
+ @defer.inlineCallbacks
+ def test_report_closed(self):
+ yield self.report_log.report_created("path_to_my_report.yaml",
+ 'httpo://foo.onion',
+ 'someid')
+ yield self.report_log.report_closed("path_to_my_report.yaml")
+
+ with open(self.report_log.file_name) as f:
+ report = yaml.safe_load(f)
+ assert "path_to_my_report.yaml" not in report
+
+ @defer.inlineCallbacks
+ def test_report_creation_failed(self):
+ yield self.report_log.report_creation_failed("path_to_my_report.yaml",
+ 'httpo://foo.onion')
+ with open(self.report_log.file_name) as f:
+ report = yaml.safe_load(f)
+ assert "path_to_my_report.yaml" in report
+ assert report["path_to_my_report.yaml"]["status"] == "creation-failed"
More information about the tor-commits
mailing list