commit cda6cd5ec73c10067ae12d8380f3408429de3a90
Author: Arturo Filastò <art(a)fuffa.org>
Date: Fri Jun 7 22:21:34 2013 +0200
Add support for closing ooni-probe reports
Also supports archiving stale reports
---
bin/archive_oonib_reports | 4 +--
oonib.conf.example | 2 ++
oonib/config.py | 1 +
oonib/report/api.py | 1 +
oonib/report/file_collector.py | 60 ++++++++++++++++++++++++++++++++++++++--
5 files changed, 63 insertions(+), 5 deletions(-)
diff --git a/bin/archive_oonib_reports b/bin/archive_oonib_reports
index 0fe6c5f..3c238af 100755
--- a/bin/archive_oonib_reports
+++ b/bin/archive_oonib_reports
@@ -10,7 +10,7 @@ from datetime import timedelta
from datetime import datetime
from oonib.otime import fromTimestamp, timestamp
from oonib.otime import InvalidTimestampFormat, utcDateNow
-from oonib import log
+from oonib import log, __version__
###############################################################################
# You can set some config options here #
@@ -88,7 +88,7 @@ def get_test_name(fields):
def get_target_or_fail(fields, report):
# set the target filename
- reportFormatVersion = fields['test_version']
+ reportFormatVersion = __version__
CC = fields['probe_cc']
# XXX: wouldn't hurt to check timestamp for sanity again?
dateInISO8601Format,__,__ = os.path.basename(report).split('_')
diff --git a/oonib.conf.example b/oonib.conf.example
index a4cb707..694296f 100644
--- a/oonib.conf.example
+++ b/oonib.conf.example
@@ -1,5 +1,6 @@
main:
report_dir: Null
+ archive_dir: Null
logfile: Null
tor_datadir: Null
database_uri: 'sqlite://oonib_test_db.db'
@@ -20,6 +21,7 @@ main:
no_save: true
profile: Null
debug: Null
+ stale_time: 3600
helpers:
http_return_request:
diff --git a/oonib/config.py b/oonib/config.py
index 8bc327f..a1a5e79 100644
--- a/oonib/config.py
+++ b/oonib/config.py
@@ -31,6 +31,7 @@ def loadConfigFile():
main = None
backend_version = __version__
+reports = {}
if not main:
main, helpers = loadConfigFile()
diff --git a/oonib/report/api.py b/oonib/report/api.py
index a451bf4..d01f47c 100644
--- a/oonib/report/api.py
+++ b/oonib/report/api.py
@@ -99,6 +99,7 @@ class NewReportHandlerDB(web.RequestHandler):
reportingBackendAPI = [
+ (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile),
(r"/report", file_collector.NewReportHandlerFile),
(r"/pcap", file_collector.PCAPReportHandler)
]
diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py
index 422c94b..193f7fd 100644
--- a/oonib/report/file_collector.py
+++ b/oonib/report/file_collector.py
@@ -1,11 +1,14 @@
import random
import string
+import time
import yaml
import json
import re
import os
-from twisted.internet import fdesc
+from datetime import datetime
+
+from twisted.internet import fdesc, reactor
from cyclone import web
@@ -38,8 +41,6 @@ def parseUpdateReportRequest(request):
return parsed_request
-
-
def parseNewReportRequest(request):
"""
Here we parse a new report request.
@@ -74,6 +75,16 @@ def parseNewReportRequest(request):
return parsed_request
+def get_report_path(report_id):
+ return os.path.join(config.main.report_dir, report_id)
+
+def stale_check(report_id):
+ if (time.time() - config.reports[report_id]) > config.main.stale_time:
+ try:
+ close_report(report_id)
+ except ReportNotFound:
+ pass
+
class NewReportHandlerFile(web.RequestHandler):
"""
Responsible for creating and updating reports by writing to flat file.
@@ -152,6 +163,10 @@ class NewReportHandlerFile(web.RequestHandler):
'report_id': report_id
}
+ config.reports[report_id] = time.time()
+
+ reactor.callLater(config.main.stale_time, stale_check, report_id)
+
self.writeToReport(report_filename, content)
self.write(response)
@@ -178,6 +193,9 @@ class NewReportHandlerFile(web.RequestHandler):
report_filename = os.path.join(config.main.report_dir,
report_id)
+ config.reports[report_id] = time.time()
+ reactor.callLater(config.main.stale_time, stale_check, report_id)
+
self.updateReport(report_filename, parsed_request['content'])
def updateReport(self, report_filename, data):
@@ -188,6 +206,42 @@ class NewReportHandlerFile(web.RequestHandler):
except IOError as e:
web.HTTPError(404, "Report not found")
+class ReportNotFound(Exception):
+ pass
+
+def close_report(report_id):
+ report_filename = get_report_path(report_id)
+ try:
+ with open(report_filename) as fd:
+ yaml_data = ''.join(fd.readline() for _ in range(12))
+ report_details = yaml.safe_load(yaml_data)
+ except IOError:
+ raise ReportNotFound
+
+ timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time']))
+ dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format(
+ timestamp=timestamp,
+ **report_details)
+
+ dst_path = os.path.join(config.main.archive_dir,
+ report_details['probe_cc'])
+
+ if not os.path.isdir(dst_path):
+ os.mkdir(dst_path)
+
+ dst_path = os.path.join(dst_path, dst_filename)
+ os.rename(report_filename, dst_path)
+
+class CloseReportHandlerFile(web.RequestHandler):
+ def get(self):
+ pass
+
+ def post(self, report_id):
+ try:
+ close_report(report_id)
+ except ReportNotFound:
+ web.HTTPError(404, "Report not found")
+
class PCAPReportHandler(web.RequestHandler):
def get(self):
pass