commit cda6cd5ec73c10067ae12d8380f3408429de3a90 Author: Arturo Filastò <art@fuffa.org> Date: Fri Jun 7 22:21:34 2013 +0200 Add support for closing ooni-probe reports Also supports archiving stale reports --- bin/archive_oonib_reports | 4 +-- oonib.conf.example | 2 ++ oonib/config.py | 1 + oonib/report/api.py | 1 + oonib/report/file_collector.py | 60 ++++++++++++++++++++++++++++++++++++++-- 5 files changed, 63 insertions(+), 5 deletions(-) diff --git a/bin/archive_oonib_reports b/bin/archive_oonib_reports index 0fe6c5f..3c238af 100755 --- a/bin/archive_oonib_reports +++ b/bin/archive_oonib_reports @@ -10,7 +10,7 @@ from datetime import timedelta from datetime import datetime from oonib.otime import fromTimestamp, timestamp from oonib.otime import InvalidTimestampFormat, utcDateNow -from oonib import log +from oonib import log, __version__ ############################################################################### # You can set some config options here # @@ -88,7 +88,7 @@ def get_test_name(fields): def get_target_or_fail(fields, report): # set the target filename - reportFormatVersion = fields['test_version'] + reportFormatVersion = __version__ CC = fields['probe_cc'] # XXX: wouldn't hurt to check timestamp for sanity again? dateInISO8601Format,__,__ = os.path.basename(report).split('_') diff --git a/oonib.conf.example b/oonib.conf.example index a4cb707..694296f 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -1,5 +1,6 @@ main: report_dir: Null + archive_dir: Null logfile: Null tor_datadir: Null database_uri: 'sqlite://oonib_test_db.db' @@ -20,6 +21,7 @@ main: no_save: true profile: Null debug: Null + stale_time: 3600 helpers: http_return_request: diff --git a/oonib/config.py b/oonib/config.py index 8bc327f..a1a5e79 100644 --- a/oonib/config.py +++ b/oonib/config.py @@ -31,6 +31,7 @@ def loadConfigFile(): main = None backend_version = __version__ +reports = {} if not main: main, helpers = loadConfigFile() diff --git a/oonib/report/api.py b/oonib/report/api.py index a451bf4..d01f47c 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -99,6 +99,7 @@ class NewReportHandlerDB(web.RequestHandler): reportingBackendAPI = [ + (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile), (r"/report", file_collector.NewReportHandlerFile), (r"/pcap", file_collector.PCAPReportHandler) ] diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index 422c94b..193f7fd 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -1,11 +1,14 @@ import random import string +import time import yaml import json import re import os -from twisted.internet import fdesc +from datetime import datetime + +from twisted.internet import fdesc, reactor from cyclone import web @@ -38,8 +41,6 @@ def parseUpdateReportRequest(request): return parsed_request - - def parseNewReportRequest(request): """ Here we parse a new report request. @@ -74,6 +75,16 @@ def parseNewReportRequest(request): return parsed_request +def get_report_path(report_id): + return os.path.join(config.main.report_dir, report_id) + +def stale_check(report_id): + if (time.time() - config.reports[report_id]) > config.main.stale_time: + try: + close_report(report_id) + except ReportNotFound: + pass + class NewReportHandlerFile(web.RequestHandler): """ Responsible for creating and updating reports by writing to flat file. @@ -152,6 +163,10 @@ class NewReportHandlerFile(web.RequestHandler): 'report_id': report_id } + config.reports[report_id] = time.time() + + reactor.callLater(config.main.stale_time, stale_check, report_id) + self.writeToReport(report_filename, content) self.write(response) @@ -178,6 +193,9 @@ class NewReportHandlerFile(web.RequestHandler): report_filename = os.path.join(config.main.report_dir, report_id) + config.reports[report_id] = time.time() + reactor.callLater(config.main.stale_time, stale_check, report_id) + self.updateReport(report_filename, parsed_request['content']) def updateReport(self, report_filename, data): @@ -188,6 +206,42 @@ class NewReportHandlerFile(web.RequestHandler): except IOError as e: web.HTTPError(404, "Report not found") +class ReportNotFound(Exception): + pass + +def close_report(report_id): + report_filename = get_report_path(report_id) + try: + with open(report_filename) as fd: + yaml_data = ''.join(fd.readline() for _ in range(12)) + report_details = yaml.safe_load(yaml_data) + except IOError: + raise ReportNotFound + + timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time'])) + dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( + timestamp=timestamp, + **report_details) + + dst_path = os.path.join(config.main.archive_dir, + report_details['probe_cc']) + + if not os.path.isdir(dst_path): + os.mkdir(dst_path) + + dst_path = os.path.join(dst_path, dst_filename) + os.rename(report_filename, dst_path) + +class CloseReportHandlerFile(web.RequestHandler): + def get(self): + pass + + def post(self, report_id): + try: + close_report(report_id) + except ReportNotFound: + web.HTTPError(404, "Report not found") + class PCAPReportHandler(web.RequestHandler): def get(self): pass