[tor-commits] [oonib/master] Add support for closing ooni-probe reports

[art at torproject.org]

commit cda6cd5ec73c10067ae12d8380f3408429de3a90
Author: Arturo Filastò <art at fuffa.org>
Date:   Fri Jun 7 22:21:34 2013 +0200

    Add support for closing ooni-probe reports
    
    Also supports archiving stale reports
---
 bin/archive_oonib_reports      |    4 +--
 oonib.conf.example             |    2 ++
 oonib/config.py                |    1 +
 oonib/report/api.py            |    1 +
 oonib/report/file_collector.py |   60 ++++++++++++++++++++++++++++++++++++++--
 5 files changed, 63 insertions(+), 5 deletions(-)

diff --git a/bin/archive_oonib_reports b/bin/archive_oonib_reports
index 0fe6c5f..3c238af 100755
--- a/bin/archive_oonib_reports
+++ b/bin/archive_oonib_reports
@@ -10,7 +10,7 @@ from datetime import timedelta
 from datetime import datetime
 from oonib.otime import fromTimestamp, timestamp
 from oonib.otime import InvalidTimestampFormat, utcDateNow
-from oonib import log
+from oonib import log, __version__
 
 ###############################################################################
 # You can set some config options here                                        #
@@ -88,7 +88,7 @@ def get_test_name(fields):
 
 def get_target_or_fail(fields, report):
     # set the target filename
-    reportFormatVersion = fields['test_version']
+    reportFormatVersion = __version__
     CC                  = fields['probe_cc']
     # XXX: wouldn't hurt to check timestamp for sanity again?
     dateInISO8601Format,__,__ = os.path.basename(report).split('_')
diff --git a/oonib.conf.example b/oonib.conf.example
index a4cb707..694296f 100644
--- a/oonib.conf.example
+++ b/oonib.conf.example
@@ -1,5 +1,6 @@
 main:
     report_dir: Null
+    archive_dir: Null
     logfile: Null
     tor_datadir: Null
     database_uri: 'sqlite://oonib_test_db.db'
@@ -20,6 +21,7 @@ main:
     no_save: true
     profile: Null
     debug: Null
+    stale_time: 3600
 
 helpers:
     http_return_request:
diff --git a/oonib/config.py b/oonib/config.py
index 8bc327f..a1a5e79 100644
--- a/oonib/config.py
+++ b/oonib/config.py
@@ -31,6 +31,7 @@ def loadConfigFile():
 
 main = None
 backend_version = __version__
+reports = {}
 
 if not main:
     main, helpers = loadConfigFile()
diff --git a/oonib/report/api.py b/oonib/report/api.py
index a451bf4..d01f47c 100644
--- a/oonib/report/api.py
+++ b/oonib/report/api.py
@@ -99,6 +99,7 @@ class NewReportHandlerDB(web.RequestHandler):
 
 
 reportingBackendAPI = [
+    (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile),
     (r"/report", file_collector.NewReportHandlerFile),
     (r"/pcap", file_collector.PCAPReportHandler)
 ]
diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py
index 422c94b..193f7fd 100644
--- a/oonib/report/file_collector.py
+++ b/oonib/report/file_collector.py
@@ -1,11 +1,14 @@
 import random
 import string
+import time
 import yaml
 import json
 import re
 import os
 
-from twisted.internet import fdesc
+from datetime import datetime
+
+from twisted.internet import fdesc, reactor
 
 from cyclone import web
 
@@ -38,8 +41,6 @@ def parseUpdateReportRequest(request):
 
     return parsed_request
 
-
-
 def parseNewReportRequest(request):
     """
     Here we parse a new report request.
@@ -74,6 +75,16 @@ def parseNewReportRequest(request):
 
     return parsed_request
 
+def get_report_path(report_id):
+    return os.path.join(config.main.report_dir, report_id)
+
+def stale_check(report_id):
+    if (time.time() - config.reports[report_id]) > config.main.stale_time:
+        try:
+            close_report(report_id)
+        except ReportNotFound:
+            pass
+
 class NewReportHandlerFile(web.RequestHandler):
     """
     Responsible for creating and updating reports by writing to flat file.
@@ -152,6 +163,10 @@ class NewReportHandlerFile(web.RequestHandler):
                 'report_id': report_id
         }
 
+        config.reports[report_id] = time.time()
+
+        reactor.callLater(config.main.stale_time, stale_check, report_id)
+
         self.writeToReport(report_filename, content)
 
         self.write(response)
@@ -178,6 +193,9 @@ class NewReportHandlerFile(web.RequestHandler):
         report_filename = os.path.join(config.main.report_dir,
                 report_id)
 
+        config.reports[report_id] = time.time()
+        reactor.callLater(config.main.stale_time, stale_check, report_id)
+
         self.updateReport(report_filename, parsed_request['content'])
 
     def updateReport(self, report_filename, data):
@@ -188,6 +206,42 @@ class NewReportHandlerFile(web.RequestHandler):
         except IOError as e:
             web.HTTPError(404, "Report not found")
 
+class ReportNotFound(Exception):
+    pass
+
+def close_report(report_id):
+    report_filename = get_report_path(report_id)
+    try:
+        with open(report_filename) as fd:
+            yaml_data = ''.join(fd.readline() for _ in range(12))
+            report_details = yaml.safe_load(yaml_data)
+    except IOError:
+        raise ReportNotFound
+
+    timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time']))
+    dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format(
+            timestamp=timestamp,
+            **report_details)
+
+    dst_path = os.path.join(config.main.archive_dir,
+                            report_details['probe_cc'])
+
+    if not os.path.isdir(dst_path):
+        os.mkdir(dst_path)
+
+    dst_path = os.path.join(dst_path, dst_filename)
+    os.rename(report_filename, dst_path)
+
+class CloseReportHandlerFile(web.RequestHandler):
+    def get(self):
+        pass
+
+    def post(self, report_id):
+        try:
+            close_report(report_id)
+        except ReportNotFound:
+            web.HTTPError(404, "Report not found")
+
 class PCAPReportHandler(web.RequestHandler):
     def get(self):
         pass