commit 19d533e4d1a473365d9794834cf4dea0a5a931a1 Author: Arturo Filastò <art@fuffa.org> Date: Wed Nov 28 14:52:31 2012 +0100 Improve oonib * More robust error handling * Add docstrings on not implemented collector parts * Refactor ooni reporting system --- ooni/__init__.py | 6 +- oonib/__init__.py | 2 + oonib/report/__init__.py | 8 ++ oonib/report/api.py | 156 ++++------------------------------------ oonib/report/file_collector.py | 145 +++++++++++++++++++++++++++++++++++++ 5 files changed, 172 insertions(+), 145 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index bf98e16..36afc9a 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -11,9 +11,9 @@ from . import runner from . import templates from . import utils +__author__ = "Arturo Filastò" +__version__ = "0.0.7.1-alpha" + __all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', 'runner', 'templates', 'utils'] - -__author__ = "Arturo Filastò" -__version__ = "0.0.7.1-alpha" diff --git a/oonib/__init__.py b/oonib/__init__.py index 1a853dd..ab7419c 100644 --- a/oonib/__init__.py +++ b/oonib/__init__.py @@ -14,6 +14,8 @@ from storm.uri import URI from storm.twisted.transact import Transactor from storm.databases.sqlite import SQLite +__version__ = '0.0.1' + from oonib import config database = SQLite(URI(config.main.database_uri)) diff --git a/oonib/report/__init__.py b/oonib/report/__init__.py index e69de29..96667b4 100644 --- a/oonib/report/__init__.py +++ b/oonib/report/__init__.py @@ -0,0 +1,8 @@ +def generateReportID(): + return otime.timestamp() + '_' + randomStr(20) + +class MissingField(Exception): + pass + +class InvalidRequestField(Exception): + pass diff --git a/oonib/report/api.py b/oonib/report/api.py index 5668857..9c42a54 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -1,7 +1,7 @@ """ -/report/do +/new -/report/pcap +/pcap This is the async pcap reporting system. It requires the client to have created a report already, but can work independently from test progress. @@ -18,47 +18,9 @@ from cyclone import web from ooni.utils import randomStr, otime from oonib import models, config - -backend_version = '0.0.1' - -def generateReportID(): - return otime.timestamp() + '_' + randomStr(20) - -class MissingField(Exception): - pass - -class InvalidRequestField(Exception): - pass - -def parseNewReportRequest(request): - """ - Here we parse a new report request. - """ - version_string = re.compile("[0-9A-Za-z_\-\.]+$") - name = re.compile("[a-zA-Z0-9_\- ]+$") - expected_request = {'software_name': name, - 'software_version': version_string, - 'test_name': name, - 'test_version': version_string, - 'progress': re.compile("[0-9]+$") - } - parsed_request = json.loads(request) - for k, regexp in expected_request.items(): - try: - value_to_check = parsed_request[k] - except KeyError: - raise MissingField(k) - print "Matching %s with %s | %s" % (regexp, value_to_check, k) - if re.match(regexp, str(value_to_check)): - continue - else: - raise InvalidRequestField(k) - return parsed_request +from oonib.report import file_collector def parseUpdateReportRequest(request): - # XXX this and the function above can probably be refactored into something - # more compact. There is quite a bit of code duplication going on here. - #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") # this is the regexp for the reports that include the timestamp @@ -80,92 +42,6 @@ def parseUpdateReportRequest(request): return parsed_request -class NewReportHandlerFile(web.RequestHandler): - """ - Responsible for creating and updating reports by writing to flat file. - """ - def post(self): - """ - Creates a new report with the input - - * Request - - {'software_name': 'XXX', - 'software_version': 'XXX', - 'test_name': 'XXX', - 'test_version': 'XXX', - 'progress': 'XXX', - 'content': 'XXX' - } - - Optional: - 'test_helper': 'XXX' - 'client_ip': 'XXX' - - * Response - - {'backend_version': 'XXX', 'report_id': 'XXX'} - - """ - # XXX here we should validate and sanitize the request - try: - report_data = parseNewReportRequest(self.request.body) - except InvalidRequestField, e: - raise web.HTTPError(400, "Invalid Request Field %s" % e) - except MissingField, e: - raise web.HTTPError(400, "Missing Request Field %s" % e) - - print "Parsed this data %s" % report_data - software_name = report_data['software_name'] - software_version = report_data['software_version'] - test_name = report_data['test_name'] - test_version = report_data['test_version'] - content = report_data['content'] - - report_id = generateReportID() - - #report_filename = '_'.join((report_id, - # report_data['software_name'], - # report_data['software_version'], - # report_data['test_name'], - # report_data['test_version'])) - - # The report filename contains the timestamp of the report plus a - # random nonce - report_filename = os.path.join(config.main.report_dir, report_id) - report_filename += '.yamloo' - - response = {'backend_version': backend_version, - 'report_id': report_id - } - - fp = open(report_filename, 'w+') - fp.write(report_data['content']) - fp.close() - self.write(response) - - def put(self): - """ - Update an already existing report. - - {'report_id': 'XXX', - 'content': 'XXX' - } - """ - parsed_request = parseUpdateReportRequest(self.request.body) - report_id = parsed_request['report_id'] - print "Got this request %s" % parsed_request - - report_filename = os.path.join(config.main.report_dir, report_id) - report_filename += '.yamloo' - try: - with open(report_filename, 'a+') as f: - # XXX this could be quite big. We should probably use the - # twisted.internet.fdesc module - print parsed_request['content'] - f.write(parsed_request['content']) - except IOError as e: - web.HTTPError(404, "Report not found") class NewReportHandlerDB(web.RequestHandler): """ @@ -177,7 +53,8 @@ class NewReportHandlerDB(web.RequestHandler): @defer.inlineCallbacks def post(self): """ - Creates a new report with the input + Creates a new report with the input to the database. + XXX this is not yet implemented. * Request @@ -193,23 +70,23 @@ class NewReportHandlerDB(web.RequestHandler): 'test_helper': 'XXX' 'client_ip': 'XXX' - * Response + * Response {'backend_version': 'XXX', 'report_id': 'XXX'} """ - parsed_request = json.loads(self.request.body) - # XXX here we should validate and sanitize the request - report_data = parsed_request + report_data = json.loads(self.request.body) new_report = models.Report() - print "Got %s as request" % parsed_request + log.debug("Got this request %s" % report_data) result = yield new_report.new(report_data) self.write(result) self.finish() def put(self): """ - Update an already existing report. + Update an already existing report with the database. + + XXX this is not yet implemented. {'report_id': 'XXX', 'content': 'XXX' @@ -217,15 +94,10 @@ class NewReportHandlerDB(web.RequestHandler): """ pass -class PCAPReportHandler(web.RequestHandler): - def get(self): - pass - - def post(self): - pass -reportingBackendAPI = [(r"/report/new", NewReportHandlerFile), - (r"/report/pcap", PCAPReportHandler) +reportingBackendAPI = [ + (r"/report", file_collector.NewReportHandlerFile), + (r"/pcap", file_collector.PCAPReportHandler) ] reportingBackend = web.Application(reportingBackendAPI, debug=True) diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py new file mode 100644 index 0000000..9896784 --- /dev/null +++ b/oonib/report/file_collector.py @@ -0,0 +1,145 @@ +import random +import string +import json +import re +import os + +from oonib.report import generateReportID +from oonib.report import MissingField, InvalidRequestField + +from cyclone import web + +def parseNewReportRequest(request): + """ + Here we parse a new report request. + """ + version_string = re.compile("[0-9A-Za-z_\-\.]+$") + name = re.compile("[a-zA-Z0-9_\- ]+$") + + expected_request = {'software_name': name, + 'software_version': version_string, + 'test_name': name, + 'test_version': version_string + } + + parsed_request = json.loads(request) + for k, regexp in expected_request.items(): + try: + value_to_check = parsed_request[k] + except KeyError: + raise MissingField(k) + print "Matching %s with %s | %s" % (regexp, value_to_check, k) + if re.match(regexp, str(value_to_check)): + continue + else: + raise InvalidRequestField(k) + return parsed_request + +class NewReportHandlerFile(web.RequestHandler): + """ + Responsible for creating and updating reports by writing to flat file. + """ + def post(self): + """ + Creates a new report with the input + + * Request + + {'software_name': 'XXX', + 'software_version': 'XXX', + 'test_name': 'XXX', + 'test_version': 'XXX', + 'progress': 'XXX', + 'content': 'XXX' + } + + Optional: + 'test_helper': 'XXX' + 'client_ip': 'XXX' + + (not implemented, nor in client, nor in backend) + The idea behind these two fields is that it would be interesting to + also collect how the request was observed from the collectors point + of view. + + We use as a unique key the client_ip address and a time window. We + then need to tell the test_helper that is selected the client_ip + address and tell it to expect a connection from a probe in that time + window. + + Once the test_helper sees a connection from that client_ip it will + store for the testing session the data that it receives. + When the probe completes the report (or the time window is over) the + final report will include also the data collected from the + collectors view point. + + * Response + + {'backend_version': 'XXX', 'report_id': 'XXX'} + + """ + # XXX here we should validate and sanitize the request + try: + report_data = parseNewReportRequest(self.request.body) + except InvalidRequestField, e: + raise web.HTTPError(400, "Invalid Request Field %s" % e) + except MissingField, e: + raise web.HTTPError(400, "Missing Request Field %s" % e) + + print "Parsed this data %s" % report_data + software_name = report_data['software_name'] + software_version = report_data['software_version'] + test_name = report_data['test_name'] + test_version = report_data['test_version'] + content = report_data['content'] + + report_id = generateReportID() + + #report_filename = '_'.join((report_id, + # report_data['software_name'], + # report_data['software_version'], + # report_data['test_name'], + # report_data['test_version'])) + + # The report filename contains the timestamp of the report plus a + # random nonce + report_filename = os.path.join(config.main.report_dir, report_id) + report_filename += '.yamloo' + + response = {'backend_version': backend_version, + 'report_id': report_id + } + + fp = open(report_filename, 'w+') + fp.write(report_data['content']) + fp.close() + self.write(response) + + def put(self): + """ + Update an already existing report. + + {'report_id': 'XXX', + 'content': 'XXX' + } + """ + parsed_request = parseUpdateReportRequest(self.request.body) + report_id = parsed_request['report_id'] + print "Got this request %s" % parsed_request + + report_filename = os.path.join(config.main.report_dir, report_id) + report_filename += '.yamloo' + try: + with open(report_filename, 'a+') as f: + # XXX-Twisted change this to use t.i.a.fdesc and perhaps make a + # nice little object for it. + f.write(parsed_request['content']) + except IOError as e: + web.HTTPError(404, "Report not found") + +class PCAPReportHandler(web.RequestHandler): + def get(self): + pass + + def post(self): + pass