commit a9b6dd9f9a8f8a51cdedecbb17351802934b10df Author: Arturo Filastò <art@fuffa.org> Date: Fri Aug 16 19:50:05 2013 +0200 Add skeleton of the various new directories for bouncer/policy/inputs/decks --- data/bouncer.yaml | 3 +++ data/policy.yaml | 15 +++++++++++++++ oonib.conf.example | 12 ++++++------ oonib/api.py | 31 +++++++++++++++++++++++++++++++ oonib/deck/handlers.py | 6 +++--- oonib/inputs/handlers.py | 15 ++++++++++----- oonib/policy/handlers.py | 15 ++++++++++----- oonib/report/handlers.py | 21 +++++++++++---------- oonib/runner.py | 18 ++---------------- 9 files changed, 91 insertions(+), 45 deletions(-) diff --git a/data/bouncer.yaml b/data/bouncer.yaml new file mode 100644 index 0000000..ba54898 --- /dev/null +++ b/data/bouncer.yaml @@ -0,0 +1,3 @@ +collector: + httpo://nkvphnp3p6agi5qq.onion: + test-helper: {dns: 93.95.227.200, http-return-json-headers: 'http://93.95.227.200'} diff --git a/data/policy.yaml b/data/policy.yaml new file mode 100644 index 0000000..fecdf29 --- /dev/null +++ b/data/policy.yaml @@ -0,0 +1,15 @@ +inputs: +- {id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1} +- {id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290} +nettest: +- {name: dnsconsistency, version: 0.5} +- {name: http_requests, version: 0.2.3} +- {name: tcpconnect, version: 0.1} +- {name: captiveportal, version: 0.2} +- {name: daphne, version: 0.1} +- {name: dnsspoof, version: 0.2} +- {name: http_body, version: 0.0.1} +- {name: http_header_field_manipulation, version: 0.1.3} +- {name: http_host, version: 0.2.3} +- {name: http_invalid_request_line, version: 0.1.4} +- {name: traceroute, version: 0.1.1} diff --git a/oonib.conf.example b/oonib.conf.example index 60e177f..a1bba84 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -1,10 +1,10 @@ main: - report_dir: Null - archive_dir: Null - inputs_dir: Null - deck_dir: Null - policy_file: Null - bouncer_file: Null + report_dir: data/reports/ + archive_dir: data/archive/ + inputs_dir: data/inputs/ + deck_dir: data/decks/ + policy_file: data/policy.yaml + bouncer_file: data/bouncer.yaml logfile: Null tor_datadir: Null diff --git a/oonib/api.py b/oonib/api.py new file mode 100644 index 0000000..ce682fc --- /dev/null +++ b/oonib/api.py @@ -0,0 +1,31 @@ +from cyclone import web + +from oonib.deck.api import deckAPI +from oonib.inputs.api import inputsAPI +from oonib.policy.api import policyAPI +from oonib.bouncer.api import bouncerAPI + +from oonib import config + +class OONIBHandler(web.RequestHandler): + pass + +class OONIBError(web.HTTPError): + pass + +oonibAPI = [] +oonibAPI += reportAPI + +if config.inputs_dir: + oonibAPI += inputsAPI + +if config.deck_dir: + oonibAPI += deckAPI + +if config.policy_file: + oonibAPI += policyAPI + +if config.bouncer_file: + oonibAPI += bouncerAPI + + diff --git a/oonib/data/.gitignore b/oonib/data/.gitignore deleted file mode 100644 index e69de29..0000000 diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index 511b6b1..97b390a 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -1,6 +1,6 @@ -from cyclone import web +from oonib.api import OONIBHandler -class DeckDescHandler(web.RequestHandler): +class DeckDescHandler(OONIBHandler): def get(self, deckID): bn = os.path.basename(deckID) try: @@ -16,7 +16,7 @@ class DeckDescHandler(web.RequestHandler): except KeyError: log.err("Deck %s missing required keys!" % deckID) -class DeckListHandler(web.RequestHandler): +class DeckListHandler(OONIBHandler): def get(self): if not config.main.deck_dir: return path = os.path.abspath(config.main.deck_dir) + "/*" diff --git a/oonib/inputs/handlers.py b/oonib/inputs/handlers.py index 722a757..5391881 100644 --- a/oonib/inputs/handlers.py +++ b/oonib/inputs/handlers.py @@ -1,10 +1,12 @@ +from oonib.api import OONIBHandler + from oonib import config -from cyclone import web + import json import os import yaml -class InputsDescHandler(web.RequestHandler): +class InputsDescHandler(OONIBHandler): def get(self, inputID): #XXX return the input descriptor # see oonib.md in ooni-spec @@ -20,7 +22,7 @@ class InputsDescHandler(web.RequestHandler): except Exception: log.err("No Input Descriptor found for id %s" % inputID) -class InputsListHandler(web.RequestHandler): +class InputsListHandler(OONIBHandler): def get(self): if not config.main.input_dir: return path = os.path.abspath(config.main.input_dir) + "/*" @@ -29,7 +31,10 @@ class InputsListHandler(web.RequestHandler): for inputname in inputnames: f = open(os.path.join(config.main.input_dir, deckname)) d = yaml.safe_load(f) - inputList.append({'id': inputname, 'name': d['name'], - 'description': d['description']}) + inputList.append({ + 'id': inputname, + 'name': d['name'], + 'description': d['description'] + }) f.close() self.write(json.dumps(inputList)) diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index 2b55fb7..ee6f089 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -1,15 +1,20 @@ -from cyclone import web +from oonib.api import OONIBHandler + from oonib import config import json import os import yaml -class NetTestPolicyHandler(web.RequestHandler): +class NetTestPolicyHandler(OONIBHandler): def get(self): - #XXX: returns a list of accepted NetTests + """ + returns a list of accepted NetTests + """ pass -class InputPolicyHandler(web.RequestHandler): +class InputPolicyHandler(OONIBHandler): def get(self): + """ + return list of input ids + """ pass - #XXX return list of input ids diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index c5e4a97..b79898c 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -5,7 +5,8 @@ import string import time import yaml -from cyclone import web +from oonib.api import OONIBHandler + from datetime import datetime from oonib import randomStr, otime, config, log from twisted.internet import fdesc, reactor @@ -117,7 +118,7 @@ def stale_check(report_id): except ReportNotFound: pass -class NewReportHandlerFile(web.RequestHandler): +class NewReportHandlerFile(OONIBHandler): """ Responsible for creating and updating reports by writing to flat file. """ @@ -165,9 +166,9 @@ class NewReportHandlerFile(web.RequestHandler): try: report_data = parseNewReportRequest(self.request.body) except InvalidRequestField, e: - raise web.HTTPError(400, "Invalid Request Field %s" % e) + raise OONIBError(400, "Invalid Request Field %s" % e) except MissingField, e: - raise web.HTTPError(400, "Missing Request Field %s" % e) + raise OONIBError(400, "Missing Request Field %s" % e) print "Parsed this data %s" % report_data software_name = report_data['software_name'] @@ -182,10 +183,10 @@ class NewReportHandlerFile(web.RequestHandler): report_header = validate_report_header(content) except MissingReportHeaderKey, key: - raise web.HTTPError(406, "Missing report header key %s" % key) + raise OONIBError(406, "Missing report header key %s" % key) except InvalidReportHeader, key: - raise web.HTTPError(406, "Invalid report header %s" % key) + raise OONIBError(406, "Invalid report header %s" % key) report_header = yaml.dump(report_header) content = "---\n" + report_header + '...\n' @@ -246,7 +247,7 @@ class NewReportHandlerFile(web.RequestHandler): fdesc.setNonBlocking(fd.fileno()) fdesc.writeToFD(fd.fileno(), data) except IOError as e: - web.HTTPError(404, "Report not found") + OONIBError(404, "Report not found") class ReportNotFound(Exception): pass @@ -274,7 +275,7 @@ def close_report(report_id): dst_path = os.path.join(dst_path, dst_filename) os.rename(report_filename, dst_path) -class CloseReportHandlerFile(web.RequestHandler): +class CloseReportHandlerFile(OONIBHandler): def get(self): pass @@ -282,9 +283,9 @@ class CloseReportHandlerFile(web.RequestHandler): try: close_report(report_id) except ReportNotFound: - web.HTTPError(404, "Report not found") + OONIBError(404, "Report not found") -class PCAPReportHandler(web.RequestHandler): +class PCAPReportHandler(OONIBHandler): def get(self): pass diff --git a/oonib/runner.py b/oonib/runner.py index 8e89f32..3bb064e 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -25,6 +25,7 @@ from oonib.deck.api import deckAPI from oonib.inputs.api import inputsAPI from oonib.policy.api import policyAPI from oonib.bouncer.api import bouncerAPI +from oonib.api import oonibAPI from oonib import oonibackend from oonib import config @@ -61,22 +62,7 @@ def setupCollector(tor_process_protocol): #XXX: also set up a separate keyed hidden service for collectors to push their status to, if the bouncer is enabled hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, data_dir=datadir) - enabledAPIs = [] - enabledAPIs += reportAPI - - if config.inputs_dir: - enabledAPIs += inputsAPI - - if config.deck_dir: - enabledAPIs += deckAPI - - if config.policy_file: - enabledAPIs += policyAPI - - if config.bouncer_file: - enabledAPIs += bouncerAPI - - hidden_service = hs_endpoint.listen(enabledAPIs) + hidden_service = hs_endpoint.listen(oonibAPI) hidden_service.addCallback(setup_complete) hidden_service.addErrback(txSetupFailed)