commit c0cad1259227019150258b5dfc2bc8d4844590a1 Author: Arturo Filastò art@fuffa.org Date: Wed Apr 30 14:57:15 2014 +0200

Refactoring and tidying up of various different parts of the codebase.

Let pylint guide the way! --- oonib/bouncer/handlers.py | 17 ++-- oonib/config.py | 11 ++- oonib/daphn3.py | 4 +- oonib/deck/api.py | 4 +- oonib/deck/handlers.py | 10 ++- oonib/errors.py | 50 ++++++++++-- oonib/handlers.py | 4 + oonib/input/api.py | 4 +- oonib/input/handlers.py | 21 +++-- oonib/log.py | 4 +- oonib/options.py | 14 ++-- oonib/otime.py | 24 ++++-- oonib/policy/handlers.py | 7 +- oonib/report/handlers.py | 155 +++++++++++++++++++------------------ oonib/runner.py | 27 ++++--- oonib/testhelpers/dns_helpers.py | 17 ++-- oonib/testhelpers/http_helpers.py | 9 ++- oonib/testhelpers/ssl_helpers.py | 2 +- oonib/testhelpers/tcp_helpers.py | 11 ++- 19 files changed, 225 insertions(+), 170 deletions(-)

diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 0a5b51f..81d13f3 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -1,10 +1,11 @@ -import json +import json import random import yaml from oonib import errors as e from oonib.handlers import OONIBHandler from oonib.config import config

+ class Bouncer(object): def __init__(self): with open(config.main.bouncer_file) as f: @@ -20,7 +21,7 @@ class Bouncer(object): for collectorName, helpers in bouncerFile['collector'].items(): if collectorName not in self.knownCollectors: self.knownCollectors.append(collectorName) - + def updateKnownHelpers(self, bouncerFile): self.knownHelpers = {} for collectorName, helpers in bouncerFile['collector'].items(): @@ -47,13 +48,13 @@ class Bouncer(object): helpers = self.knownHelpers[helper_name] except KeyError: raise e.TestHelperNotFound - + helpers_dict = {} for helper in helpers: helpers_dict[helper['collector-name']] = helper['helper-address']

return helpers_dict - + def filterHelperAddresses(self, requested_helpers): """ Returns a dict of collectors that support all the requested_helpers. @@ -79,7 +80,7 @@ class Bouncer(object): } }

- or + or

{'error': 'test-helper-not-found'}

@@ -105,9 +106,11 @@ class Bouncer(object): response = {'error': 'test-helper-not-found'} return response

- response['default'] = {'collector': random.choice(self.knownCollectors)} + response['default'] = {'collector': + random.choice(self.knownCollectors)} return response

+ class BouncerQueryHandler(OONIBHandler): def initialize(self): self.bouncer = Bouncer() @@ -122,7 +125,7 @@ class BouncerQueryHandler(OONIBHandler): requested_helpers = query['test-helpers'] except KeyError: raise e.TestHelpersKeyMissing - + if not isinstance(requested_helpers, list): raise e.InvalidRequest

diff --git a/oonib/config.py b/oonib/config.py index 1388e6b..31128a9 100644 --- a/oonib/config.py +++ b/oonib/config.py @@ -6,17 +6,16 @@ from oonib import __version__ from oonib.options import OONIBOptions import os

+ class Config(object): - main = None - helpers = None + main = {} + helpers = {} reports = {} backend_version = __version__ opts = OONIBOptions()

- def __init__(self): - self.opts.parseOptions() - def load(self): + self.opts.parseOptions() try: config_file = self.opts['config'] except KeyError: @@ -35,7 +34,7 @@ class Config(object): self.helpers[name] = Storage(helper.items())

self.check_paths() - + def check_paths(self): def check_path(directory, complaint): if not (directory and os.path.isdir(directory)): diff --git a/oonib/daphn3.py b/oonib/daphn3.py index 9c5f290..4b4eac4 100644 --- a/oonib/daphn3.py +++ b/oonib/daphn3.py @@ -96,7 +96,7 @@ def daphn3Mutate(steps, step_idx, mutation_idx): if idx == step_idx: step_string = step.values()[0] step_key = step.keys()[0] - mutated_string = daphn3MutateString(step_string, + mutated_string = daphn3MutateString(step_string, mutation_idx) mutated_steps.append({step_key: mutated_string}) else: @@ -121,7 +121,7 @@ class Daphn3Protocol(protocol.Protocol): def _current_step_data(self): step_idx, mutation_idx = self.factory.mutation log.debug("Mutating %s %s" % (step_idx, mutation_idx)) - mutated_step = daphn3Mutate(self.steps, + mutated_step = daphn3Mutate(self.steps, step_idx, mutation_idx) log.debug("Mutated packet into %s" % mutated_step) return mutated_step[self.current_step].values()[0] diff --git a/oonib/deck/api.py b/oonib/deck/api.py index a2f1164..1c19d32 100644 --- a/oonib/deck/api.py +++ b/oonib/deck/api.py @@ -5,6 +5,6 @@ from oonib.config import config deckAPI = [ (r"/deck", handlers.DeckListHandler), (r"/deck/([a-z0-9]{64})$", handlers.DeckDescHandler), - (r"/deck/([a-z0-9]{64})/file$", web.StaticFileHandler, {"path": - config.main.deck_dir}), + (r"/deck/([a-z0-9]{64})/file$", web.StaticFileHandler, + {"path": config.main.deck_dir}), ] diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index b9589b0..ca17e95 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -1,5 +1,4 @@ import glob -import json import os import re import yaml @@ -9,6 +8,7 @@ from oonib.handlers import OONIBHandler from oonib import log from oonib.config import config

+ class DeckDescHandler(OONIBHandler): def get(self, deckID): # note: @@ -32,18 +32,20 @@ class DeckDescHandler(OONIBHandler): raise e.MissingDeckKeys self.write(response)

+ class DeckListHandler(OONIBHandler): def get(self): - if not config.main.deck_dir: + if not config.main.deck_dir: self.set_status(501) raise e.NoDecksConfigured

path = os.path.abspath(config.main.deck_dir) + "/*" decknames = map(os.path.basename, glob.iglob(path)) - decknames = filter(lambda y: re.match("[a-z0-9]{64}.desc", y), decknames) + decknames = filter(lambda y: re.match("[a-z0-9]{64}.desc", y), + decknames) deckList = [] for deckname in decknames: - with open(os.path.join(config.main.deck_dir, deckname)) as f: + with open(os.path.join(config.main.deck_dir, deckname)) as f: d = yaml.safe_load(f) deckList.append({ 'id': deckname, diff --git a/oonib/errors.py b/oonib/errors.py index 6f768d0..3f1167a 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -1,87 +1,125 @@ from cyclone.web import HTTPError

+ class OONIBError(HTTPError): status_code = 500 log_message = 'oonib-error' + def __init__(self): pass

+ class InvalidRequest(OONIBError): status_code = 400 log_message = 'invalid-request'

+ class NoHelperFound(OONIBError): status_code = 404 log_message = 'no-helper-found'

+ class InvalidInputHash(OONIBError): status_code = 406 log_message = 'invalid-input-hash'

+ class InvalidNettestName(OONIBError): status_code = 406 log_message = 'invalid-nettest-name'

+ class InputHashNotProvided(OONIBError): status_code = 406 log_message = 'input-hash-not-provided'

+ +class InputDescriptorNotFound(OONIBError): + status_code = 404 + log_message = 'input-descriptor-not-found' + + class InvalidRequestField(OONIBError): def __init__(self, field_name): self.status_code = 400 self.log_message = "invalid-request-field %s" % field_name

+ class MissingRequestField(OONIBError): def __init__(self, field_name): self.status_code = 400 self.log_message = "missing-request-field %s" % field_name

+ class MissingReportHeaderKey(OONIBError): def __init__(self, key): self.status_code = 406 self.log_message = "missing-report-header-key %s" % key

+ class MissingDeckKeys(OONIBError): status_code = 400 log_message = "missing-deck-keys"

+ class MissingDeck(OONIBError): status_code = 400 log_message = "missing-deck"

+ class NoDecksConfigured(OONIBError): status_code = 501 log_message = "no-decks-configured"

+ class InvalidReportHeader(OONIBError): def __init__(self, key): self.status_code = 406 self.log_message = "invalid-report-header %s" % key

+ class ReportNotFound(OONIBError): status_code = 404 log_message = "report-not-found"

+ class NoValidCollector(OONIBError): pass

+ class TestHelpersKeyMissing(OONIBError): status_code = 400 log_message = "test-helpers-key-missing"

+ class TestHelperNotFound(OONIBError): status_code = 404 log_message = "test-helper-not-found"

-class ConfigFileNotSpecified(Exception): pass

-class ConfigFileDoesNotExist(Exception): pass +class ConfigFileNotSpecified(Exception): + pass

-class InvalidReportDirectory(Exception): pass

-class InvalidArchiveDirectory(Exception): pass +class ConfigFileDoesNotExist(Exception): + pass

-class InvalidInputDirectory(Exception): pass

-class InvalidDeckDirectory(Exception): pass +class InvalidReportDirectory(Exception): + pass + + +class InvalidArchiveDirectory(Exception): + pass

+ +class InvalidInputDirectory(Exception): + pass + + +class InvalidDeckDirectory(Exception): + pass + + +class InvalidTimestampFormat(Exception): + pass diff --git a/oonib/handlers.py b/oonib/handlers.py index 20c831b..f92d684 100644 --- a/oonib/handlers.py +++ b/oonib/handlers.py @@ -3,12 +3,16 @@ import types from cyclone import escape from cyclone import web

+from oonib import log + + class OONIBHandler(web.RequestHandler): def write_error(self, status_code, exception=None, **kw): self.set_status(status_code) if hasattr(exception, 'log_message'): self.write({'error': exception.log_message}) else: + log.error(exception) self.write({'error': 'error'})

def write(self, chunk): diff --git a/oonib/input/api.py b/oonib/input/api.py index 811da29..32ecd10 100644 --- a/oonib/input/api.py +++ b/oonib/input/api.py @@ -5,6 +5,6 @@ from oonib.config import config inputAPI = [ (r"/input", handlers.InputListHandler), (r"/input/([a-f0-9]{64})", handlers.InputDescHandler), - (r"/input/([a-f0-9]{64})/file$", web.StaticFileHandler, {"path": - config.main.input_dir}), + (r"/input/([a-f0-9]{64})/file$", web.StaticFileHandler, + {"path": config.main.input_dir}), ] diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py index 33bec55..53501fa 100644 --- a/oonib/input/handlers.py +++ b/oonib/input/handlers.py @@ -1,38 +1,35 @@ import glob -import json import os import yaml

from oonib.handlers import OONIBHandler from oonib import log +from oonib import errors as e from oonib.config import config

+ class InputDescHandler(OONIBHandler): def get(self, inputID): bn = os.path.basename(inputID) + ".desc" try: f = open(os.path.join(config.main.input_dir, bn)) except IOError: - log.err("No Input Descriptor found for id %s" % inputID) - self.set_status(404) - self.write({'error': 'missing-input'}) - return + log.err("No Input Descriptor found for id %s" % inputID) + raise e.InputDescriptorNotFound with f: inputDesc = yaml.safe_load(f) - + response = {'id': inputID} for k in ['name', 'description', 'version', 'author', 'date']: try: response[k] = inputDesc[k] - except Exception, e: # XXX this should probably be KeyError - log.exception(e) - log.err("Invalid Input Descriptor found for id %s" % inputID) - self.set_status(500) - self.write({'error': 'invalid-input-descriptor'}) - return + except KeyError: + log.err("Invalid Input Descriptor found for id %s" % inputID) + raise e.InputDescriptorNotFound

self.write(response)

+ class InputListHandler(OONIBHandler): def get(self): path = os.path.abspath(config.main.input_dir) + "/*.desc" diff --git a/oonib/log.py b/oonib/log.py index 7a6e4cb..36fe414 100644 --- a/oonib/log.py +++ b/oonib/log.py @@ -87,8 +87,8 @@ def msg(msg, *arg, **kw):

def debug(msg, *arg, **kw): - if config.main.debug: - print "[D] %s" % log_encode(msg) + if config.main.get('debug'): + print "[D] %s" % msg

def warn(msg, *arg, **kw): diff --git a/oonib/options.py b/oonib/options.py index d5c87b3..9abb355 100644 --- a/oonib/options.py +++ b/oonib/options.py @@ -1,14 +1,12 @@ from twisted.python import usage -from twisted.python.runtime import platformType

-if platformType == "win32": - from twisted.scripts._twistw import ServerOptions -else: - from twisted.scripts._twistd_unix import ServerOptions

class OONIBOptions(usage.Options): - synopsis = """%s [options] [path to test].py """ + synopsis = """%s [options]""" + + longdesc = ("oonib provides the backend component of ooni-probe." + "oonib provides test helper services and a reporting " + "backend service. oonib is intended to be run as a " + "daemon and most options are set in its configuration file")

- longdesc = ("oonib provides the backend component of ooni-probe. oonib provides test helper services and a reporting backend service. oonib is intended to be run as a daemon and most options are set in its configuration file") - optParameters = [["config", "c", "oonib.conf", "Path to config file"]] diff --git a/oonib/otime.py b/oonib/otime.py index e38089b..67a6bc6 100644 --- a/oonib/otime.py +++ b/oonib/otime.py @@ -1,41 +1,46 @@ import time +from oonib import errors as e from datetime import datetime

+ def utcDateNow(): """ Returns the datetime object of the current UTC time. """ return datetime.utcnow()

+ def utcTimeNow(): """ Returns seconds since epoch in UTC time, it's of type float. """ return time.mktime(time.gmtime())

+ def dateToTime(date): """ Takes as input a datetime object and outputs the seconds since epoch. """ return time.mktime(date.timetuple())

+ def prettyDateNow(): """ Returns a good looking string for the local time. """ return datetime.now().ctime()

+ def utcPrettyDateNow(): """ Returns a good looking string for utc time. """ return datetime.utcnow().ctime()

+ def timeToPrettyDate(time_val): return time.ctime(time_val)

-class InvalidTimestampFormat(Exception): - pass

def fromTimestamp(s): """ @@ -47,17 +52,23 @@ def fromTimestamp(s): ex. 1912-06-23T101234Z"

Note: we currently only support parsing strings that are generated from the - timestamp function and have no intention in supporting the full standard. + timestamp function and have no intention in supporting the full + standard. """ try: date_part, time_part = s.split('T') hours, minutes, seconds = time_part[:2], time_part[2:4], time_part[4:6] year, month, day = date_part.split('-') except: - raise InvalidTimestampFormat(s) + raise e.InvalidTimestampFormat(s) + + return datetime(int(year), + int(month), + int(day), + int(hours), + int(minutes), + int(seconds))

- return datetime(int(year), int(month), int(day), int(hours), int(minutes), - int(seconds))

def timestamp(t=None): """ @@ -86,4 +97,3 @@ def timestamp(t=None): t = datetime.utcnow() ISO8601 = "%Y-%m-%dT%H%M%SZ" return t.strftime(ISO8601) - diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index cbf1c08..e81dd21 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -1,5 +1,3 @@ -import json -import os import yaml

from oonib import errors as e @@ -7,6 +5,7 @@ from oonib.handlers import OONIBHandler

from oonib.config import config

+ class Policy(object): nettest = None input = None @@ -38,10 +37,12 @@ class Policy(object): if not any(nt['name'] == nettest_name for nt in self.nettest): raise e.InvalidNettestName

+ class PolicyHandler(OONIBHandler): def initialize(self): self.policy = Policy()

+ class NetTestPolicyHandler(PolicyHandler): def get(self): """ @@ -49,10 +50,10 @@ class NetTestPolicyHandler(PolicyHandler): """ self.write(self.policy.nettest)

+ class InputPolicyHandler(PolicyHandler): def get(self): """ return list of input ids """ self.write(self.policy.input) - diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 34aa8d9..01a665a 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -1,5 +1,3 @@ -import random -import string import time import yaml import json @@ -16,35 +14,48 @@ from datetime import datetime from oonib import randomStr, otime, log from oonib.config import config

-class MissingField(Exception): - pass

-class InvalidRequestField(Exception): - pass +def report_file_name(report_details): + timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time'])) + dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( + timestamp=timestamp, + **report_details) + return dst_filename

class Report(object): - def __init__(self, report_id): + delayed_call = None + + def __init__(self, report_id, + stale_time, + report_dir, + archive_dir, + reports): self.report_id = report_id - self.delayed_call = None + + self.stale_time = stale_time + self.report_dir = report_dir + self.archive_dir = archive_dir + self.reports = reports

self.refresh() - + def refresh(self): self.last_updated = time.time() if self.delayed_call: self.delayed_call.cancel() - self.delayed_call = reactor.callLater(config.main.stale_time, self.stale_check) + self.delayed_call = reactor.callLater(self.stale_time, + self.stale_check)

def stale_check(self): - if (time.time() - self.last_updated) > config.main.stale_time: + if (time.time() - self.last_updated) > self.stale_time: try: self.close() - except ReportNotFound: + except e.ReportNotFound: pass

def close(self): def get_report_path(report_id): - return os.path.join(config.main.report_dir, report_id) + return os.path.join(self.report_dir, report_id)

report_filename = get_report_path(self.report_id) try: @@ -52,14 +63,10 @@ class Report(object): g = yaml.safe_load_all(fd) report_details = g.next() except IOError: - raise ReportNotFound - - timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time'])) - dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( - timestamp=timestamp, - **report_details) + raise e.ReportNotFound

- dst_path = os.path.join(config.main.archive_dir, + dst_filename = report_file_name(report_details) + dst_path = os.path.join(self.archive_dir, report_details['probe_cc'])

if not os.path.isdir(dst_path): @@ -68,7 +75,8 @@ class Report(object): dst_path = os.path.join(dst_path, dst_filename) os.rename(report_filename, dst_path)

- del config.reports[self.report_id] + self.delayed_call.cancel() + del self.reports[self.report_id]

def parseUpdateReportRequest(request): #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") @@ -85,10 +93,10 @@ def parseUpdateReportRequest(request): try: report_id = parsed_request['report_id'] except KeyError: - raise MissingField('report_id') + raise e.MissingField('report_id')

if not re.match(report_id_regexp, report_id): - raise InvalidRequestField('report_id') + raise e.InvalidRequestField('report_id')

return parsed_request

@@ -117,79 +125,82 @@ def parseNewReportRequest(request): try: value_to_check = parsed_request[k] except KeyError: - raise MissingField(k) + raise e.MissingField(k)

print "Matching %s with %s | %s" % (regexp, value_to_check, k) if re.match(regexp, str(value_to_check)): continue else: - raise InvalidRequestField(k) - + raise e.InvalidRequestField(k) + try: requested_test_helper = parsed_request['test_helper'] if not re.match(test_helper, str(requested_test_helper)): - raise InvalidRequestField('test_helper') + raise e.InvalidRequestField('test_helper') except KeyError: pass

return parsed_request

-class InvalidReportHeader(Exception): - pass - -class MissingReportHeaderKey(InvalidReportHeader): - pass - def validate_report_header(report_header): required_keys = ['probe_asn', 'probe_cc', 'probe_ip', 'software_name', 'software_version', 'test_name', 'test_version'] for key in required_keys: if key not in report_header: - raise MissingReportHeaderKey(key) + raise e.MissingReportHeaderKey(key)

if report_header['probe_asn'] is None: report_header['probe_asn'] = 'AS0'

if not re.match('AS[0-9]+$', report_header['probe_asn']): - raise InvalidReportHeader('probe_asn') + raise e.InvalidReportHeader('probe_asn')

# If no country is known, set it to be ZZ (user assigned value in ISO 3166) if report_header['probe_cc'] is None: report_header['probe_cc'] = 'ZZ'

if not re.match('[a-zA-Z]{2}$', report_header['probe_cc']): - raise InvalidReportHeader('probe_cc') + raise e.InvalidReportHeader('probe_cc')

if not re.match('[a-z_-]+$', report_header['test_name']): - raise InvalidReportHeader('test_name') + raise e.InvalidReportHeader('test_name')

if not re.match('([0-9]+.)+[0-9]+$', report_header['test_version']): - raise InvalidReportHeader('test_version') + raise e.InvalidReportHeader('test_version')

return report_header

+class ReportHandler(OONIBHandler): + def initialize(self): + self.archive_dir = config.main.archive_dir + self.report_dir = config.main.report_dir + self.reports = config.reports + self.policy_file = config.main.policy_file + self.helpers = config.helpers + self.stale_time = config.main.stale_time + class UpdateReportMixin(object): def updateReport(self, report_id, parsed_request):

log.debug("Got this request %s" % parsed_request) - report_filename = os.path.join(config.main.report_dir, + report_filename = os.path.join(self.report_dir, report_id) - - config.reports[report_id].refresh() + + self.reports[report_id].refresh()

try: with open(report_filename, 'a+') as fd: - fdesc.setNonBlocking(fd.fileno()) - fdesc.writeToFD(fd.fileno(), parsed_request['content']) - except IOError as exc: + fd.write(parsed_request['content']) + except IOError: e.OONIBError(404, "Report not found") - self.write({}) + self.write({'status': 'success'})

-class NewReportHandlerFile(OONIBHandler, UpdateReportMixin): +class NewReportHandlerFile(ReportHandler, UpdateReportMixin): """ Responsible for creating and updating reports by writing to flat file. """ + inputHashes = None

def checkPolicy(self): policy = Policy() @@ -236,42 +247,31 @@ class NewReportHandlerFile(OONIBHandler, UpdateReportMixin): {'backend_version': 'XXX', 'report_id': 'XXX'}

""" - # XXX here we should validate and sanitize the request - try: - report_data = parseNewReportRequest(self.request.body) - except InvalidRequestField as exc: - raise e.InvalidRequestField(exc) - except MissingField as exc: - raise e.MissingRequestField(exc) + # Note: the request is being validated inside of parseNewReportRequest. + report_data = parseNewReportRequest(self.request.body)

log.debug("Parsed this data %s" % report_data)

software_name = str(report_data['software_name']) software_version = str(report_data['software_version']) - + probe_asn = str(report_data['probe_asn']) probe_cc = str(report_data.get('probe_cc', 'ZZ'))

self.testName = str(report_data['test_name']) self.testVersion = str(report_data['test_version']) - - if config.main.policy_file: + + if self.policy_file: try: self.inputHashes = report_data['input_hashes'] except KeyError: raise e.InputHashNotProvided self.checkPolicy() - + if 'content' in report_data: content = yaml.safe_load(report_data['content']) - try: - report_header = validate_report_header(content) - - except MissingReportHeaderKey, key: - raise e.MissingReportHeaderKey(key) + report_header = validate_report_header(content)

- except InvalidReportHeader, key: - raise e.InvalidReportHeader(key) else: content = { 'software_name': software_name, @@ -298,22 +298,26 @@ class NewReportHandlerFile(OONIBHandler, UpdateReportMixin):

# The report filename contains the timestamp of the report plus a # random nonce - report_filename = os.path.join(config.main.report_dir, report_id) + report_filename = os.path.join(self.report_dir, report_id)

response = { 'backend_version': config.backend_version, 'report_id': report_id } - + requested_helper = report_data.get('test_helper')

if requested_helper: try: - response['test_helper_address'] = config.helpers[requested_helper].address + response['test_helper_address'] = self.helpers[requested_helper].address except KeyError: raise e.TestHelperNotFound - - config.reports[report_id] = Report(report_id) + + self.reports[report_id] = Report(report_id, + self.stale_time, + self.report_dir, + self.archive_dir, + self.reports)

self.writeToReport(report_filename, content)

@@ -338,7 +342,7 @@ class NewReportHandlerFile(OONIBHandler, UpdateReportMixin):

self.updateReport(report_id, parsed_request)

-class UpdateReportHandlerFile(OONIBHandler, UpdateReportMixin): +class UpdateReportHandlerFile(ReportHandler, UpdateReportMixin): def post(self, report_id): try: parsed_request = json.loads(self.request.body) @@ -346,20 +350,17 @@ class UpdateReportHandlerFile(OONIBHandler, UpdateReportMixin): raise e.InvalidRequest self.updateReport(report_id, parsed_request)

-class ReportNotFound(Exception): - pass - -class CloseReportHandlerFile(OONIBHandler): +class CloseReportHandlerFile(ReportHandler): def get(self): pass

def post(self, report_id): - if report_id in config.reports: - config.reports[report_id].close() + if report_id in self.reports: + self.reports[report_id].close() else: raise e.ReportNotFound

-class PCAPReportHandler(OONIBHandler): +class PCAPReportHandler(ReportHandler): def get(self): pass

diff --git a/oonib/runner.py b/oonib/runner.py index c8fafa1..d413221 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -1,7 +1,3 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò, Isis Lovecruft -# :licence: see LICENSE for details """ In here we define a runner for the oonib backend system. """ @@ -13,14 +9,12 @@ import os

from shutil import rmtree

-from twisted.internet import reactor -from twisted.application import service, internet, app +from twisted.internet import reactor, endpoints from twisted.python.runtime import platformType

from txtorcon import TCPHiddenServiceEndpoint, TorConfig from txtorcon import launch_tor

-from oonib.report.api import reportAPI from oonib.api import ooniBackend, ooniBouncer from oonib.config import config

@@ -28,15 +22,16 @@ from oonib import oonibackend from oonib import log

from txtorcon import __version__ as txtorcon_version -if tuple(map(int, txtorcon_version.split('.'))) < (0,9,0): +if tuple(map(int, txtorcon_version.split('.'))) < (0, 9, 0): """ Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint listens on all interfaces by default. """ def create_listener(self, proto): self._update_onion(self.hiddenservice.dir) - self.tcp_endpoint = TCP4ServerEndpoint(self.reactor, self.listen_port, # XXX missing import -- has this code ever been tested? - interface='127.0.0.1') + self.tcp_endpoint = endpoints.TCP4ServerEndpoint(self.reactor, + self.listen_port, + interface='127.0.0.1') d = self.tcp_endpoint.listen(self.protocolfactory) d.addCallback(self._add_attributes).addErrback(self._retry_local_port) return d @@ -64,13 +59,17 @@ else:

def setupHSEndpoint(self, tor_process_protocol, torconfig, endpoint): endpointName = endpoint.settings['name'] + def setup_complete(port): - print("Exposed %s Tor hidden service on httpo://%s" % (endpointName, - port.onion_uri)) + print("Exposed %s Tor hidden service " + "on httpo://%s" % (endpointName, port.onion_uri))

public_port = 80 - hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, - data_dir=os.path.join(torconfig.DataDirectory, endpointName)) + data_dir = os.path.join(torconfig.DataDirectory, endpointName) + hs_endpoint = TCPHiddenServiceEndpoint(reactor, + torconfig, + public_port, + data_dir=data_dir) d = hs_endpoint.listen(endpoint) d.addCallback(setup_complete) d.addErrback(self.txSetupFailed) diff --git a/oonib/testhelpers/dns_helpers.py b/oonib/testhelpers/dns_helpers.py index 9b2e5e3..fbf932b 100644 --- a/oonib/testhelpers/dns_helpers.py +++ b/oonib/testhelpers/dns_helpers.py @@ -1,14 +1,12 @@ -from twisted.internet.protocol import Factory, Protocol -from twisted.internet import reactor -from twisted.names import dns from twisted.names import client, server

from oonib.config import config

+ class DNSTestHelper(server.DNSServerFactory): - def __init__(self, authorities = None, - caches = None, clients = None, - verbose = 0): + def __init__(self, authorities=None, + caches=None, clients=None, + verbose=0): try: host, port = config.helpers.dns.split(':') port = int(port) @@ -17,8 +15,9 @@ class DNSTestHelper(server.DNSServerFactory): except: host, port = '8.8.8.8', 53 resolver = client.Resolver(servers=[(host, port)]) - server.DNSServerFactory.__init__(self, authorities = authorities, - caches = caches, clients = [resolver], - verbose = verbose) + server.DNSServerFactory.__init__(self, authorities=authorities, + caches=caches, clients=[resolver], + verbose=verbose) + def handleQuery(self, message, protocol, address): server.DNSServerFactory.handleQuery(self, message, protocol, address) diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py index 245e08a..b2afc59 100644 --- a/oonib/testhelpers/http_helpers.py +++ b/oonib/testhelpers/http_helpers.py @@ -93,10 +93,11 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): headers_dict[k] = [] headers_dict[k].append(v)

- response = {'request_headers': self.headers, - 'request_line': self.requestLine, - 'headers_dict': headers_dict - } + response = { + 'request_headers': self.headers, + 'request_line': self.requestLine, + 'headers_dict': headers_dict + } json_response = json.dumps(response) self.transport.write('HTTP/1.1 200 OK\r\n\r\n') self.transport.write('%s' % json_response) diff --git a/oonib/testhelpers/ssl_helpers.py b/oonib/testhelpers/ssl_helpers.py index d511ddd..4b84a8c 100644 --- a/oonib/testhelpers/ssl_helpers.py +++ b/oonib/testhelpers/ssl_helpers.py @@ -3,7 +3,7 @@ from oonib.config import config

class SSLContext(ssl.DefaultOpenSSLContextFactory): def __init__(self, *args, **kw): - ssl.DefaultOpenSSLContextFactory.__init__(self, + ssl.DefaultOpenSSLContextFactory.__init__(self, config.helpers.ssl.private_key, config.helpers.ssl.certificate)

diff --git a/oonib/testhelpers/tcp_helpers.py b/oonib/testhelpers/tcp_helpers.py index 91c334b..614cf87 100644 --- a/oonib/testhelpers/tcp_helpers.py +++ b/oonib/testhelpers/tcp_helpers.py @@ -1,16 +1,16 @@ - from twisted.internet.protocol import Protocol, Factory, ServerFactory -from twisted.internet.error import ConnectionDone

from oonib.config import config from oonib import log from oonib.daphn3 import Daphn3Protocol from oonib.daphn3 import read_pcap, read_yaml

+ class TCPEchoProtocol(Protocol): def dataReceived(self, data): self.transport.write(data)

+ class TCPEchoHelper(Factory): """ A very simple echo protocol implementation @@ -24,8 +24,11 @@ elif config.helpers.daphn3.pcap_file: daphn3Steps = read_yaml(config.helpers.daphn3.pcap_file)

else: - daphn3Steps = [{'client': 'client_packet'}, - {'server': 'server_packet'}] + daphn3Steps = [ + {'client': 'client_packet'}, + {'server': 'server_packet'} + ] +

class Daphn3ServerProtocol(Daphn3Protocol): def nextStep(self):