[tor-commits] [oonib/master] Refactoring and tidying up of various different parts of the codebase.
art at torproject.org
art at torproject.org
Wed Apr 30 17:33:44 UTC 2014
commit c0cad1259227019150258b5dfc2bc8d4844590a1
Author: Arturo Filastò <art at fuffa.org>
Date: Wed Apr 30 14:57:15 2014 +0200
Refactoring and tidying up of various different parts of the codebase.
Let pylint guide the way!
---
oonib/bouncer/handlers.py | 17 ++--
oonib/config.py | 11 ++-
oonib/daphn3.py | 4 +-
oonib/deck/api.py | 4 +-
oonib/deck/handlers.py | 10 ++-
oonib/errors.py | 50 ++++++++++--
oonib/handlers.py | 4 +
oonib/input/api.py | 4 +-
oonib/input/handlers.py | 21 +++--
oonib/log.py | 4 +-
oonib/options.py | 14 ++--
oonib/otime.py | 24 ++++--
oonib/policy/handlers.py | 7 +-
oonib/report/handlers.py | 155 +++++++++++++++++++------------------
oonib/runner.py | 27 ++++---
oonib/testhelpers/dns_helpers.py | 17 ++--
oonib/testhelpers/http_helpers.py | 9 ++-
oonib/testhelpers/ssl_helpers.py | 2 +-
oonib/testhelpers/tcp_helpers.py | 11 ++-
19 files changed, 225 insertions(+), 170 deletions(-)
diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py
index 0a5b51f..81d13f3 100644
--- a/oonib/bouncer/handlers.py
+++ b/oonib/bouncer/handlers.py
@@ -1,10 +1,11 @@
-import json
+import json
import random
import yaml
from oonib import errors as e
from oonib.handlers import OONIBHandler
from oonib.config import config
+
class Bouncer(object):
def __init__(self):
with open(config.main.bouncer_file) as f:
@@ -20,7 +21,7 @@ class Bouncer(object):
for collectorName, helpers in bouncerFile['collector'].items():
if collectorName not in self.knownCollectors:
self.knownCollectors.append(collectorName)
-
+
def updateKnownHelpers(self, bouncerFile):
self.knownHelpers = {}
for collectorName, helpers in bouncerFile['collector'].items():
@@ -47,13 +48,13 @@ class Bouncer(object):
helpers = self.knownHelpers[helper_name]
except KeyError:
raise e.TestHelperNotFound
-
+
helpers_dict = {}
for helper in helpers:
helpers_dict[helper['collector-name']] = helper['helper-address']
return helpers_dict
-
+
def filterHelperAddresses(self, requested_helpers):
"""
Returns a dict of collectors that support all the requested_helpers.
@@ -79,7 +80,7 @@ class Bouncer(object):
}
}
- or
+ or
{'error': 'test-helper-not-found'}
@@ -105,9 +106,11 @@ class Bouncer(object):
response = {'error': 'test-helper-not-found'}
return response
- response['default'] = {'collector': random.choice(self.knownCollectors)}
+ response['default'] = {'collector':
+ random.choice(self.knownCollectors)}
return response
+
class BouncerQueryHandler(OONIBHandler):
def initialize(self):
self.bouncer = Bouncer()
@@ -122,7 +125,7 @@ class BouncerQueryHandler(OONIBHandler):
requested_helpers = query['test-helpers']
except KeyError:
raise e.TestHelpersKeyMissing
-
+
if not isinstance(requested_helpers, list):
raise e.InvalidRequest
diff --git a/oonib/config.py b/oonib/config.py
index 1388e6b..31128a9 100644
--- a/oonib/config.py
+++ b/oonib/config.py
@@ -6,17 +6,16 @@ from oonib import __version__
from oonib.options import OONIBOptions
import os
+
class Config(object):
- main = None
- helpers = None
+ main = {}
+ helpers = {}
reports = {}
backend_version = __version__
opts = OONIBOptions()
- def __init__(self):
- self.opts.parseOptions()
-
def load(self):
+ self.opts.parseOptions()
try:
config_file = self.opts['config']
except KeyError:
@@ -35,7 +34,7 @@ class Config(object):
self.helpers[name] = Storage(helper.items())
self.check_paths()
-
+
def check_paths(self):
def check_path(directory, complaint):
if not (directory and os.path.isdir(directory)):
diff --git a/oonib/daphn3.py b/oonib/daphn3.py
index 9c5f290..4b4eac4 100644
--- a/oonib/daphn3.py
+++ b/oonib/daphn3.py
@@ -96,7 +96,7 @@ def daphn3Mutate(steps, step_idx, mutation_idx):
if idx == step_idx:
step_string = step.values()[0]
step_key = step.keys()[0]
- mutated_string = daphn3MutateString(step_string,
+ mutated_string = daphn3MutateString(step_string,
mutation_idx)
mutated_steps.append({step_key: mutated_string})
else:
@@ -121,7 +121,7 @@ class Daphn3Protocol(protocol.Protocol):
def _current_step_data(self):
step_idx, mutation_idx = self.factory.mutation
log.debug("Mutating %s %s" % (step_idx, mutation_idx))
- mutated_step = daphn3Mutate(self.steps,
+ mutated_step = daphn3Mutate(self.steps,
step_idx, mutation_idx)
log.debug("Mutated packet into %s" % mutated_step)
return mutated_step[self.current_step].values()[0]
diff --git a/oonib/deck/api.py b/oonib/deck/api.py
index a2f1164..1c19d32 100644
--- a/oonib/deck/api.py
+++ b/oonib/deck/api.py
@@ -5,6 +5,6 @@ from oonib.config import config
deckAPI = [
(r"/deck", handlers.DeckListHandler),
(r"/deck/([a-z0-9]{64})$", handlers.DeckDescHandler),
- (r"/deck/([a-z0-9]{64})/file$", web.StaticFileHandler, {"path":
- config.main.deck_dir}),
+ (r"/deck/([a-z0-9]{64})/file$", web.StaticFileHandler,
+ {"path": config.main.deck_dir}),
]
diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py
index b9589b0..ca17e95 100644
--- a/oonib/deck/handlers.py
+++ b/oonib/deck/handlers.py
@@ -1,5 +1,4 @@
import glob
-import json
import os
import re
import yaml
@@ -9,6 +8,7 @@ from oonib.handlers import OONIBHandler
from oonib import log
from oonib.config import config
+
class DeckDescHandler(OONIBHandler):
def get(self, deckID):
# note:
@@ -32,18 +32,20 @@ class DeckDescHandler(OONIBHandler):
raise e.MissingDeckKeys
self.write(response)
+
class DeckListHandler(OONIBHandler):
def get(self):
- if not config.main.deck_dir:
+ if not config.main.deck_dir:
self.set_status(501)
raise e.NoDecksConfigured
path = os.path.abspath(config.main.deck_dir) + "/*"
decknames = map(os.path.basename, glob.iglob(path))
- decknames = filter(lambda y: re.match("[a-z0-9]{64}.desc", y), decknames)
+ decknames = filter(lambda y: re.match("[a-z0-9]{64}.desc", y),
+ decknames)
deckList = []
for deckname in decknames:
- with open(os.path.join(config.main.deck_dir, deckname)) as f:
+ with open(os.path.join(config.main.deck_dir, deckname)) as f:
d = yaml.safe_load(f)
deckList.append({
'id': deckname,
diff --git a/oonib/errors.py b/oonib/errors.py
index 6f768d0..3f1167a 100644
--- a/oonib/errors.py
+++ b/oonib/errors.py
@@ -1,87 +1,125 @@
from cyclone.web import HTTPError
+
class OONIBError(HTTPError):
status_code = 500
log_message = 'oonib-error'
+
def __init__(self):
pass
+
class InvalidRequest(OONIBError):
status_code = 400
log_message = 'invalid-request'
+
class NoHelperFound(OONIBError):
status_code = 404
log_message = 'no-helper-found'
+
class InvalidInputHash(OONIBError):
status_code = 406
log_message = 'invalid-input-hash'
+
class InvalidNettestName(OONIBError):
status_code = 406
log_message = 'invalid-nettest-name'
+
class InputHashNotProvided(OONIBError):
status_code = 406
log_message = 'input-hash-not-provided'
+
+class InputDescriptorNotFound(OONIBError):
+ status_code = 404
+ log_message = 'input-descriptor-not-found'
+
+
class InvalidRequestField(OONIBError):
def __init__(self, field_name):
self.status_code = 400
self.log_message = "invalid-request-field %s" % field_name
+
class MissingRequestField(OONIBError):
def __init__(self, field_name):
self.status_code = 400
self.log_message = "missing-request-field %s" % field_name
+
class MissingReportHeaderKey(OONIBError):
def __init__(self, key):
self.status_code = 406
self.log_message = "missing-report-header-key %s" % key
+
class MissingDeckKeys(OONIBError):
status_code = 400
log_message = "missing-deck-keys"
+
class MissingDeck(OONIBError):
status_code = 400
log_message = "missing-deck"
+
class NoDecksConfigured(OONIBError):
status_code = 501
log_message = "no-decks-configured"
+
class InvalidReportHeader(OONIBError):
def __init__(self, key):
self.status_code = 406
self.log_message = "invalid-report-header %s" % key
+
class ReportNotFound(OONIBError):
status_code = 404
log_message = "report-not-found"
+
class NoValidCollector(OONIBError):
pass
+
class TestHelpersKeyMissing(OONIBError):
status_code = 400
log_message = "test-helpers-key-missing"
+
class TestHelperNotFound(OONIBError):
status_code = 404
log_message = "test-helper-not-found"
-class ConfigFileNotSpecified(Exception): pass
-class ConfigFileDoesNotExist(Exception): pass
+class ConfigFileNotSpecified(Exception):
+ pass
-class InvalidReportDirectory(Exception): pass
-class InvalidArchiveDirectory(Exception): pass
+class ConfigFileDoesNotExist(Exception):
+ pass
-class InvalidInputDirectory(Exception): pass
-class InvalidDeckDirectory(Exception): pass
+class InvalidReportDirectory(Exception):
+ pass
+
+
+class InvalidArchiveDirectory(Exception):
+ pass
+
+class InvalidInputDirectory(Exception):
+ pass
+
+
+class InvalidDeckDirectory(Exception):
+ pass
+
+
+class InvalidTimestampFormat(Exception):
+ pass
diff --git a/oonib/handlers.py b/oonib/handlers.py
index 20c831b..f92d684 100644
--- a/oonib/handlers.py
+++ b/oonib/handlers.py
@@ -3,12 +3,16 @@ import types
from cyclone import escape
from cyclone import web
+from oonib import log
+
+
class OONIBHandler(web.RequestHandler):
def write_error(self, status_code, exception=None, **kw):
self.set_status(status_code)
if hasattr(exception, 'log_message'):
self.write({'error': exception.log_message})
else:
+ log.error(exception)
self.write({'error': 'error'})
def write(self, chunk):
diff --git a/oonib/input/api.py b/oonib/input/api.py
index 811da29..32ecd10 100644
--- a/oonib/input/api.py
+++ b/oonib/input/api.py
@@ -5,6 +5,6 @@ from oonib.config import config
inputAPI = [
(r"/input", handlers.InputListHandler),
(r"/input/([a-f0-9]{64})", handlers.InputDescHandler),
- (r"/input/([a-f0-9]{64})/file$", web.StaticFileHandler, {"path":
- config.main.input_dir}),
+ (r"/input/([a-f0-9]{64})/file$", web.StaticFileHandler,
+ {"path": config.main.input_dir}),
]
diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py
index 33bec55..53501fa 100644
--- a/oonib/input/handlers.py
+++ b/oonib/input/handlers.py
@@ -1,38 +1,35 @@
import glob
-import json
import os
import yaml
from oonib.handlers import OONIBHandler
from oonib import log
+from oonib import errors as e
from oonib.config import config
+
class InputDescHandler(OONIBHandler):
def get(self, inputID):
bn = os.path.basename(inputID) + ".desc"
try:
f = open(os.path.join(config.main.input_dir, bn))
except IOError:
- log.err("No Input Descriptor found for id %s" % inputID)
- self.set_status(404)
- self.write({'error': 'missing-input'})
- return
+ log.err("No Input Descriptor found for id %s" % inputID)
+ raise e.InputDescriptorNotFound
with f:
inputDesc = yaml.safe_load(f)
-
+
response = {'id': inputID}
for k in ['name', 'description', 'version', 'author', 'date']:
try:
response[k] = inputDesc[k]
- except Exception, e: # XXX this should probably be KeyError
- log.exception(e)
- log.err("Invalid Input Descriptor found for id %s" % inputID)
- self.set_status(500)
- self.write({'error': 'invalid-input-descriptor'})
- return
+ except KeyError:
+ log.err("Invalid Input Descriptor found for id %s" % inputID)
+ raise e.InputDescriptorNotFound
self.write(response)
+
class InputListHandler(OONIBHandler):
def get(self):
path = os.path.abspath(config.main.input_dir) + "/*.desc"
diff --git a/oonib/log.py b/oonib/log.py
index 7a6e4cb..36fe414 100644
--- a/oonib/log.py
+++ b/oonib/log.py
@@ -87,8 +87,8 @@ def msg(msg, *arg, **kw):
def debug(msg, *arg, **kw):
- if config.main.debug:
- print "[D] %s" % log_encode(msg)
+ if config.main.get('debug'):
+ print "[D] %s" % msg
def warn(msg, *arg, **kw):
diff --git a/oonib/options.py b/oonib/options.py
index d5c87b3..9abb355 100644
--- a/oonib/options.py
+++ b/oonib/options.py
@@ -1,14 +1,12 @@
from twisted.python import usage
-from twisted.python.runtime import platformType
-if platformType == "win32":
- from twisted.scripts._twistw import ServerOptions
-else:
- from twisted.scripts._twistd_unix import ServerOptions
class OONIBOptions(usage.Options):
- synopsis = """%s [options] [path to test].py """
+ synopsis = """%s [options]"""
+
+ longdesc = ("oonib provides the backend component of ooni-probe."
+ "oonib provides test helper services and a reporting "
+ "backend service. oonib is intended to be run as a "
+ "daemon and most options are set in its configuration file")
- longdesc = ("oonib provides the backend component of ooni-probe. oonib provides test helper services and a reporting backend service. oonib is intended to be run as a daemon and most options are set in its configuration file")
-
optParameters = [["config", "c", "oonib.conf", "Path to config file"]]
diff --git a/oonib/otime.py b/oonib/otime.py
index e38089b..67a6bc6 100644
--- a/oonib/otime.py
+++ b/oonib/otime.py
@@ -1,41 +1,46 @@
import time
+from oonib import errors as e
from datetime import datetime
+
def utcDateNow():
"""
Returns the datetime object of the current UTC time.
"""
return datetime.utcnow()
+
def utcTimeNow():
"""
Returns seconds since epoch in UTC time, it's of type float.
"""
return time.mktime(time.gmtime())
+
def dateToTime(date):
"""
Takes as input a datetime object and outputs the seconds since epoch.
"""
return time.mktime(date.timetuple())
+
def prettyDateNow():
"""
Returns a good looking string for the local time.
"""
return datetime.now().ctime()
+
def utcPrettyDateNow():
"""
Returns a good looking string for utc time.
"""
return datetime.utcnow().ctime()
+
def timeToPrettyDate(time_val):
return time.ctime(time_val)
-class InvalidTimestampFormat(Exception):
- pass
def fromTimestamp(s):
"""
@@ -47,17 +52,23 @@ def fromTimestamp(s):
ex. 1912-06-23T101234Z"
Note: we currently only support parsing strings that are generated from the
- timestamp function and have no intention in supporting the full standard.
+ timestamp function and have no intention in supporting the full
+ standard.
"""
try:
date_part, time_part = s.split('T')
hours, minutes, seconds = time_part[:2], time_part[2:4], time_part[4:6]
year, month, day = date_part.split('-')
except:
- raise InvalidTimestampFormat(s)
+ raise e.InvalidTimestampFormat(s)
+
+ return datetime(int(year),
+ int(month),
+ int(day),
+ int(hours),
+ int(minutes),
+ int(seconds))
- return datetime(int(year), int(month), int(day), int(hours), int(minutes),
- int(seconds))
def timestamp(t=None):
"""
@@ -86,4 +97,3 @@ def timestamp(t=None):
t = datetime.utcnow()
ISO8601 = "%Y-%m-%dT%H%M%SZ"
return t.strftime(ISO8601)
-
diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py
index cbf1c08..e81dd21 100644
--- a/oonib/policy/handlers.py
+++ b/oonib/policy/handlers.py
@@ -1,5 +1,3 @@
-import json
-import os
import yaml
from oonib import errors as e
@@ -7,6 +5,7 @@ from oonib.handlers import OONIBHandler
from oonib.config import config
+
class Policy(object):
nettest = None
input = None
@@ -38,10 +37,12 @@ class Policy(object):
if not any(nt['name'] == nettest_name for nt in self.nettest):
raise e.InvalidNettestName
+
class PolicyHandler(OONIBHandler):
def initialize(self):
self.policy = Policy()
+
class NetTestPolicyHandler(PolicyHandler):
def get(self):
"""
@@ -49,10 +50,10 @@ class NetTestPolicyHandler(PolicyHandler):
"""
self.write(self.policy.nettest)
+
class InputPolicyHandler(PolicyHandler):
def get(self):
"""
return list of input ids
"""
self.write(self.policy.input)
-
diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py
index 34aa8d9..01a665a 100644
--- a/oonib/report/handlers.py
+++ b/oonib/report/handlers.py
@@ -1,5 +1,3 @@
-import random
-import string
import time
import yaml
import json
@@ -16,35 +14,48 @@ from datetime import datetime
from oonib import randomStr, otime, log
from oonib.config import config
-class MissingField(Exception):
- pass
-class InvalidRequestField(Exception):
- pass
+def report_file_name(report_details):
+ timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time']))
+ dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format(
+ timestamp=timestamp,
+ **report_details)
+ return dst_filename
class Report(object):
- def __init__(self, report_id):
+ delayed_call = None
+
+ def __init__(self, report_id,
+ stale_time,
+ report_dir,
+ archive_dir,
+ reports):
self.report_id = report_id
- self.delayed_call = None
+
+ self.stale_time = stale_time
+ self.report_dir = report_dir
+ self.archive_dir = archive_dir
+ self.reports = reports
self.refresh()
-
+
def refresh(self):
self.last_updated = time.time()
if self.delayed_call:
self.delayed_call.cancel()
- self.delayed_call = reactor.callLater(config.main.stale_time, self.stale_check)
+ self.delayed_call = reactor.callLater(self.stale_time,
+ self.stale_check)
def stale_check(self):
- if (time.time() - self.last_updated) > config.main.stale_time:
+ if (time.time() - self.last_updated) > self.stale_time:
try:
self.close()
- except ReportNotFound:
+ except e.ReportNotFound:
pass
def close(self):
def get_report_path(report_id):
- return os.path.join(config.main.report_dir, report_id)
+ return os.path.join(self.report_dir, report_id)
report_filename = get_report_path(self.report_id)
try:
@@ -52,14 +63,10 @@ class Report(object):
g = yaml.safe_load_all(fd)
report_details = g.next()
except IOError:
- raise ReportNotFound
-
- timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time']))
- dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format(
- timestamp=timestamp,
- **report_details)
+ raise e.ReportNotFound
- dst_path = os.path.join(config.main.archive_dir,
+ dst_filename = report_file_name(report_details)
+ dst_path = os.path.join(self.archive_dir,
report_details['probe_cc'])
if not os.path.isdir(dst_path):
@@ -68,7 +75,8 @@ class Report(object):
dst_path = os.path.join(dst_path, dst_filename)
os.rename(report_filename, dst_path)
- del config.reports[self.report_id]
+ self.delayed_call.cancel()
+ del self.reports[self.report_id]
def parseUpdateReportRequest(request):
#db_report_id_regexp = re.compile("[a-zA-Z0-9]+$")
@@ -85,10 +93,10 @@ def parseUpdateReportRequest(request):
try:
report_id = parsed_request['report_id']
except KeyError:
- raise MissingField('report_id')
+ raise e.MissingField('report_id')
if not re.match(report_id_regexp, report_id):
- raise InvalidRequestField('report_id')
+ raise e.InvalidRequestField('report_id')
return parsed_request
@@ -117,79 +125,82 @@ def parseNewReportRequest(request):
try:
value_to_check = parsed_request[k]
except KeyError:
- raise MissingField(k)
+ raise e.MissingField(k)
print "Matching %s with %s | %s" % (regexp, value_to_check, k)
if re.match(regexp, str(value_to_check)):
continue
else:
- raise InvalidRequestField(k)
-
+ raise e.InvalidRequestField(k)
+
try:
requested_test_helper = parsed_request['test_helper']
if not re.match(test_helper, str(requested_test_helper)):
- raise InvalidRequestField('test_helper')
+ raise e.InvalidRequestField('test_helper')
except KeyError:
pass
return parsed_request
-class InvalidReportHeader(Exception):
- pass
-
-class MissingReportHeaderKey(InvalidReportHeader):
- pass
-
def validate_report_header(report_header):
required_keys = ['probe_asn', 'probe_cc', 'probe_ip', 'software_name',
'software_version', 'test_name', 'test_version']
for key in required_keys:
if key not in report_header:
- raise MissingReportHeaderKey(key)
+ raise e.MissingReportHeaderKey(key)
if report_header['probe_asn'] is None:
report_header['probe_asn'] = 'AS0'
if not re.match('AS[0-9]+$', report_header['probe_asn']):
- raise InvalidReportHeader('probe_asn')
+ raise e.InvalidReportHeader('probe_asn')
# If no country is known, set it to be ZZ (user assigned value in ISO 3166)
if report_header['probe_cc'] is None:
report_header['probe_cc'] = 'ZZ'
if not re.match('[a-zA-Z]{2}$', report_header['probe_cc']):
- raise InvalidReportHeader('probe_cc')
+ raise e.InvalidReportHeader('probe_cc')
if not re.match('[a-z_\-]+$', report_header['test_name']):
- raise InvalidReportHeader('test_name')
+ raise e.InvalidReportHeader('test_name')
if not re.match('([0-9]+\.)+[0-9]+$', report_header['test_version']):
- raise InvalidReportHeader('test_version')
+ raise e.InvalidReportHeader('test_version')
return report_header
+class ReportHandler(OONIBHandler):
+ def initialize(self):
+ self.archive_dir = config.main.archive_dir
+ self.report_dir = config.main.report_dir
+ self.reports = config.reports
+ self.policy_file = config.main.policy_file
+ self.helpers = config.helpers
+ self.stale_time = config.main.stale_time
+
class UpdateReportMixin(object):
def updateReport(self, report_id, parsed_request):
log.debug("Got this request %s" % parsed_request)
- report_filename = os.path.join(config.main.report_dir,
+ report_filename = os.path.join(self.report_dir,
report_id)
-
- config.reports[report_id].refresh()
+
+ self.reports[report_id].refresh()
try:
with open(report_filename, 'a+') as fd:
- fdesc.setNonBlocking(fd.fileno())
- fdesc.writeToFD(fd.fileno(), parsed_request['content'])
- except IOError as exc:
+ fd.write(parsed_request['content'])
+ except IOError:
e.OONIBError(404, "Report not found")
- self.write({})
+ self.write({'status': 'success'})
-class NewReportHandlerFile(OONIBHandler, UpdateReportMixin):
+class NewReportHandlerFile(ReportHandler, UpdateReportMixin):
"""
Responsible for creating and updating reports by writing to flat file.
"""
+ inputHashes = None
def checkPolicy(self):
policy = Policy()
@@ -236,42 +247,31 @@ class NewReportHandlerFile(OONIBHandler, UpdateReportMixin):
{'backend_version': 'XXX', 'report_id': 'XXX'}
"""
- # XXX here we should validate and sanitize the request
- try:
- report_data = parseNewReportRequest(self.request.body)
- except InvalidRequestField as exc:
- raise e.InvalidRequestField(exc)
- except MissingField as exc:
- raise e.MissingRequestField(exc)
+ # Note: the request is being validated inside of parseNewReportRequest.
+ report_data = parseNewReportRequest(self.request.body)
log.debug("Parsed this data %s" % report_data)
software_name = str(report_data['software_name'])
software_version = str(report_data['software_version'])
-
+
probe_asn = str(report_data['probe_asn'])
probe_cc = str(report_data.get('probe_cc', 'ZZ'))
self.testName = str(report_data['test_name'])
self.testVersion = str(report_data['test_version'])
-
- if config.main.policy_file:
+
+ if self.policy_file:
try:
self.inputHashes = report_data['input_hashes']
except KeyError:
raise e.InputHashNotProvided
self.checkPolicy()
-
+
if 'content' in report_data:
content = yaml.safe_load(report_data['content'])
- try:
- report_header = validate_report_header(content)
-
- except MissingReportHeaderKey, key:
- raise e.MissingReportHeaderKey(key)
+ report_header = validate_report_header(content)
- except InvalidReportHeader, key:
- raise e.InvalidReportHeader(key)
else:
content = {
'software_name': software_name,
@@ -298,22 +298,26 @@ class NewReportHandlerFile(OONIBHandler, UpdateReportMixin):
# The report filename contains the timestamp of the report plus a
# random nonce
- report_filename = os.path.join(config.main.report_dir, report_id)
+ report_filename = os.path.join(self.report_dir, report_id)
response = {
'backend_version': config.backend_version,
'report_id': report_id
}
-
+
requested_helper = report_data.get('test_helper')
if requested_helper:
try:
- response['test_helper_address'] = config.helpers[requested_helper].address
+ response['test_helper_address'] = self.helpers[requested_helper].address
except KeyError:
raise e.TestHelperNotFound
-
- config.reports[report_id] = Report(report_id)
+
+ self.reports[report_id] = Report(report_id,
+ self.stale_time,
+ self.report_dir,
+ self.archive_dir,
+ self.reports)
self.writeToReport(report_filename, content)
@@ -338,7 +342,7 @@ class NewReportHandlerFile(OONIBHandler, UpdateReportMixin):
self.updateReport(report_id, parsed_request)
-class UpdateReportHandlerFile(OONIBHandler, UpdateReportMixin):
+class UpdateReportHandlerFile(ReportHandler, UpdateReportMixin):
def post(self, report_id):
try:
parsed_request = json.loads(self.request.body)
@@ -346,20 +350,17 @@ class UpdateReportHandlerFile(OONIBHandler, UpdateReportMixin):
raise e.InvalidRequest
self.updateReport(report_id, parsed_request)
-class ReportNotFound(Exception):
- pass
-
-class CloseReportHandlerFile(OONIBHandler):
+class CloseReportHandlerFile(ReportHandler):
def get(self):
pass
def post(self, report_id):
- if report_id in config.reports:
- config.reports[report_id].close()
+ if report_id in self.reports:
+ self.reports[report_id].close()
else:
raise e.ReportNotFound
-class PCAPReportHandler(OONIBHandler):
+class PCAPReportHandler(ReportHandler):
def get(self):
pass
diff --git a/oonib/runner.py b/oonib/runner.py
index c8fafa1..d413221 100644
--- a/oonib/runner.py
+++ b/oonib/runner.py
@@ -1,7 +1,3 @@
-# -*- encoding: utf-8 -*-
-#
-# :authors: Arturo Filastò, Isis Lovecruft
-# :licence: see LICENSE for details
"""
In here we define a runner for the oonib backend system.
"""
@@ -13,14 +9,12 @@ import os
from shutil import rmtree
-from twisted.internet import reactor
-from twisted.application import service, internet, app
+from twisted.internet import reactor, endpoints
from twisted.python.runtime import platformType
from txtorcon import TCPHiddenServiceEndpoint, TorConfig
from txtorcon import launch_tor
-from oonib.report.api import reportAPI
from oonib.api import ooniBackend, ooniBouncer
from oonib.config import config
@@ -28,15 +22,16 @@ from oonib import oonibackend
from oonib import log
from txtorcon import __version__ as txtorcon_version
-if tuple(map(int, txtorcon_version.split('.'))) < (0,9,0):
+if tuple(map(int, txtorcon_version.split('.'))) < (0, 9, 0):
"""
Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint
listens on all interfaces by default.
"""
def create_listener(self, proto):
self._update_onion(self.hiddenservice.dir)
- self.tcp_endpoint = TCP4ServerEndpoint(self.reactor, self.listen_port, # XXX missing import -- has this code ever been tested?
- interface='127.0.0.1')
+ self.tcp_endpoint = endpoints.TCP4ServerEndpoint(self.reactor,
+ self.listen_port,
+ interface='127.0.0.1')
d = self.tcp_endpoint.listen(self.protocolfactory)
d.addCallback(self._add_attributes).addErrback(self._retry_local_port)
return d
@@ -64,13 +59,17 @@ else:
def setupHSEndpoint(self, tor_process_protocol, torconfig, endpoint):
endpointName = endpoint.settings['name']
+
def setup_complete(port):
- print("Exposed %s Tor hidden service on httpo://%s" % (endpointName,
- port.onion_uri))
+ print("Exposed %s Tor hidden service "
+ "on httpo://%s" % (endpointName, port.onion_uri))
public_port = 80
- hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port,
- data_dir=os.path.join(torconfig.DataDirectory, endpointName))
+ data_dir = os.path.join(torconfig.DataDirectory, endpointName)
+ hs_endpoint = TCPHiddenServiceEndpoint(reactor,
+ torconfig,
+ public_port,
+ data_dir=data_dir)
d = hs_endpoint.listen(endpoint)
d.addCallback(setup_complete)
d.addErrback(self.txSetupFailed)
diff --git a/oonib/testhelpers/dns_helpers.py b/oonib/testhelpers/dns_helpers.py
index 9b2e5e3..fbf932b 100644
--- a/oonib/testhelpers/dns_helpers.py
+++ b/oonib/testhelpers/dns_helpers.py
@@ -1,14 +1,12 @@
-from twisted.internet.protocol import Factory, Protocol
-from twisted.internet import reactor
-from twisted.names import dns
from twisted.names import client, server
from oonib.config import config
+
class DNSTestHelper(server.DNSServerFactory):
- def __init__(self, authorities = None,
- caches = None, clients = None,
- verbose = 0):
+ def __init__(self, authorities=None,
+ caches=None, clients=None,
+ verbose=0):
try:
host, port = config.helpers.dns.split(':')
port = int(port)
@@ -17,8 +15,9 @@ class DNSTestHelper(server.DNSServerFactory):
except:
host, port = '8.8.8.8', 53
resolver = client.Resolver(servers=[(host, port)])
- server.DNSServerFactory.__init__(self, authorities = authorities,
- caches = caches, clients = [resolver],
- verbose = verbose)
+ server.DNSServerFactory.__init__(self, authorities=authorities,
+ caches=caches, clients=[resolver],
+ verbose=verbose)
+
def handleQuery(self, message, protocol, address):
server.DNSServerFactory.handleQuery(self, message, protocol, address)
diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py
index 245e08a..b2afc59 100644
--- a/oonib/testhelpers/http_helpers.py
+++ b/oonib/testhelpers/http_helpers.py
@@ -93,10 +93,11 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin):
headers_dict[k] = []
headers_dict[k].append(v)
- response = {'request_headers': self.headers,
- 'request_line': self.requestLine,
- 'headers_dict': headers_dict
- }
+ response = {
+ 'request_headers': self.headers,
+ 'request_line': self.requestLine,
+ 'headers_dict': headers_dict
+ }
json_response = json.dumps(response)
self.transport.write('HTTP/1.1 200 OK\r\n\r\n')
self.transport.write('%s' % json_response)
diff --git a/oonib/testhelpers/ssl_helpers.py b/oonib/testhelpers/ssl_helpers.py
index d511ddd..4b84a8c 100644
--- a/oonib/testhelpers/ssl_helpers.py
+++ b/oonib/testhelpers/ssl_helpers.py
@@ -3,7 +3,7 @@ from oonib.config import config
class SSLContext(ssl.DefaultOpenSSLContextFactory):
def __init__(self, *args, **kw):
- ssl.DefaultOpenSSLContextFactory.__init__(self,
+ ssl.DefaultOpenSSLContextFactory.__init__(self,
config.helpers.ssl.private_key,
config.helpers.ssl.certificate)
diff --git a/oonib/testhelpers/tcp_helpers.py b/oonib/testhelpers/tcp_helpers.py
index 91c334b..614cf87 100644
--- a/oonib/testhelpers/tcp_helpers.py
+++ b/oonib/testhelpers/tcp_helpers.py
@@ -1,16 +1,16 @@
-
from twisted.internet.protocol import Protocol, Factory, ServerFactory
-from twisted.internet.error import ConnectionDone
from oonib.config import config
from oonib import log
from oonib.daphn3 import Daphn3Protocol
from oonib.daphn3 import read_pcap, read_yaml
+
class TCPEchoProtocol(Protocol):
def dataReceived(self, data):
self.transport.write(data)
+
class TCPEchoHelper(Factory):
"""
A very simple echo protocol implementation
@@ -24,8 +24,11 @@ elif config.helpers.daphn3.pcap_file:
daphn3Steps = read_yaml(config.helpers.daphn3.pcap_file)
else:
- daphn3Steps = [{'client': 'client_packet'},
- {'server': 'server_packet'}]
+ daphn3Steps = [
+ {'client': 'client_packet'},
+ {'server': 'server_packet'}
+ ]
+
class Daphn3ServerProtocol(Daphn3Protocol):
def nextStep(self):
More information about the tor-commits
mailing list