April 2014 - tor-commits - lists.torproject.org

[oonib/master] Merge pull request #44 from TheTorProject/fix/issue_d
by art＠torproject.org 30 Apr '14

30 Apr '14

commit bc337c58d3beb87c3bc31694e530af9cf5e987d9 Merge: 72a86de d60d0cc Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Wed Apr 30 19:03:08 2014 +0200 Merge pull request #44 from TheTorProject/fix/issue_d Verify the sha 256 sum of the zlib package. scripts/build_tor2web_tor.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

1 0

[oonib/master] Merge pull request #45 from TheTorProject/fix/issue_c
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 42b4985d44a3d714540f7a2455df106f2d154e6f Merge: bc337c5 f40f3c5 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Wed Apr 30 19:03:31 2014 +0200 Merge pull request #45 from TheTorProject/fix/issue_c Escape all strings before writing them to the log file. oonib/log.py | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-)

1 0

[oonib/master] Merge pull request #46 from TheTorProject/fix/issue_e
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 0ee9875aefe47e963bf2939221464bb2d8029e1b Merge: 42b4985 1452e77 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Wed Apr 30 19:04:58 2014 +0200 Merge pull request #46 from TheTorProject/fix/issue_e Set limits on the number of headers that can be sent and their length. oonib/testhelpers/http_helpers.py | 39 +++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-)

1 0

[oonib/master] Set limits on the number of headers that can be sent and their length.
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 1452e777377d2ecccc4b7bf0dff3d815fbb018b0 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Apr 23 16:41:50 2014 +0200 Set limits on the number of headers that can be sent and their length. --- oonib/testhelpers/http_helpers.py | 39 +++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py index 6c3e123..245e08a 100644 --- a/oonib/testhelpers/http_helpers.py +++ b/oonib/testhelpers/http_helpers.py @@ -2,18 +2,15 @@ import json import random import string -from twisted.application import internet, service -from twisted.internet import protocol, reactor, defer -from twisted.protocols import basic -from twisted.web import resource, server, static, http -from twisted.web.microdom import escape +from twisted.internet import protocol, defer from cyclone.web import RequestHandler, Application from twisted.protocols import policies, basic from twisted.web.http import Request -from oonib import randomStr +from oonib import log + class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): """ @@ -41,6 +38,7 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): length = 0 maxHeaders = 500 + maxHeaderLineLength = 16384 requestLine = '' timeOut = 60 * 60 * 12 @@ -53,6 +51,10 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): self.setTimeout(self.timeOut) def lineReceived(self, line): + if len(self.__header) >= self.maxHeaderLineLength: + log.err("Maximum header length reached.") + return self.transport.loseConnection() + if self.__first_line: self.requestLine = line self.__first_line = 0 @@ -66,7 +68,7 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): elif line[0] in ' \t': # This is to support header field value folding over multiple lines # as specified by rfc2616. - self.__header = self.__header+'\n'+line + self.__header += '\n'+line else: if self.__header: self.headerReceived(self.__header) @@ -80,6 +82,10 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): log.err("Got malformed HTTP Header request field") log.err("%s" % line) + if len(self.headers) >= self.maxHeaders: + log.err("Maximum number of headers received.") + return self.transport.loseConnection() + def allHeadersReceived(self): headers_dict = {} for k, v in self.headers: @@ -88,9 +94,9 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): headers_dict[k].append(v) response = {'request_headers': self.headers, - 'request_line': self.requestLine, - 'headers_dict': headers_dict - } + 'request_line': self.requestLine, + 'headers_dict': headers_dict + } json_response = json.dumps(response) self.transport.write('HTTP/1.1 200 OK\r\n\r\n') self.transport.write('%s' % json_response) @@ -99,22 +105,25 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): class HTTPReturnJSONHeadersHelper(protocol.ServerFactory): protocol = SimpleHTTPChannel + def buildProtocol(self, addr): return self.protocol() + class HTTPTrapAll(RequestHandler): def _execute(self, transforms, *args, **kwargs): self._transforms = transforms defer.maybeDeferred(self.prepare).addCallbacks( - self._execute_handler, - lambda f: self._handle_request_exception(f.value), - callbackArgs=(args, kwargs)) + self._execute_handler, + lambda f: self._handle_request_exception(f.value), + callbackArgs=(args, kwargs)) def _execute_handler(self, r, args, kwargs): if not self._finished: args = [self.decode_argument(arg) for arg in args] kwargs = dict((k, self.decode_argument(v, name=k)) - for (k, v) in kwargs.iteritems()) + for (k, v) in kwargs.iteritems()) + # This is where we do the patching # XXX this is somewhat hackish d = defer.maybeDeferred(self.all, *args, **kwargs) @@ -130,6 +139,7 @@ class HTTPRandomPage(HTTPTrapAll): XXX this is currently disabled as it is not of use to any test. """ isLeaf = True + def _gen_random_string(self, length): return ''.join(random.choice(string.letters) for x in range(length)) @@ -152,4 +162,3 @@ HTTPRandomPageHelper = Application([ # XXX add regexps here (r"/(.*)/(.*)", HTTPRandomPage) ]) -

1 0

[oonib/master] Add unittests for the report handlers.
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 699c26c018d0b19ee61c57d7c27207faee1e556c Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Apr 30 14:57:57 2014 +0200 Add unittests for the report handlers. Add a useful base class for implementing any unittest. --- oonib/test/handler_helpers.py | 62 +++++++++++++++ oonib/test/test_report.py | 173 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+) diff --git a/oonib/test/__init__.py b/oonib/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/test/handler_helpers.py b/oonib/test/handler_helpers.py new file mode 100644 index 0000000..2d704a2 --- /dev/null +++ b/oonib/test/handler_helpers.py @@ -0,0 +1,62 @@ +import socket +import json + +from twisted.internet import reactor, defer +from twisted.trial import unittest + +from cyclone import httpclient + + +def random_unused_port(bind_address='127.0.0.1'): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(('127.0.0.1', 0)) + port = s.getsockname()[1] + s.close() + return port + +reports = {} + + +def mock_initialize(self): + self.report_dir = '.' + self.archive_dir = '.' + self.reports = reports + self.policy_file = None + self.helpers = {} + self.stale_time = 10 + + +class HandlerTestCase(unittest.TestCase): + app = None + _port = None + _listener = None + + @property + def port(self): + if not self._port: + self._port = random_unused_port() + return self._port + + def setUp(self, *args, **kw): + if self.app: + self._listener = reactor.listenTCP(self.port, self.app) + return unittest.TestCase.setUp(self, *args, **kw) + + def tearDown(self): + if self._listener: + for report in reports.values(): + try: + report.delayed_call.cancel() + except: + pass + self._listener.stopListening() + + @defer.inlineCallbacks + def request(self, path, method="GET", postdata=None): + url = "http://localhost:%s%s" % (self.port, path) + if isinstance(postdata, dict): + postdata = json.dumps(postdata) + + response = yield httpclient.fetch(url, method=method, + postdata=postdata) + defer.returnValue(response) diff --git a/oonib/test/test_report.py b/oonib/test/test_report.py new file mode 100644 index 0000000..9ed02de --- /dev/null +++ b/oonib/test/test_report.py @@ -0,0 +1,173 @@ +import os +import json +import yaml + +from twisted.internet import defer + +from cyclone import web + +from oonib.report.handlers import report_file_name +from oonib.report.api import reportAPI +from oonib.test.handler_helpers import HandlerTestCase, mock_initialize + +sample_report_entry = """--- +agent: agent +input: null +requests: +- request: + body: null + headers: + - - ACCePT-LAnGuagE + - ['en-US,en;q=0.8'] + - - aCCEPT-ENcODInG + - ['gzip,deflate,sdch'] + - - aCcEPT + - ['text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'] + - - User-AGeNt + - ['Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.7) Gecko/20091221 + Firefox/3.5.7'] + - - aCCEpt-cHArSEt + - ['ISO-8859-1,utf-8;q=0.7,*;q=0.3'] + - - HOsT + - [KIXnnZDJfGKRNab.com] + method: GET + url: http://12.34.56.78 +response: + body: '{"headers_dict": {"ACCePT-LAnGuagE": ["en-US,en;q=0.8"], "aCCEPT-ENcODInG": + ["gzip,deflate,sdch"], "HOsT": ["KIXnnZDJfGKRNab.com"], "aCcEPT": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], + "User-AGeNt": ["Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.7) + Gecko/20091221 Firefox/3.5.7"], "aCCEpt-cHArSEt": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"], + "Connection": ["close"]}, "request_line": "GET / HTTP/1.1", "request_headers": + [["Connection", "close"], ["ACCePT-LAnGuagE", "en-US,en;q=0.8"], ["aCCEPT-ENcODInG", + "gzip,deflate,sdch"], ["aCcEPT", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], + ["User-AGeNt", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.7) + Gecko/20091221 Firefox/3.5.7"], ["aCCEpt-cHArSEt", "ISO-8859-1,utf-8;q=0.7,*;q=0.3"], + ["HOsT", "KIXnnZDJfGKRNab.com"]]}' + code: 200 + headers: [] +socksproxy: null +tampering: +header_field_name: false +header_field_number: false +header_field_value: false +header_name_capitalization: false +header_name_diff: [] +request_line_capitalization: false +total: false +... +""" + +for _, handler in reportAPI: + handler.initialize = mock_initialize + + +class TestReport(HandlerTestCase): + app = web.Application(reportAPI, name='reportAPI') + + @defer.inlineCallbacks + def update_report(self, report_id, content=sample_report_entry): + data = { + 'content': content + } + response = yield self.request( + '/report/%s' % report_id, + "POST", data) + defer.returnValue(response) + + @defer.inlineCallbacks + def test_create_valid_report(self): + data = { + 'software_name': 'ooni-test', + 'software_version': '0.1', + 'test_name': 'some-test', + 'test_version': '0.1', + 'probe_asn': 'AS0' + } + response = yield self.request('/report', "POST", data) + response_body = json.loads(response.body) + self.assertIn('backend_version', response_body) + self.assertIn('report_id', response_body) + + @defer.inlineCallbacks + def test_create_invalid_report(self): + data = { + 'software_name': 'ooni-test', + 'software_version': '0.1', + 'test_name': 'some-test', + 'test_version': '0.1', + 'probe_asn': 'XXX' + } + response = yield self.request('/report', "POST", data) + response_body = json.loads(response.body) + self.assertIn('error', response_body) + self.assertEqual(response_body['error'], 'invalid-request-field probe_asn') + + @defer.inlineCallbacks + def test_create_and_update_report(self): + report_header = { + 'software_name': 'ooni-test', + 'software_version': '0.1', + 'test_name': 'some-test', + 'test_version': '0.1', + 'probe_asn': 'AS0' + } + response = yield self.request('/report', "POST", report_header) + response_body = json.loads(response.body) + self.assertIn('backend_version', response_body) + self.assertIn('report_id', response_body) + + report_id = response_body['report_id'] + response = yield self.update_report(report_id) + response_body = json.loads(response.body) + + with open(report_id) as f: + written_report = yaml.safe_load_all(f) + + written_report_header = written_report.next() + for key in report_header.keys(): + self.assertEqual(written_report_header[key], report_header[key]) + self.assertEqual(yaml.safe_load(sample_report_entry), + written_report.next()) + + @defer.inlineCallbacks + def test_create_update_and_close_report(self): + report_header = { + 'software_name': 'ooni-test', + 'software_version': '0.1', + 'test_name': 'some-test', + 'test_version': '0.1', + 'probe_asn': 'AS0' + } + response = yield self.request('/report', "POST", report_header) + response_body = json.loads(response.body) + self.assertIn('backend_version', response_body) + self.assertIn('report_id', response_body) + + report_entry_count = 100 + + report_id = response_body['report_id'] + for i in range(report_entry_count): + yield self.update_report(report_id) + + with open(report_id) as f: + written_report = yaml.safe_load_all(f) + + written_report_header = written_report.next() + for key in report_header.keys(): + self.assertEqual(written_report_header[key], + report_header[key]) + + self.assertEqual(yaml.safe_load(sample_report_entry), + written_report.next()) + + response = yield self.request('/report/%s/close' % report_id, "POST") + + written_report_path = os.path.join(written_report_header['probe_cc'], + report_file_name(written_report_header)) + with open(written_report_path) as f: + written_report = yaml.safe_load_all(f) + written_report.next() + + for i in range(report_entry_count): + self.assertEqual(yaml.safe_load(sample_report_entry), + written_report.next())

1 0

[oonib/master] Refactoring and tidying up of various different parts of the codebase.
by art＠torproject.org 30 Apr '14

30 Apr '14

commit c0cad1259227019150258b5dfc2bc8d4844590a1 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Apr 30 14:57:15 2014 +0200 Refactoring and tidying up of various different parts of the codebase. Let pylint guide the way! --- oonib/bouncer/handlers.py | 17 ++-- oonib/config.py | 11 ++- oonib/daphn3.py | 4 +- oonib/deck/api.py | 4 +- oonib/deck/handlers.py | 10 ++- oonib/errors.py | 50 ++++++++++-- oonib/handlers.py | 4 + oonib/input/api.py | 4 +- oonib/input/handlers.py | 21 +++-- oonib/log.py | 4 +- oonib/options.py | 14 ++-- oonib/otime.py | 24 ++++-- oonib/policy/handlers.py | 7 +- oonib/report/handlers.py | 155 +++++++++++++++++++------------------ oonib/runner.py | 27 ++++--- oonib/testhelpers/dns_helpers.py | 17 ++-- oonib/testhelpers/http_helpers.py | 9 ++- oonib/testhelpers/ssl_helpers.py | 2 +- oonib/testhelpers/tcp_helpers.py | 11 ++- 19 files changed, 225 insertions(+), 170 deletions(-) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 0a5b51f..81d13f3 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -1,10 +1,11 @@ -import json +import json import random import yaml from oonib import errors as e from oonib.handlers import OONIBHandler from oonib.config import config + class Bouncer(object): def __init__(self): with open(config.main.bouncer_file) as f: @@ -20,7 +21,7 @@ class Bouncer(object): for collectorName, helpers in bouncerFile['collector'].items(): if collectorName not in self.knownCollectors: self.knownCollectors.append(collectorName) - + def updateKnownHelpers(self, bouncerFile): self.knownHelpers = {} for collectorName, helpers in bouncerFile['collector'].items(): @@ -47,13 +48,13 @@ class Bouncer(object): helpers = self.knownHelpers[helper_name] except KeyError: raise e.TestHelperNotFound - + helpers_dict = {} for helper in helpers: helpers_dict[helper['collector-name']] = helper['helper-address'] return helpers_dict - + def filterHelperAddresses(self, requested_helpers): """ Returns a dict of collectors that support all the requested_helpers. @@ -79,7 +80,7 @@ class Bouncer(object): } } - or + or {'error': 'test-helper-not-found'} @@ -105,9 +106,11 @@ class Bouncer(object): response = {'error': 'test-helper-not-found'} return response - response['default'] = {'collector': random.choice(self.knownCollectors)} + response['default'] = {'collector': + random.choice(self.knownCollectors)} return response + class BouncerQueryHandler(OONIBHandler): def initialize(self): self.bouncer = Bouncer() @@ -122,7 +125,7 @@ class BouncerQueryHandler(OONIBHandler): requested_helpers = query['test-helpers'] except KeyError: raise e.TestHelpersKeyMissing - + if not isinstance(requested_helpers, list): raise e.InvalidRequest diff --git a/oonib/config.py b/oonib/config.py index 1388e6b..31128a9 100644 --- a/oonib/config.py +++ b/oonib/config.py @@ -6,17 +6,16 @@ from oonib import __version__ from oonib.options import OONIBOptions import os + class Config(object): - main = None - helpers = None + main = {} + helpers = {} reports = {} backend_version = __version__ opts = OONIBOptions() - def __init__(self): - self.opts.parseOptions() - def load(self): + self.opts.parseOptions() try: config_file = self.opts['config'] except KeyError: @@ -35,7 +34,7 @@ class Config(object): self.helpers[name] = Storage(helper.items()) self.check_paths() - + def check_paths(self): def check_path(directory, complaint): if not (directory and os.path.isdir(directory)): diff --git a/oonib/daphn3.py b/oonib/daphn3.py index 9c5f290..4b4eac4 100644 --- a/oonib/daphn3.py +++ b/oonib/daphn3.py @@ -96,7 +96,7 @@ def daphn3Mutate(steps, step_idx, mutation_idx): if idx == step_idx: step_string = step.values()[0] step_key = step.keys()[0] - mutated_string = daphn3MutateString(step_string, + mutated_string = daphn3MutateString(step_string, mutation_idx) mutated_steps.append({step_key: mutated_string}) else: @@ -121,7 +121,7 @@ class Daphn3Protocol(protocol.Protocol): def _current_step_data(self): step_idx, mutation_idx = self.factory.mutation log.debug("Mutating %s %s" % (step_idx, mutation_idx)) - mutated_step = daphn3Mutate(self.steps, + mutated_step = daphn3Mutate(self.steps, step_idx, mutation_idx) log.debug("Mutated packet into %s" % mutated_step) return mutated_step[self.current_step].values()[0] diff --git a/oonib/deck/api.py b/oonib/deck/api.py index a2f1164..1c19d32 100644 --- a/oonib/deck/api.py +++ b/oonib/deck/api.py @@ -5,6 +5,6 @@ from oonib.config import config deckAPI = [ (r"/deck", handlers.DeckListHandler), (r"/deck/([a-z0-9]{64})$", handlers.DeckDescHandler), - (r"/deck/([a-z0-9]{64})/file$", web.StaticFileHandler, {"path": - config.main.deck_dir}), + (r"/deck/([a-z0-9]{64})/file$", web.StaticFileHandler, + {"path": config.main.deck_dir}), ] diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index b9589b0..ca17e95 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -1,5 +1,4 @@ import glob -import json import os import re import yaml @@ -9,6 +8,7 @@ from oonib.handlers import OONIBHandler from oonib import log from oonib.config import config + class DeckDescHandler(OONIBHandler): def get(self, deckID): # note: @@ -32,18 +32,20 @@ class DeckDescHandler(OONIBHandler): raise e.MissingDeckKeys self.write(response) + class DeckListHandler(OONIBHandler): def get(self): - if not config.main.deck_dir: + if not config.main.deck_dir: self.set_status(501) raise e.NoDecksConfigured path = os.path.abspath(config.main.deck_dir) + "/*" decknames = map(os.path.basename, glob.iglob(path)) - decknames = filter(lambda y: re.match("[a-z0-9]{64}.desc", y), decknames) + decknames = filter(lambda y: re.match("[a-z0-9]{64}.desc", y), + decknames) deckList = [] for deckname in decknames: - with open(os.path.join(config.main.deck_dir, deckname)) as f: + with open(os.path.join(config.main.deck_dir, deckname)) as f: d = yaml.safe_load(f) deckList.append({ 'id': deckname, diff --git a/oonib/errors.py b/oonib/errors.py index 6f768d0..3f1167a 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -1,87 +1,125 @@ from cyclone.web import HTTPError + class OONIBError(HTTPError): status_code = 500 log_message = 'oonib-error' + def __init__(self): pass + class InvalidRequest(OONIBError): status_code = 400 log_message = 'invalid-request' + class NoHelperFound(OONIBError): status_code = 404 log_message = 'no-helper-found' + class InvalidInputHash(OONIBError): status_code = 406 log_message = 'invalid-input-hash' + class InvalidNettestName(OONIBError): status_code = 406 log_message = 'invalid-nettest-name' + class InputHashNotProvided(OONIBError): status_code = 406 log_message = 'input-hash-not-provided' + +class InputDescriptorNotFound(OONIBError): + status_code = 404 + log_message = 'input-descriptor-not-found' + + class InvalidRequestField(OONIBError): def __init__(self, field_name): self.status_code = 400 self.log_message = "invalid-request-field %s" % field_name + class MissingRequestField(OONIBError): def __init__(self, field_name): self.status_code = 400 self.log_message = "missing-request-field %s" % field_name + class MissingReportHeaderKey(OONIBError): def __init__(self, key): self.status_code = 406 self.log_message = "missing-report-header-key %s" % key + class MissingDeckKeys(OONIBError): status_code = 400 log_message = "missing-deck-keys" + class MissingDeck(OONIBError): status_code = 400 log_message = "missing-deck" + class NoDecksConfigured(OONIBError): status_code = 501 log_message = "no-decks-configured" + class InvalidReportHeader(OONIBError): def __init__(self, key): self.status_code = 406 self.log_message = "invalid-report-header %s" % key + class ReportNotFound(OONIBError): status_code = 404 log_message = "report-not-found" + class NoValidCollector(OONIBError): pass + class TestHelpersKeyMissing(OONIBError): status_code = 400 log_message = "test-helpers-key-missing" + class TestHelperNotFound(OONIBError): status_code = 404 log_message = "test-helper-not-found" -class ConfigFileNotSpecified(Exception): pass -class ConfigFileDoesNotExist(Exception): pass +class ConfigFileNotSpecified(Exception): + pass -class InvalidReportDirectory(Exception): pass -class InvalidArchiveDirectory(Exception): pass +class ConfigFileDoesNotExist(Exception): + pass -class InvalidInputDirectory(Exception): pass -class InvalidDeckDirectory(Exception): pass +class InvalidReportDirectory(Exception): + pass + + +class InvalidArchiveDirectory(Exception): + pass + +class InvalidInputDirectory(Exception): + pass + + +class InvalidDeckDirectory(Exception): + pass + + +class InvalidTimestampFormat(Exception): + pass diff --git a/oonib/handlers.py b/oonib/handlers.py index 20c831b..f92d684 100644 --- a/oonib/handlers.py +++ b/oonib/handlers.py @@ -3,12 +3,16 @@ import types from cyclone import escape from cyclone import web +from oonib import log + + class OONIBHandler(web.RequestHandler): def write_error(self, status_code, exception=None, **kw): self.set_status(status_code) if hasattr(exception, 'log_message'): self.write({'error': exception.log_message}) else: + log.error(exception) self.write({'error': 'error'}) def write(self, chunk): diff --git a/oonib/input/api.py b/oonib/input/api.py index 811da29..32ecd10 100644 --- a/oonib/input/api.py +++ b/oonib/input/api.py @@ -5,6 +5,6 @@ from oonib.config import config inputAPI = [ (r"/input", handlers.InputListHandler), (r"/input/([a-f0-9]{64})", handlers.InputDescHandler), - (r"/input/([a-f0-9]{64})/file$", web.StaticFileHandler, {"path": - config.main.input_dir}), + (r"/input/([a-f0-9]{64})/file$", web.StaticFileHandler, + {"path": config.main.input_dir}), ] diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py index 33bec55..53501fa 100644 --- a/oonib/input/handlers.py +++ b/oonib/input/handlers.py @@ -1,38 +1,35 @@ import glob -import json import os import yaml from oonib.handlers import OONIBHandler from oonib import log +from oonib import errors as e from oonib.config import config + class InputDescHandler(OONIBHandler): def get(self, inputID): bn = os.path.basename(inputID) + ".desc" try: f = open(os.path.join(config.main.input_dir, bn)) except IOError: - log.err("No Input Descriptor found for id %s" % inputID) - self.set_status(404) - self.write({'error': 'missing-input'}) - return + log.err("No Input Descriptor found for id %s" % inputID) + raise e.InputDescriptorNotFound with f: inputDesc = yaml.safe_load(f) - + response = {'id': inputID} for k in ['name', 'description', 'version', 'author', 'date']: try: response[k] = inputDesc[k] - except Exception, e: # XXX this should probably be KeyError - log.exception(e) - log.err("Invalid Input Descriptor found for id %s" % inputID) - self.set_status(500) - self.write({'error': 'invalid-input-descriptor'}) - return + except KeyError: + log.err("Invalid Input Descriptor found for id %s" % inputID) + raise e.InputDescriptorNotFound self.write(response) + class InputListHandler(OONIBHandler): def get(self): path = os.path.abspath(config.main.input_dir) + "/*.desc" diff --git a/oonib/log.py b/oonib/log.py index 7a6e4cb..36fe414 100644 --- a/oonib/log.py +++ b/oonib/log.py @@ -87,8 +87,8 @@ def msg(msg, *arg, **kw): def debug(msg, *arg, **kw): - if config.main.debug: - print "[D] %s" % log_encode(msg) + if config.main.get('debug'): + print "[D] %s" % msg def warn(msg, *arg, **kw): diff --git a/oonib/options.py b/oonib/options.py index d5c87b3..9abb355 100644 --- a/oonib/options.py +++ b/oonib/options.py @@ -1,14 +1,12 @@ from twisted.python import usage -from twisted.python.runtime import platformType -if platformType == "win32": - from twisted.scripts._twistw import ServerOptions -else: - from twisted.scripts._twistd_unix import ServerOptions class OONIBOptions(usage.Options): - synopsis = """%s [options] [path to test].py """ + synopsis = """%s [options]""" + + longdesc = ("oonib provides the backend component of ooni-probe." + "oonib provides test helper services and a reporting " + "backend service. oonib is intended to be run as a " + "daemon and most options are set in its configuration file") - longdesc = ("oonib provides the backend component of ooni-probe. oonib provides test helper services and a reporting backend service. oonib is intended to be run as a daemon and most options are set in its configuration file") - optParameters = [["config", "c", "oonib.conf", "Path to config file"]] diff --git a/oonib/otime.py b/oonib/otime.py index e38089b..67a6bc6 100644 --- a/oonib/otime.py +++ b/oonib/otime.py @@ -1,41 +1,46 @@ import time +from oonib import errors as e from datetime import datetime + def utcDateNow(): """ Returns the datetime object of the current UTC time. """ return datetime.utcnow() + def utcTimeNow(): """ Returns seconds since epoch in UTC time, it's of type float. """ return time.mktime(time.gmtime()) + def dateToTime(date): """ Takes as input a datetime object and outputs the seconds since epoch. """ return time.mktime(date.timetuple()) + def prettyDateNow(): """ Returns a good looking string for the local time. """ return datetime.now().ctime() + def utcPrettyDateNow(): """ Returns a good looking string for utc time. """ return datetime.utcnow().ctime() + def timeToPrettyDate(time_val): return time.ctime(time_val) -class InvalidTimestampFormat(Exception): - pass def fromTimestamp(s): """ @@ -47,17 +52,23 @@ def fromTimestamp(s): ex. 1912-06-23T101234Z" Note: we currently only support parsing strings that are generated from the - timestamp function and have no intention in supporting the full standard. + timestamp function and have no intention in supporting the full + standard. """ try: date_part, time_part = s.split('T') hours, minutes, seconds = time_part[:2], time_part[2:4], time_part[4:6] year, month, day = date_part.split('-') except: - raise InvalidTimestampFormat(s) + raise e.InvalidTimestampFormat(s) + + return datetime(int(year), + int(month), + int(day), + int(hours), + int(minutes), + int(seconds)) - return datetime(int(year), int(month), int(day), int(hours), int(minutes), - int(seconds)) def timestamp(t=None): """ @@ -86,4 +97,3 @@ def timestamp(t=None): t = datetime.utcnow() ISO8601 = "%Y-%m-%dT%H%M%SZ" return t.strftime(ISO8601) - diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index cbf1c08..e81dd21 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -1,5 +1,3 @@ -import json -import os import yaml from oonib import errors as e @@ -7,6 +5,7 @@ from oonib.handlers import OONIBHandler from oonib.config import config + class Policy(object): nettest = None input = None @@ -38,10 +37,12 @@ class Policy(object): if not any(nt['name'] == nettest_name for nt in self.nettest): raise e.InvalidNettestName + class PolicyHandler(OONIBHandler): def initialize(self): self.policy = Policy() + class NetTestPolicyHandler(PolicyHandler): def get(self): """ @@ -49,10 +50,10 @@ class NetTestPolicyHandler(PolicyHandler): """ self.write(self.policy.nettest) + class InputPolicyHandler(PolicyHandler): def get(self): """ return list of input ids """ self.write(self.policy.input) - diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 34aa8d9..01a665a 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -1,5 +1,3 @@ -import random -import string import time import yaml import json @@ -16,35 +14,48 @@ from datetime import datetime from oonib import randomStr, otime, log from oonib.config import config -class MissingField(Exception): - pass -class InvalidRequestField(Exception): - pass +def report_file_name(report_details): + timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time'])) + dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( + timestamp=timestamp, + **report_details) + return dst_filename class Report(object): - def __init__(self, report_id): + delayed_call = None + + def __init__(self, report_id, + stale_time, + report_dir, + archive_dir, + reports): self.report_id = report_id - self.delayed_call = None + + self.stale_time = stale_time + self.report_dir = report_dir + self.archive_dir = archive_dir + self.reports = reports self.refresh() - + def refresh(self): self.last_updated = time.time() if self.delayed_call: self.delayed_call.cancel() - self.delayed_call = reactor.callLater(config.main.stale_time, self.stale_check) + self.delayed_call = reactor.callLater(self.stale_time, + self.stale_check) def stale_check(self): - if (time.time() - self.last_updated) > config.main.stale_time: + if (time.time() - self.last_updated) > self.stale_time: try: self.close() - except ReportNotFound: + except e.ReportNotFound: pass def close(self): def get_report_path(report_id): - return os.path.join(config.main.report_dir, report_id) + return os.path.join(self.report_dir, report_id) report_filename = get_report_path(self.report_id) try: @@ -52,14 +63,10 @@ class Report(object): g = yaml.safe_load_all(fd) report_details = g.next() except IOError: - raise ReportNotFound - - timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time'])) - dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( - timestamp=timestamp, - **report_details) + raise e.ReportNotFound - dst_path = os.path.join(config.main.archive_dir, + dst_filename = report_file_name(report_details) + dst_path = os.path.join(self.archive_dir, report_details['probe_cc']) if not os.path.isdir(dst_path): @@ -68,7 +75,8 @@ class Report(object): dst_path = os.path.join(dst_path, dst_filename) os.rename(report_filename, dst_path) - del config.reports[self.report_id] + self.delayed_call.cancel() + del self.reports[self.report_id] def parseUpdateReportRequest(request): #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") @@ -85,10 +93,10 @@ def parseUpdateReportRequest(request): try: report_id = parsed_request['report_id'] except KeyError: - raise MissingField('report_id') + raise e.MissingField('report_id') if not re.match(report_id_regexp, report_id): - raise InvalidRequestField('report_id') + raise e.InvalidRequestField('report_id') return parsed_request @@ -117,79 +125,82 @@ def parseNewReportRequest(request): try: value_to_check = parsed_request[k] except KeyError: - raise MissingField(k) + raise e.MissingField(k) print "Matching %s with %s | %s" % (regexp, value_to_check, k) if re.match(regexp, str(value_to_check)): continue else: - raise InvalidRequestField(k) - + raise e.InvalidRequestField(k) + try: requested_test_helper = parsed_request['test_helper'] if not re.match(test_helper, str(requested_test_helper)): - raise InvalidRequestField('test_helper') + raise e.InvalidRequestField('test_helper') except KeyError: pass return parsed_request -class InvalidReportHeader(Exception): - pass - -class MissingReportHeaderKey(InvalidReportHeader): - pass - def validate_report_header(report_header): required_keys = ['probe_asn', 'probe_cc', 'probe_ip', 'software_name', 'software_version', 'test_name', 'test_version'] for key in required_keys: if key not in report_header: - raise MissingReportHeaderKey(key) + raise e.MissingReportHeaderKey(key) if report_header['probe_asn'] is None: report_header['probe_asn'] = 'AS0' if not re.match('AS[0-9]+$', report_header['probe_asn']): - raise InvalidReportHeader('probe_asn') + raise e.InvalidReportHeader('probe_asn') # If no country is known, set it to be ZZ (user assigned value in ISO 3166) if report_header['probe_cc'] is None: report_header['probe_cc'] = 'ZZ' if not re.match('[a-zA-Z]{2}$', report_header['probe_cc']): - raise InvalidReportHeader('probe_cc') + raise e.InvalidReportHeader('probe_cc') if not re.match('[a-z_\-]+$', report_header['test_name']): - raise InvalidReportHeader('test_name') + raise e.InvalidReportHeader('test_name') if not re.match('([0-9]+\.)+[0-9]+$', report_header['test_version']): - raise InvalidReportHeader('test_version') + raise e.InvalidReportHeader('test_version') return report_header +class ReportHandler(OONIBHandler): + def initialize(self): + self.archive_dir = config.main.archive_dir + self.report_dir = config.main.report_dir + self.reports = config.reports + self.policy_file = config.main.policy_file + self.helpers = config.helpers + self.stale_time = config.main.stale_time + class UpdateReportMixin(object): def updateReport(self, report_id, parsed_request): log.debug("Got this request %s" % parsed_request) - report_filename = os.path.join(config.main.report_dir, + report_filename = os.path.join(self.report_dir, report_id) - - config.reports[report_id].refresh() + + self.reports[report_id].refresh() try: with open(report_filename, 'a+') as fd: - fdesc.setNonBlocking(fd.fileno()) - fdesc.writeToFD(fd.fileno(), parsed_request['content']) - except IOError as exc: + fd.write(parsed_request['content']) + except IOError: e.OONIBError(404, "Report not found") - self.write({}) + self.write({'status': 'success'}) -class NewReportHandlerFile(OONIBHandler, UpdateReportMixin): +class NewReportHandlerFile(ReportHandler, UpdateReportMixin): """ Responsible for creating and updating reports by writing to flat file. """ + inputHashes = None def checkPolicy(self): policy = Policy() @@ -236,42 +247,31 @@ class NewReportHandlerFile(OONIBHandler, UpdateReportMixin): {'backend_version': 'XXX', 'report_id': 'XXX'} """ - # XXX here we should validate and sanitize the request - try: - report_data = parseNewReportRequest(self.request.body) - except InvalidRequestField as exc: - raise e.InvalidRequestField(exc) - except MissingField as exc: - raise e.MissingRequestField(exc) + # Note: the request is being validated inside of parseNewReportRequest. + report_data = parseNewReportRequest(self.request.body) log.debug("Parsed this data %s" % report_data) software_name = str(report_data['software_name']) software_version = str(report_data['software_version']) - + probe_asn = str(report_data['probe_asn']) probe_cc = str(report_data.get('probe_cc', 'ZZ')) self.testName = str(report_data['test_name']) self.testVersion = str(report_data['test_version']) - - if config.main.policy_file: + + if self.policy_file: try: self.inputHashes = report_data['input_hashes'] except KeyError: raise e.InputHashNotProvided self.checkPolicy() - + if 'content' in report_data: content = yaml.safe_load(report_data['content']) - try: - report_header = validate_report_header(content) - - except MissingReportHeaderKey, key: - raise e.MissingReportHeaderKey(key) + report_header = validate_report_header(content) - except InvalidReportHeader, key: - raise e.InvalidReportHeader(key) else: content = { 'software_name': software_name, @@ -298,22 +298,26 @@ class NewReportHandlerFile(OONIBHandler, UpdateReportMixin): # The report filename contains the timestamp of the report plus a # random nonce - report_filename = os.path.join(config.main.report_dir, report_id) + report_filename = os.path.join(self.report_dir, report_id) response = { 'backend_version': config.backend_version, 'report_id': report_id } - + requested_helper = report_data.get('test_helper') if requested_helper: try: - response['test_helper_address'] = config.helpers[requested_helper].address + response['test_helper_address'] = self.helpers[requested_helper].address except KeyError: raise e.TestHelperNotFound - - config.reports[report_id] = Report(report_id) + + self.reports[report_id] = Report(report_id, + self.stale_time, + self.report_dir, + self.archive_dir, + self.reports) self.writeToReport(report_filename, content) @@ -338,7 +342,7 @@ class NewReportHandlerFile(OONIBHandler, UpdateReportMixin): self.updateReport(report_id, parsed_request) -class UpdateReportHandlerFile(OONIBHandler, UpdateReportMixin): +class UpdateReportHandlerFile(ReportHandler, UpdateReportMixin): def post(self, report_id): try: parsed_request = json.loads(self.request.body) @@ -346,20 +350,17 @@ class UpdateReportHandlerFile(OONIBHandler, UpdateReportMixin): raise e.InvalidRequest self.updateReport(report_id, parsed_request) -class ReportNotFound(Exception): - pass - -class CloseReportHandlerFile(OONIBHandler): +class CloseReportHandlerFile(ReportHandler): def get(self): pass def post(self, report_id): - if report_id in config.reports: - config.reports[report_id].close() + if report_id in self.reports: + self.reports[report_id].close() else: raise e.ReportNotFound -class PCAPReportHandler(OONIBHandler): +class PCAPReportHandler(ReportHandler): def get(self): pass diff --git a/oonib/runner.py b/oonib/runner.py index c8fafa1..d413221 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -1,7 +1,3 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò, Isis Lovecruft -# :licence: see LICENSE for details """ In here we define a runner for the oonib backend system. """ @@ -13,14 +9,12 @@ import os from shutil import rmtree -from twisted.internet import reactor -from twisted.application import service, internet, app +from twisted.internet import reactor, endpoints from twisted.python.runtime import platformType from txtorcon import TCPHiddenServiceEndpoint, TorConfig from txtorcon import launch_tor -from oonib.report.api import reportAPI from oonib.api import ooniBackend, ooniBouncer from oonib.config import config @@ -28,15 +22,16 @@ from oonib import oonibackend from oonib import log from txtorcon import __version__ as txtorcon_version -if tuple(map(int, txtorcon_version.split('.'))) < (0,9,0): +if tuple(map(int, txtorcon_version.split('.'))) < (0, 9, 0): """ Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint listens on all interfaces by default. """ def create_listener(self, proto): self._update_onion(self.hiddenservice.dir) - self.tcp_endpoint = TCP4ServerEndpoint(self.reactor, self.listen_port, # XXX missing import -- has this code ever been tested? - interface='127.0.0.1') + self.tcp_endpoint = endpoints.TCP4ServerEndpoint(self.reactor, + self.listen_port, + interface='127.0.0.1') d = self.tcp_endpoint.listen(self.protocolfactory) d.addCallback(self._add_attributes).addErrback(self._retry_local_port) return d @@ -64,13 +59,17 @@ else: def setupHSEndpoint(self, tor_process_protocol, torconfig, endpoint): endpointName = endpoint.settings['name'] + def setup_complete(port): - print("Exposed %s Tor hidden service on httpo://%s" % (endpointName, - port.onion_uri)) + print("Exposed %s Tor hidden service " + "on httpo://%s" % (endpointName, port.onion_uri)) public_port = 80 - hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, - data_dir=os.path.join(torconfig.DataDirectory, endpointName)) + data_dir = os.path.join(torconfig.DataDirectory, endpointName) + hs_endpoint = TCPHiddenServiceEndpoint(reactor, + torconfig, + public_port, + data_dir=data_dir) d = hs_endpoint.listen(endpoint) d.addCallback(setup_complete) d.addErrback(self.txSetupFailed) diff --git a/oonib/testhelpers/dns_helpers.py b/oonib/testhelpers/dns_helpers.py index 9b2e5e3..fbf932b 100644 --- a/oonib/testhelpers/dns_helpers.py +++ b/oonib/testhelpers/dns_helpers.py @@ -1,14 +1,12 @@ -from twisted.internet.protocol import Factory, Protocol -from twisted.internet import reactor -from twisted.names import dns from twisted.names import client, server from oonib.config import config + class DNSTestHelper(server.DNSServerFactory): - def __init__(self, authorities = None, - caches = None, clients = None, - verbose = 0): + def __init__(self, authorities=None, + caches=None, clients=None, + verbose=0): try: host, port = config.helpers.dns.split(':') port = int(port) @@ -17,8 +15,9 @@ class DNSTestHelper(server.DNSServerFactory): except: host, port = '8.8.8.8', 53 resolver = client.Resolver(servers=[(host, port)]) - server.DNSServerFactory.__init__(self, authorities = authorities, - caches = caches, clients = [resolver], - verbose = verbose) + server.DNSServerFactory.__init__(self, authorities=authorities, + caches=caches, clients=[resolver], + verbose=verbose) + def handleQuery(self, message, protocol, address): server.DNSServerFactory.handleQuery(self, message, protocol, address) diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py index 245e08a..b2afc59 100644 --- a/oonib/testhelpers/http_helpers.py +++ b/oonib/testhelpers/http_helpers.py @@ -93,10 +93,11 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): headers_dict[k] = [] headers_dict[k].append(v) - response = {'request_headers': self.headers, - 'request_line': self.requestLine, - 'headers_dict': headers_dict - } + response = { + 'request_headers': self.headers, + 'request_line': self.requestLine, + 'headers_dict': headers_dict + } json_response = json.dumps(response) self.transport.write('HTTP/1.1 200 OK\r\n\r\n') self.transport.write('%s' % json_response) diff --git a/oonib/testhelpers/ssl_helpers.py b/oonib/testhelpers/ssl_helpers.py index d511ddd..4b84a8c 100644 --- a/oonib/testhelpers/ssl_helpers.py +++ b/oonib/testhelpers/ssl_helpers.py @@ -3,7 +3,7 @@ from oonib.config import config class SSLContext(ssl.DefaultOpenSSLContextFactory): def __init__(self, *args, **kw): - ssl.DefaultOpenSSLContextFactory.__init__(self, + ssl.DefaultOpenSSLContextFactory.__init__(self, config.helpers.ssl.private_key, config.helpers.ssl.certificate) diff --git a/oonib/testhelpers/tcp_helpers.py b/oonib/testhelpers/tcp_helpers.py index 91c334b..614cf87 100644 --- a/oonib/testhelpers/tcp_helpers.py +++ b/oonib/testhelpers/tcp_helpers.py @@ -1,16 +1,16 @@ - from twisted.internet.protocol import Protocol, Factory, ServerFactory -from twisted.internet.error import ConnectionDone from oonib.config import config from oonib import log from oonib.daphn3 import Daphn3Protocol from oonib.daphn3 import read_pcap, read_yaml + class TCPEchoProtocol(Protocol): def dataReceived(self, data): self.transport.write(data) + class TCPEchoHelper(Factory): """ A very simple echo protocol implementation @@ -24,8 +24,11 @@ elif config.helpers.daphn3.pcap_file: daphn3Steps = read_yaml(config.helpers.daphn3.pcap_file) else: - daphn3Steps = [{'client': 'client_packet'}, - {'server': 'server_packet'}] + daphn3Steps = [ + {'client': 'client_packet'}, + {'server': 'server_packet'} + ] + class Daphn3ServerProtocol(Daphn3Protocol): def nextStep(self):

1 0

[oonib/master] Merge pull request #47 from TheTorProject/fix/refactoring
by art＠torproject.org 30 Apr '14

30 Apr '14

commit ebb8c55980a6e8a804643e453a4c6777cd0316e1 Merge: 0ee9875 699c26c Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Wed Apr 30 19:32:26 2014 +0200 Merge pull request #47 from TheTorProject/fix/refactoring Fix/refactoring oonib/bouncer/handlers.py | 17 ++-- oonib/config.py | 11 ++- oonib/daphn3.py | 4 +- oonib/deck/api.py | 4 +- oonib/deck/handlers.py | 10 ++- oonib/errors.py | 50 +++++++++-- oonib/handlers.py | 4 + oonib/input/api.py | 4 +- oonib/input/handlers.py | 21 ++--- oonib/log.py | 4 +- oonib/options.py | 14 ++- oonib/otime.py | 24 +++-- oonib/policy/handlers.py | 7 +- oonib/report/handlers.py | 155 ++++++++++++++++----------------- oonib/runner.py | 27 +++--- oonib/test/handler_helpers.py | 62 +++++++++++++ oonib/test/test_report.py | 173 +++++++++++++++++++++++++++++++++++++ oonib/testhelpers/dns_helpers.py | 17 ++-- oonib/testhelpers/http_helpers.py | 9 +- oonib/testhelpers/ssl_helpers.py | 2 +- oonib/testhelpers/tcp_helpers.py | 11 ++- 21 files changed, 460 insertions(+), 170 deletions(-)

1 0

[ooni-probe/master] Merge pull request #315 from TheTorProject/feature/manpage
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 1d308e3f51df4b5d289140f4c5469a62111c7a71 Merge: e6518fd 9f3f95a Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Wed Apr 30 19:32:45 2014 +0200 Merge pull request #315 from TheTorProject/feature/manpage Add ooniprobe manpage. data/ooni.1 | 1100 +++++++++++++++++++++++++++++++++++++++++++++++++++ docs/source/conf.py | 2 +- ooni/oonicli.py | 11 +- 3 files changed, 1110 insertions(+), 3 deletions(-)

1 0

[ooni-probe/master] Add ooniprobe manpage.
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 9f3f95a990b9a2f33d93b407a2bc68758b92cbec Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Apr 29 19:13:00 2014 +0200 Add ooniprobe manpage. --- data/ooni.1 | 1100 +++++++++++++++++++++++++++++++++++++++++++++++++++ docs/source/conf.py | 2 +- ooni/oonicli.py | 11 +- 3 files changed, 1110 insertions(+), 3 deletions(-) diff --git a/data/ooni.1 b/data/ooni.1 new file mode 100644 index 0000000..076b31b --- /dev/null +++ b/data/ooni.1 @@ -0,0 +1,1100 @@ +.\" Man page generated from reStructuredText. +. +.TH "OONI" "1" "April 29, 2014" "0.1" "OONI" +.SH NAME +ooniprobe - a network censorship measurement tool. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. + +.SH SYNOPSIS +.B ooniprobe +.RB [ \-hnsp ] +.RB [ --version ] +.RB [ --spew ] +.RB [ \-o +.IR reportfile ] +.RB [ \-i +.IR testdeck ] +.RB [ \-c +.IR collector ] +.RB [ \-b +.IR bouncer ] +.RB [ \-l +.IR logfile ] +.RB [ \-O +.IR pcapfile ] +.RB [ \-f +.IR configfile ] +.RB [ \-d +.IR datadir ] +.I "test_name" + +.SH DESCRIPTION +.sp +ooniprobe is tool for performing internet censorship measurements. Our goal is +to achieve a command data format and set of methodologies for conducting +censorship related research. + +.SH OPTIONS + +.TP +.BR \-\^h " or " \-\-help +Display this help and exit. +.TP +.BR \-\^n " or " \-\-no\-collector +Disable reporting the net test results to an oonib collector. +.TP +.BR \-\^s " or " \-\-list +List all of the available net tests. +.TP +.BR \-\^p " or " \-\-printdeck +Print to standard output the specified command line options as an ooniprobe test deck. +.TP +.BR \-\^o " or " \-\-reportfile +Specify the path to report file to write. +.TP +.BR \-\^i " or " \-\-testdeck +Specify as input a test deck: a yaml file containing the tests to run and their +arguments. +.TP +.BR \-\^c " or " \-\-collector +Specify the address of the collector of net test results. It is advisable to +always specify and bouncer and let it return a collector for the test or test +deck you are running. +.TP +.BR \-\^b " or " \-\-bouncer +Address of the bouncer that will inform the probe of which collector to use and +the addresses of test helpers. default: httpo://nkvphnp3p6agi5qq.onion +.TP +.BR \-\^l " or " \-\-logfile +Path to the log file to write +.TP +.BR \-\^O " or " \-\-pcapfile +Path to the pcap file name. +.TP +.BR \-\^f " or " \-\-configfile +Specify a path to the ooniprobe configuration file. +.TP +.BR \-\^d " or " \-\-datadir +Specify a path to the ooniprobe data directory +.TP +.BR \-\-spew +Print an insanely verbose log of everything that happens. +Useful when debugging freezes or locks in complex code. +.TP +.BR \-\-version +Display the ooniprobe version and exit. + +.SH OONIPROBE +.sp +Is the tool that volunteers and researches interested in contributing data to +the project should be running. +.sp +ooniprobe allows the user to select what test should be run and what backend +should be used for storing the test report and/or assisting them in the running +of the test. +.sp +ooniprobe tests are divided into two categories: \fBTraffic Manipulation\fP and +\fBContent Blocking\fP\&. +.sp +\fBTraffic Manipulation\fP tests aim to detect the presence of some sort of +tampering with the internet traffic between the probe and a remote test helper +backend. As such they usually require the selection of a oonib backend +component for running the test. +.sp +\fBContent Blocking\fP are aimed at enumerating the kind of content that is +blocked from the probes network point of view. As such they usually require to +have specified an input list for running the test. +.SS Threat Model +.sp +Our adversary is capable of doing country wide network surveillance and +manipulation of network traffic. +.sp +The goals of our adversary are: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.IP \(bu 2 +Restrict access to certain content, while not degrading overall quality of +the network +.IP \(bu 2 +Monitor the network in a way that they are able to identify misuse of it in +real time +.UNINDENT +.UNINDENT +.UNINDENT +.sp +More specifc to the running of network filtering detection tests: +.INDENT 0.0 +.IP 1. 3 +Detect actors performing censorship detection tests +.IP 2. 3 +Fool people running such tests into believing that the network is +unrestricted + +.UNINDENT +.sp +\fINote\fP that while 2) => 1) it is not true that 1) => 2) as the identification of +such actors does not necessarily have to happen in real time. +While our intention is to minimize the risk of users running OONI probe to be +identified, this comes with a tradeoff in accuracy. It is therefore necessary in +certain tests to trade\-off fingerprintability in favour of tests accuracy. +.sp +This is why we divide tests based on what risk the user running it can face, +allowing the user to freely choose what threat model they wish to adere to. +.SS Installation +.sp +\fBRead this before running ooniprobe!\fP +.sp +Running ooniprobe is a potentially risky activity. This greatly depends on the +jurisdiction in which you are in and which test you are running. It is +technically possible for a person observing your internet connection to be +aware of the fact that you are running ooniprobe. This means that if running +network measurement tests is something considered to be illegal in your country +then you could be spotted. +.sp +Futhermore, ooniprobe takes no precautions to protect the install target machine +from forensics analysis. If the fact that you have installed or used ooni +probe is a liability for you, please be aware of this risk. +.SS Debian based systems +.sp +\fIsudo sh \-c \(aqecho "deb http://deb.ooni.nu/ooni wheezy main" >> /etc/apt/sources.list\(aq\fP +.sp +\fIgpg \-\-keyserver pgp.mit.edu \-\-recv\-key 0x49B8CDF4\fP +.sp +\fIgpg \-\-export 89AB86D4788F3785FE9EDA31F9E2D9B049B8CDF4 | sudo apt\-key add \-\fP +.sp +\fIsudo apt\-get update && sudo apt\-get install ooniprobe\fP +.SS Linux +.sp +We believe that ooniprobe runs reasonably well on Debian GNU/Linux wheezy as +well as versions of Ubuntu such as natty and later releases. Running ooniprobe +without installing it is supported with the following commands: +.sp +\fIgit clone https://git.torproject.org/ooni\-probe.git\fP +.sp +\fIcd ooni\-probe\fP +.sp +\fI\&./setup\-dependencies.sh\fP +.sp +\fIpython setup.py install\fP +.SS Setting up development environment +.sp +On debian based systems this can be done with: +.sp +\fIVsudo apt\-get install libgeoip\-dev python\-virtualenv virtualenvwrapper\fP +.sp +\fImkvirtualenv ooniprobe\fP +.sp +\fIpython setup.py install\fP +.sp +\fIpip install \-r requirements\-dev.txt\fP +.SS Other platforms (with Vagrant) +.sp +\fI\%Install Vagrant\fP +and \fI\%Install Virtualbox\fP +.sp +\fBOn OSX:\fP +.sp +If you don\(aqt have it install \fI\%homebrew\fP +.sp +\fIbrew install git\fP +.sp +\fBOn debian/ubuntu:\fP +.sp +\fIsudo apt\-get install git\fP +.INDENT 0.0 +.IP 1. 3 +Open a Terminal and run: +.UNINDENT +.sp +\fIgit clone https://git.torproject.org/ooni\-probe.git\fP +.sp +\fIcd ooni\-probe/\fP +.sp +\fIvagrant up\fP +.INDENT 0.0 +.IP 2. 3 +Login to the box with: +.UNINDENT +.sp +\fIvagrant ssh\fP +.sp +ooniprobe will be installed in \fI/ooni\fP\&. +.INDENT 0.0 +.IP 3. 3 +You can run tests with: +.UNINDENT +.sp +\fIooniprobe blocking/http_requests \-f /ooni/inputs/input\-pack/alexa\-top\-1k.txt\fP +.SS Using ooniprobe +.sp +\fBNet test\fP is a set of measurements to assess what kind of internet censorship is occurring. +.sp +\fBDecks\fP are collections of ooniprobe nettests with some associated inputs. +.sp +\fBCollector\fP is a service used to report the results of measurements. +.sp +\fBTest helper\fP is a service used by a probe for successfully performing its measurements. +.sp +\fBBouncer\fP is a service used to discover the addresses of test helpers and collectors. +.SS Configuring ooniprobe +.sp +You may edit the configuration for ooniprobe by editing the configuration file +found inside of \fI~/.ooni/ooniprobe.conf\fP\&. +.sp +By default ooniprobe will not include personal identifying information in the +test result, nor create a pcap file. This behavior can be personalized. +.SS Running decks +.sp +You will find all the installed decks inside of \fI/usr/share/ooni/decks\fP\&. +.sp +You may then run a deck by using the command line option \fI\-i\fP: +.sp +As root: +.sp +\fIooniprobe \-i /usr/share/ooni/decks/mlab.deck\fP +.sp +Or as a user: +.sp +\fIooniprobe \-i /usr/share/ooni/decks/mlab_no_root.deck\fP +.sp +Or: +.sp +As root: +.sp +\fIooniprobe \-i /usr/share/ooni/decks/complete.deck\fP +.sp +Or as a user: +.sp +\fIooniprobe \-i /usr/share/ooni/decks/complete_no_root.deck\fP +.sp +The above tests will require around 20\-30 minutes to complete depending on your network speed. +.sp +If you would prefer to run some faster tests you should run: +As root: +.sp +\fIooniprobe \-i /usr/share/ooni/decks/fast.deck\fP +.sp +Or as a user: +.sp +\fIooniprobe \-i /usr/share/ooni/decks/fast_no_root.deck\fP +.SS Running net tests +.sp +You may list all the installed stable net tests with: +.sp +\fIooniprobe \-s\fP +.sp +You may then run a nettest by specifying its name for example: +.sp +\fIooniprobe manipulation/http_header_field_manipulation\fP +.sp +It is also possible to specify inputs to tests as URLs: +.sp +\fIooniprobe blocking/http_requests \-f httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1\fP +.sp +You can find the result of the test in your current working directory. +.sp +By default the report result will be collected by the default ooni collector +and the addresses of test helpers will be obtained from the default bouncer. +.sp +You may also specify your own collector or bouncer with the options \fI\-c\fP and +\fI\-b\fP\&. +.SS (Optional) Install obfsproxy +.sp +Install the latest version of obfsproxy for your platform. +.sp +\fI\%Download Obfsproxy\fP +.SS Bridges and obfsproxy bridges +.sp +ooniprobe submits reports to oonib report collectors through Tor to a hidden +service endpoint. By default, ooniprobe uses the installed system Tor, but can +also be configured to launch Tor (see the advanced.start_tor option in +ooniprobe.conf), and ooniprobe supports bridges (and obfsproxy bridges, if +obfsproxy is installed). The tor.bridges option in ooniprobe.conf sets the path +to a file that should contain a set of "bridge" lines (of the same format as +used in torrc, and as returned by \fI\%https://bridges.torproject.org\fP). If obfsproxy +bridges are to be used, the path to the obfsproxy binary must be configured. +See option advanced.obfsproxy_binary, in ooniprobe.conf. +.SS Setting capabilities on your virtualenv python binary +.sp +If your distributation supports capabilities you can avoid needing to run OONI as root: +.sp +\fIsetcap cap_net_admin,cap_net_raw+eip /path/to/your/virtualenv\(aqs/python\fP +.SS Core ooniprobe Tests +.sp +The source for \fI\%Content blocking tests\fP +and \fI\%Traffic Manipulation tests\fP +can be found in the nettests/blocking and nettests/manipulation directories +respectively. +.SS Content Blocking Tests +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.IP \(bu 2 +\fI\%DNSConsistency\fP +.IP \(bu 2 +\fI\%HTTP Requests\fP +.IP \(bu 2 +\fI\%TCP Connect\fP +.UNINDENT +.UNINDENT +.UNINDENT +.SS Traffic Manipulation Tests +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.IP \(bu 2 +\fI\%HTTP Invalid Request Line:\fP +.IP \(bu 2 +\fI\%DNS Spoof\fP +.IP \(bu 2 +\fI\%HTTP Header Field Manipulation\fP +.IP \(bu 2 +\fI\%Traceroute\fP +.IP \(bu 2 +\fI\%HTTP Host\fP +.UNINDENT +.UNINDENT +.UNINDENT +.SS Other tests +.sp +We also have some other tests that are currently not fully supported or still +being experimented with. +.sp +You can find these in: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.IP \(bu 2 +\fI\%ooni/nettests/experimental\fP +.UNINDENT +.UNINDENT +.UNINDENT +.sp +Tests that don\(aqt do a measurement but are useful for scanning can be found in: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.IP \(bu 2 +\fI\%ooni/nettests/scanning\fP +.UNINDENT +.UNINDENT +.UNINDENT +.sp +Tests that involve running third party tools may be found in: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.IP \(bu 2 +\fI\%ooni/nettests/third_party\fP +.UNINDENT +.UNINDENT +.UNINDENT +.SS Reports +.sp +The reports collected by ooniprobe are stored on +\fI\%https://ooni.torproject.org/reports/0.1/\fP \fBCC\fP / +.sp +Where \fBCC\fP is the two letter country code as specified by \fI\%ISO 31666\-2\fP\&. +.sp +For example the reports for Italy (\fBCC\fP is \fBit\fP) of the may be found in: +.sp +\fI\%https://ooni.torproject.org/reports/0.1/IT/\fP +.sp +This directory shall contain the various reports for the test using the +following convention: +.sp +\fBtestName\fP \- \fBdateInISO8601Format\fP \- \fBprobeASNumber\fP .yamloo +.sp +The date is expressed using \fI\%ISO 8601\fP +including seconds and with no \fB:\fP to delimit hours, minutes, days. +.sp +Like so: +.sp +\fBYEAR\fP \- \fBMONTH\fP \- \fBDAY\fP T \fBHOURS\fP \fBMINUTES\fP \fBSECONDS\fP Z +.sp +Look \fI\%here for the up to date list of ISO 8601 country codes\fP +.sp +The time is \fBalways\fP expressed in UTC. +.sp +If a collision is detected then an int (starting with 1) will get appended to +the test. +.sp +For example if two report that are created on the first of January 2012 at Noon +(UTC time) sharp from MIT (AS3) will be stored here: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +https://ooni.torproject.org/reports/0.1/US/2012\-01\-01T120000Z_AS3.yamloo +https://ooni.torproject.org/reports/0.1/US/2012\-01\-01T120000Z_AS3.1.yamloo +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Note: it is highly unlikely that reports get created with the same exact +timestamp from the same exact ASN. If this does happen it could be index of +some malicious report poisoning attack in progress. +.SS Report format version changelog +.sp +In here shall go details about the major changes to the reporting format. +.SS version 0.1 +.sp +Initial format version. +.SS Writing OONI tests +.sp +The OONI testing API is heavily influenced and partially based on the python +\fBunittest\fP module and \fBtwisted.trial\fP\&. +.SS Test Cases +.sp +The atom of OONI Testing is called a Test Case. A test case class may contain +multiple Test Methods. +.INDENT 0.0 +.TP +.B class ooni.nettest.NetTestCase +This is the base of the OONI nettest universe. When you write a nettest +you will subclass this object. +.INDENT 7.0 +.IP \(bu 2 +inputs: can be set to a static set of inputs. All the tests (the methods +starting with the "test" prefix) will be run once per input. At every run +the _input_ attribute of the TestCase instance will be set to the value of +the current iteration over inputs. Any python iterable object can be set +to inputs. +.IP \(bu 2 +inputFile: attribute should be set to an array containing the command line +argument that should be used as the input file. Such array looks like +this: +.INDENT 2.0 +.INDENT 3.5 +\fB["commandlinearg", "c", "default value" "The description"]\fP +.UNINDENT +.UNINDENT +.sp +The second value of such arrray is the shorthand for the command line arg. +The user will then be able to specify inputs to the test via: +.INDENT 2.0 +.INDENT 3.5 +\fBooniprobe mytest.py \-\-commandlinearg path/to/file.txt\fP +.UNINDENT +.UNINDENT +.sp +or +.INDENT 2.0 +.INDENT 3.5 +\fBooniprobe mytest.py \-c path/to/file.txt\fP +.UNINDENT +.UNINDENT +.IP \(bu 2 +inputProcessor: should be set to a function that takes as argument a +filename and it will return the input to be passed to the test +instance. +.IP \(bu 2 +name: should be set to the name of the test. +.IP \(bu 2 +author: should contain the name and contact details for the test author. +The format for such string is as follows: +.INDENT 2.0 +.INDENT 3.5 +\fBThe Name <email(a)example.com>\fP +.UNINDENT +.UNINDENT +.IP \(bu 2 +version: is the version string of the test. +.IP \(bu 2 +requiresRoot: set to True if the test must be run as root. +.IP \(bu 2 +usageOptions: a subclass of twisted.python.usage.Options for processing of command line arguments +.IP \(bu 2 +localOptions: contains the parsed command line arguments. +.UNINDENT +.sp +Quirks: +Every class that is prefixed with test \fImust\fP return a twisted.internet.defer.Deferred. +.UNINDENT +.sp +If the test you plan to write is not listed on the \fI\%Tor OONI trac page\fP, you should +add it to the list and then add a description about it following the \fI\%Test +Template\fP +.sp +Tests are driven by inputs. For every input a new test instance is created, +internally the _setUp method is called that is defined inside of test +templates, then the setUp method that is overwritable by users. +.sp +Gotchas: +\fBnever\fP call reactor.start of reactor.stop inside of your test method and all +will be good. +.SS Inputs +.sp +Inputs are what is given as input to every iteration of the Test Case. +Iflyou have 100 inputs, then every test case will be run 100 times. +.sp +To configure a static set of inputs you should define the +\fBooni.nettest.NetTestCase\fP attribute \fBinputs\fP\&. The test will be +run \fBlen(inputs)\fP times. Any iterable object is a valid \fBinputs\fP +attribute. +.sp +If you would like to have inputs be determined from a user specified input +file, then you must set the \fBinputFile\fP attribute. This is an array that +specifies what command line option may be used to control this value. +.sp +By default the \fBinputProcessor\fP is set to read the file line by line and +strip newline characters. To change this behavior you must set the +\fBinputProcessor\fP attribute to a function that takes as argument a file +descriptor and yield the next item. The default \fBinputProcessor\fP looks like +this: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +def lineByLine(filename): + fp = open(filename) + for x in fp.xreadlines(): + yield x.strip() + fp.close() +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Setup and command line passing +.sp +Tests may define the \fIsetUp\fP method that will be called every time the +Test Case object is instantiated, in here you may place some common logic +to all your Test Methods that should be run before any testing occurs. +.sp +Command line arguments can be parsed thanks to the twisted +\fBtwisted.python.usage.UsageOptions\fP class. +.sp +You will have to subclass this and define the NetTestCase attribute +usageOptions to point to a subclass of this. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +class UsageOptions(usage.Options): + optParameters = [[\(aqbackend\(aq, \(aqb\(aq, \(aqhttp://127.0.0.1:57001\(aq, + \(aqURL of the test backend to use\(aq] + ] + +class MyTestCase(nettest.NetTestCase): + usageOptions = UsageOptions + + inputFile = [\(aqfile\(aq, \(aqf\(aq, None, "Some foo file"] + requiredOptions = [\(aqbackend\(aq] + + def test_my_test(self): + self.localOptions[\(aqbackend\(aq] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +You will then be able to access the parsed command line arguments via the +class attribute localOptions. +.sp +The \fIrequiredOptions\fP attributes specifies an array of parameters that are +required for the test to run properly. +.sp +\fIinputFile\fP is a special class attribute that will be used for processing +of the inputFile. The filename that is read here will be given to the +\fBooni.nettest.NetTestCase.inputProcessor\fP method that will yield, by +default, one line of the file at a time. +.SS Test Methods +.sp +These shall be defined inside of your \fBooni.nettest.NetTestCase\fP +subclass. These will be class methods. +.sp +All class methods that are prefixed with test_ shall be run. Functions +that are relevant to your test should be all lowercase separated by +underscore. +.sp +To add data to the test report you may write directly to the report object +like so: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +def test_my_function(): + result = do_something() + self.report[\(aqsomething\(aq] = result +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +OONI will then handle the writing of the data to the final test report. +.sp +To access the current input you can use the \fBinput\fP attribute, for example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +def test_with_input(): + do_something_with_input(self.input) +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will at each iteration over the list of inputs do something with the +input. +.SS Test Templates +.sp +Test templates assist you in writing tests. They already contain all the +common functionality that is useful to running a test of that type. They +also take care of writing the data they collect that is relevant to the +test run to the report file. +.sp +Currently implemented test templates are \fBooni.templates.scapyt\fP for +tests based on Scapy, \fBooni.templates.tcpt\fP for tests based on TCP, +\fBooni.templates.httpt\fP for tests based on HTTP, and +\fBooni.templates.dnst\fP for tests based on DNS. +.SS Scapy based tests +.sp +Scapy based tests will be a subclass of \fBooni.templates.scapyt.BaseScapyTest\fP\&. +.sp +It provides a wrapper around the scapy send and receive function that will +write the sent and received packets to the report with sanitization of the +src and destination IP addresses. +.sp +It has the same syntax as the Scapy sr function, except that it will +return a deferred. +.sp +To implement a simple ICMP ping based on this function you can do like so +(Taken from \fBnettest.examples.example_scapyt.ExampleICMPPingScapy\fP) +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +from twisted.python import usage + +from scapy.all import IP, ICMP + +from ooni.templates import scapyt + +class UsageOptions(usage.Options): + optParameters = [[\(aqtarget\(aq, \(aqt\(aq, \(aq127.0.0.1\(aq, "Specify the target to ping"]] + +class ExampleICMPPingScapy(scapyt.BaseScapyTest): + name = "Example ICMP Ping Test" + + usageOptions = UsageOptions + + def test_icmp_ping(self): + def finished(packets): + print packets + answered, unanswered = packets + for snd, rcv in answered: + rcv.show() + + packets = IP(dst=self.localOptions[\(aqtarget\(aq])/ICMP() + d = self.sr(packets) + d.addCallback(finished) + return d +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The arguments taken by self.sr() are exactly the same as the scapy send and +receive function, the only difference is that instead of using the regular +scapy super socket it uses our twisted driven wrapper around it. +.sp +Alternatively this test can also be written using the +\fBtwisted.defer.inlineCallbacks()\fP decorator, that makes it look more similar to +regular sequential code. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +from twisted.python import usage +from twisted.internet import defer + +from scapy.all import IP, ICMP + +from ooni.templates import scapyt + +class UsageOptions(usage.Options): + optParameters = [[\(aqtarget\(aq, \(aqt\(aq, \(aq127.0.0.1\(aq, "Specify the target to ping"]] + +class ExampleICMPPingScapyYield(scapyt.BaseScapyTest): + name = "Example ICMP Ping Test" + + usageOptions = UsageOptions + + @defer.inlineCallbacks + def test_icmp_ping(self): + packets = IP(dst=self.localOptions[\(aqtarget\(aq])/ICMP() + answered, unanswered = yield self.sr(packets) + for snd, rcv in answered: + rcv.show() +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Report Format +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +########################################### +# OONI Probe Report for Example ICMP Ping Test test +# Thu Nov 22 18:20:43 2012 +########################################### +\-\-\- +{probe_asn: null, probe_cc: null, probe_ip: 127.0.0.1, software_name: ooniprobe, software_version: 0.0.7.1\-alpha, + start_time: 1353601243.0, test_name: Example ICMP Ping Test, test_version: 0.1} +\&... +\-\-\- +input: null +report: + answer_flags: [ipsrc] + answered_packets: + \- \- raw_packet: !!binary | + RQAAHAEdAAAuAbjKCAgICH8AAAEAAAAAAAAAAA== + summary: IP / ICMP 8.8.8.8 > 127.0.0.1 echo\-reply 0 + sent_packets: + \- \- raw_packet: !!binary | + RQAAHAABAABAAevPfwAAAQgICAgIAPf/AAAAAA== + summary: IP / ICMP 127.0.0.1 > 8.8.8.8 echo\-request 0 +test_name: test_icmp_ping +test_started: 1353604843.553605 +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.SS TCP based tests +.sp +TCP based tests will subclass \fBooni.templates.tcpt.TCPTest\fP\&. +.sp +This test template facilitates the sending of TCP payloads to the wire and +recording the response. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +from twisted.internet.error import ConnectionRefusedError +from ooni.utils import log +from ooni.templates import tcpt + +class ExampleTCPT(tcpt.TCPTest): + def test_hello_world(self): + def got_response(response): + print "Got this data %s" % response + + def connection_failed(failure): + failure.trap(ConnectionRefusedError) + print "Connection Refused" + + self.address = "127.0.0.1" + self.port = 57002 + payload = "Hello World!\en\er" + d = self.sendPayload(payload) + d.addErrback(connection_failed) + d.addCallback(got_response) + return d +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The possible failures for a TCP connection are: +.sp +\fBtwisted.internet.error.NoRouteError\fP that corresponds to errno.ENETUNREACH +.sp +\fBtwisted.internet.error.ConnectionRefusedError\fP that corresponds to +errno.ECONNREFUSED +.sp +\fBtwisted.internet.error.TCPTimedOutError\fP that corresponds to errno.ETIMEDOUT +.SS Report format +.sp +The basic report of a TCP test looks like the following (this is an report +generated by running the above example against a TCP echo server). +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +########################################### +# OONI Probe Report for Base TCP Test test +# Thu Nov 22 18:18:28 2012 +########################################### +\-\-\- +{probe_asn: null, probe_cc: null, probe_ip: 127.0.0.1, software_name: ooniprobe, software_version: 0.0.7.1\-alpha, + start_time: 1353601108.0, test_name: Base TCP Test, test_version: \(aq0.1\(aq} +\&... +\-\-\- +input: null +report: + errors: [] + received: ["Hello World!\en\er"] + sent: ["Hello World!\en\er"] +test_name: test_hello_world +test_started: 1353604708.705081 +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +TODO finish this with more details +.SS HTTP based tests +.sp +HTTP based tests will be a subclass of \fBooni.templates.httpt.HTTPTest\fP\&. +.sp +It provides methods \fBooni.templates.httpt.HTTPTest.processResponseBody()\fP and +\fBooni.templates.httpt.HTTPTest.processResponseHeaders()\fP for interacting with the +response body and headers respectively. +.sp +For example, to implement a HTTP test that returns the sha256 hash of the +response body (based on \fBnettests.examples.example_httpt\fP): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +from ooni.utils import log +from ooni.templates import httpt +from hashlib import sha256 + +class SHA256HTTPBodyTest(httpt.HTTPTest): + name = "ChecksumHTTPBodyTest" + author = "Aaron Gibson" + version = 0.1 + + inputFile = [\(aqurl file\(aq, \(aqf\(aq, None, + \(aqList of URLS to perform GET requests to\(aq] + requiredOptions = [\(aqurl file\(aq] + + def test_http(self): + if self.input: + url = self.input + return self.doRequest(url) + else: + raise Exception("No input specified") + + def processResponseBody(self, body): + body_sha256sum = sha256(body).hexdigest() + self.report[\(aqchecksum\(aq] = body_sha256sum +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Report format +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +########################################### +# OONI Probe Report for ChecksumHTTPBodyTest test +# Thu Dec 6 17:31:57 2012 +########################################### +\-\-\- +options: + collector: null + help: 0 + logfile: null + pcapfile: null + reportfile: null + resume: 0 + subargs: [\-f, hosts] + test: nettests/examples/example_http_checksum.py +probe_asn: null +probe_cc: null +probe_ip: 127.0.0.1 +software_name: ooniprobe +software_version: 0.0.7.1\-alpha +start_time: 1354786317.0 +test_name: ChecksumHTTPBodyTest +test_version: 0.1 +\&... +\-\-\- +input: http://www.google.com +report: + agent: agent + checksum: d630fa2efd547d3656e349e96ff7af5496889dad959e8e29212af1ff843e7aa1 + requests: + \- request: + body: null + headers: + \- \- User\-Agent + \- \- [Opera/9.00 (Windows NT 5.1; U; en), \(aqOpera 9.0, Windows XP\(aq] + method: GET + url: http://www.google.com + response: + body: \(aq<!doctype html><html ... snip ... </html>\(aq + code: 200 + headers: + \- \- X\-XSS\-Protection + \- [1; mode=block] + \- \- Set\-Cookie + \- [\(aqPREF=ID=fada4216eb3684f9:FF=0:TM=1354800717:LM=1354800717:S=IT\-2GCkNAocyXlVa; + expires=Sat, 06\-Dec\-2014 13:31:57 GMT; path=/; domain=.google.com\(aq, \(aqNID=66=KWaLbNQumuGuYf0HrWlGm54u9l\-DKJwhFCMQXfhQPZM\-qniRhmF6QRGXUKXb_8CIUuCOHnyoC5oAX5jWNrsfk\-LLJLW530UiMp6hemTtDMh_e6GSiEB4GR3yOP_E0TCN; + expires=Fri, 07\-Jun\-2013 13:31:57 GMT; path=/; domain=.google.com; HttpOnly\(aq] + \- \- Expires + \- [\(aq\-1\(aq] + \- \- Server + \- [gws] + \- \- Connection + \- [close] + \- \- Cache\-Control + \- [\(aqprivate, max\-age=0\(aq] + \- \- Date + \- [\(aqThu, 06 Dec 2012 13:31:57 GMT\(aq] + \- \- P3P + \- [\(aqCP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 + for more info."\(aq] + \- \- Content\-Type + \- [text/html; charset=UTF\-8] + \- \- X\-Frame\-Options + \- [SAMEORIGIN] + socksproxy: null +test_name: test_http +test_runtime: 0.08298492431640625 +test_started: 1354800717.478403 +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.SS DNS based tests +.sp +DNS based tests will be a subclass of \fBooni.templates.dnst.DNSTest\fP\&. +.sp +It provides methods \fBooni.templates.dnst.DNSTest.performPTRLookup()\fP +and \fBooni.templates.dnst.DNSTest.performALookup()\fP +.sp +For example (taken from \fBnettests.examples.example_dnst\fP): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +from ooni.templates.dnst import DNSTest + +class ExampleDNSTest(DNSTest): + def test_a_lookup(self): + def gotResult(result): + # Result is an array containing all the A record lookup results + print result + + d = self.performALookup(\(aqtorproject.org\(aq, (\(aq8.8.8.8\(aq, 53)) + d.addCallback(gotResult) + return d +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Report format +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +########################################### +# OONI Probe Report for Base DNS Test test +# Thu Dec 6 17:42:51 2012 +########################################### +\-\-\- +options: + collector: null + help: 0 + logfile: null + pcapfile: null + reportfile: null + resume: 0 + subargs: [] + test: nettests/examples/example_dnst.py +probe_asn: null +probe_cc: null +probe_ip: 127.0.0.1 +software_name: ooniprobe +software_version: 0.0.7.1\-alpha +start_time: 1354786971.0 +test_name: Base DNS Test +test_version: 0.1 +\&... +\-\-\- +input: null +report: + queries: + \- addrs: [82.195.75.101, 86.59.30.40, 38.229.72.14, 38.229.72.16] + answers: + \- [<RR name=torproject.org type=A class=IN ttl=782s auth=False>, <A address=82.195.75.101 + ttl=782>] + \- [<RR name=torproject.org type=A class=IN ttl=782s auth=False>, <A address=86.59.30.40 + ttl=782>] + \- [<RR name=torproject.org type=A class=IN ttl=782s auth=False>, <A address=38.229.72.14 + ttl=782>] + \- [<RR name=torproject.org type=A class=IN ttl=782s auth=False>, <A address=38.229.72.16 + ttl=782>] + query: \(aq[Query(\(aq\(aqtorproject.org\(aq\(aq, 1, 1)]\(aq + query_type: A + resolver: [8.8.8.8, 53] +test_name: test_a_lookup +test_runtime: 0.028924942016601562 +test_started: 1354801371.980114 +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +For a more complex example, see: \fBnettests.blocking.dnsconsistency\fP + +.SH GLOSSARY +.sp +Here we will summarize some of the jargon that is unique to OONI. +.sp +\fBTest Case\fP: a set of measurements performed on a to be tested network that +are logically grouped together +.sp +\fBReport\fP: is the output of a test run containing all the information that is +require for a researcher to assess what is the output of a test. +.sp +\fBYamlooni\fP: The format we use for Reports, that is based on YAML. +.sp +\fBInput\fP: What is given as input to a TestCase to perform a measurement. +.SH AUTHOR +The Tor Project +.SH COPYRIGHT +2014, The Tor Project +. diff --git a/docs/source/conf.py b/docs/source/conf.py index 1362e06..778192a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -216,7 +216,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'ooni', u'OONI Documentation', + ('index', 'ooniprobe', u'an internet censorship measurement tool', [u'The Tor Project'], 1) ] diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 846eadc..889b108 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -10,7 +10,7 @@ from twisted.internet import reactor from twisted.python import usage from twisted.python.util import spewer -from ooni import errors +from ooni import errors, __version__ from ooni.settings import config from ooni.director import Director @@ -69,6 +69,13 @@ class Options(usage.Options): """ sys.settrace(spewer) + def opt_version(self): + """ + Display the ooniprobe version and exit. + """ + print "ooniprobe version:", __version__ + sys.exit(0) + def parseArgs(self, *args): if self['testdeck'] or self['list']: return @@ -81,7 +88,7 @@ class Options(usage.Options): def parseOptions(): print "WARNING: running ooniprobe involves some risk that varies greatly" print " from country to country. You should be aware of this when" - print " running the tool. Read more about this in the README." + print " running the tool. Read more about this in the manpage or README." cmd_line_options = Options() if len(sys.argv) == 1: cmd_line_options.getUsage()

1 0

[ooni-probe/master] Fix typo inside of oonicli
by art＠torproject.org 30 Apr '14

30 Apr '14

commit b705eae8ffaee6134ea0c444c77c985bcbb2d0b9 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Apr 15 21:41:55 2014 +0200 Fix typo inside of oonicli --- ooni/oonicli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 7ac1e76..5c3c0ce 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -233,7 +233,7 @@ def runWithDirector(logging=True, start_tor=True): if global_options['collector']: collector = global_options['collector'] elif 'collector' in config.reports and config.reports['collector']: - collector = config.report['collector'] + collector = config.reports['collector'] elif net_test_loader.collector: collector = net_test_loader.collector

1 0