tor-commits April 2016

tor-commits@lists.torproject.org

20 participants
1645 discussions

[ooni-probe/master] Fix bug that lead to the probe_asn and probe_cc not being set
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 29e63ed80eb9298e3ce86f7d4582b2c6ee909afe Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 30 14:07:22 2016 +0200 Fix bug that lead to the probe_asn and probe_cc not being set --- ooni/director.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ooni/director.py b/ooni/director.py index 4aa5fba..ab8da65 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -241,12 +241,16 @@ class Director(object): net_test_loader: an instance of :class:ooni.nettest.NetTestLoader """ + # Here we set the test details again since the geoip lookups may + # not have already been done and probe_asn and probe_ip + # are not set. + net_test_loader.setTestDetails() + if self.allTestsDone.called: self.allTestsDone = defer.Deferred() if config.privacy.includepcap: self.startSniffing(net_test_loader.testDetails) - report = Report(net_test_loader.testDetails, report_filename, self.reportEntryManager, collector_address, no_yamloo)

1 0

[ooni-probe/master] Add test for YAML based reporter
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 1e9f2316859825343ae2ae2f28b6fd5ff2b4266b Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Mar 26 19:48:00 2016 +0100 Add test for YAML based reporter --- ooni/tests/test_reporter.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 94192ee..9774449 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -36,6 +36,13 @@ oonib_new_report_message = { 'supported_formats': ["yaml", "json"] } +# This is used for testing legacy collectors +oonib_new_report_yaml_message = { + 'report_id': "20140129T202038Z_AS0_" + "A" * 50, + 'backend_version': "1.0" +} + + oonib_generic_error_message = { 'error': 'generic-error' } @@ -103,6 +110,13 @@ class TestOONIBReporter(unittest.TestCase): yield self.oonib_reporter.writeReportEntry(req) assert self.oonib_reporter.agent.request.called + @defer.inlineCallbacks + def test_write_report_entry_in_yaml(self): + self.mock_response = oonib_new_report_yaml_message + yield self.oonib_reporter.createReport() + req = {'content': 'something'} + yield self.oonib_reporter.writeReportEntry(req) + assert self.oonib_reporter.agent.request.called class TestOONIBReportLog(unittest.TestCase):

1 0

[ooni-probe/master] [Refactoring] oonicli logic
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 18385338507bf4c3f5959e0de68a3ad33f291353 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 30 14:08:01 2016 +0200 [Refactoring] oonicli logic --- ooni/oonicli.py | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 9509a18..63312b5 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -209,25 +209,22 @@ def setupAnnotations(global_options): global_options["annotations"] = annotations return annotations -def setupCollector(global_options, net_test_loader): - collector = None - if not global_options['no-collector']: - if global_options['collector']: - collector = global_options['collector'] - elif 'collector' in config.reports \ - and config.reports['collector']: - collector = config.reports['collector'] - elif net_test_loader.collector: - collector = net_test_loader.collector - - if collector and collector.startswith('httpo:') \ +def setupCollector(global_options, collector_address): + if global_options['collector']: + collector_address = global_options['collector'] + elif 'collector' in config.reports \ + and config.reports['collector']: + collector_address = config.reports['collector'] + + if collector_address.startswith('httpo:') \ and (not (config.tor_state or config.tor.socks_port)): raise errors.TorNotRunning - return collector + return collector_address def createDeck(global_options, url=None): - log.msg("Creating deck for: %s" % (url)) + if url: + log.msg("Creating deck for: %s" % (url)) if global_options['no-yamloo']: log.msg("Will not write to a yamloo report file") @@ -311,18 +308,16 @@ def runTestWithDirector(director, global_options, url=None, # If a collector is not specified in the deck, or the # deck is a singleton, the default collector set in # ooniprobe.conf will be used - - collector = setupCollector(global_options, net_test_loader) - - if collector and collector.startswith('httpo:') \ - and (not (config.tor_state or config.tor.socks_port)): - raise errors.TorNotRunning + collector_address = None + if not global_options['no-collector']: + collector_address = setupCollector(global_options, + net_test_loader.collector) net_test_loader.annotations = global_options['annotations'] yield director.startNetTest(net_test_loader, global_options['reportfile'], - collector, + collector_address, global_options['no-yamloo']) d.addCallback(setup_nettest)

1 0

[ooni-probe/master] Merge branch 'master' into feature/json_reports
by art＠torproject.org 29 Apr '16

29 Apr '16

commit dd58c40cb351b4d9d1b33fbef08919a8c225fad7 Merge: 1838533 a0fc5e6 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon Apr 4 17:58:05 2016 +0200 Merge branch 'master' into feature/json_reports README.rst | 41 ++++++--------------------------- docs/source/tests/psiphon.rst | 4 ++-- ooni/nettest.py | 3 +++ ooni/nettests/manipulation/http_host.py | 2 -- ooni/tasks.py | 1 - ooni/templates/httpt.py | 4 +--- 6 files changed, 13 insertions(+), 42 deletions(-) diff --cc ooni/templates/httpt.py index d2cc6e1,2f1136c..b9ca2e4 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@@ -152,20 -134,14 +152,19 @@@ class HTTPTest(NetTestCase) } } if response: - request_response['response'] = { - 'headers': list(response.headers.getAllRawHeaders()), - 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '', + if self.localOptions.get('withoutbody', 0) is 0: + response_body = _representBody(response_body) + else: + response_body = '' + session['response'] = { + 'headers': _representHeaders(response.headers), + 'body': response_body, 'code': response.code - } + } + session['failure'] = None if failure_string: - request_response['failure'] = failure_string - self.report['requests'].append(request_response) + session['failure'] = failure_string - + self.report['requests'].append(session) def _processResponseBody(self, response_body, request, response, body_processor): log.debug("Processing response body")

1 0

[ooni-probe/master] Add support for detecting the charset of the HTML response body via the meta tag
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 37564aa2876ba18516bb28e0e8eec7b489eea1c3 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 14 17:16:46 2016 +0200 Add support for detecting the charset of the HTML response body via the meta tag --- ooni/templates/httpt.py | 39 ++++++++++++++++++++++++++------------- ooni/tests/test_templates.py | 10 ++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 6ca486a..edab3fa 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,3 +1,4 @@ +import re import random from twisted.internet import defer @@ -16,6 +17,7 @@ from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni.utils.trueheaders import TrueHeaders from ooni.errors import handleAllFailures +META_CHARSET_REGEXP = re.compile('<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"\']*([^\s"\'/>]*)') class InvalidSocksProxyOption(Exception): pass @@ -35,6 +37,30 @@ class StreamListener(StreamListenerMixin): except: log.err("Tor Exit ip detection failed") + + +def _representBody(body): + # XXX perhaps add support for decoding gzip in the future. + body = body.replace('\0', '') + decoded = False + charsets = ['ascii', 'utf-8'] + + # If we are able to detect the charset of body from the meta tag + # try to decode using that one first + charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE) + if charset: + charsets.insert(0, charset.group(1)) + for encoding in charsets: + try: + body = unicode(body, encoding) + decoded = True + break + except UnicodeDecodeError: + pass + if not decoded: + body = base64Dict(body) + return body + class HTTPTest(NetTestCase): """ A utility class for dealing with HTTP based testing. It provides methods to @@ -128,19 +154,6 @@ class HTTPTest(NetTestCase): represented_headers[name] = value[0] return represented_headers - def _representBody(body): - # XXX perhaps add support for decoding gzip in the future. - try: - body = unicode(body, 'ascii') - body = body.replace('\0', '') - except UnicodeDecodeError: - try: - body = unicode(body, 'utf-8') - body = body.replace('\0', '') - except UnicodeDecodeError: - body = base64Dict(body) - return body - log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers']) session = { diff --git a/ooni/tests/test_templates.py b/ooni/tests/test_templates.py index bf05e56..931e052 100644 --- a/ooni/tests/test_templates.py +++ b/ooni/tests/test_templates.py @@ -44,6 +44,16 @@ class TestHTTPT(unittest.TestCase): yield self.assertFailure(http_test.doRequest('http://invaliddomain/'), DNSLookupError) assert http_test.report['requests'][0]['failure'] == 'dns_lookup_error' + def test_charset_detection(self): + no_charset_html = """ + <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html> +<head> + <title>Foo</title> +""" + with_charset_html = no_charset_html + '\n<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">' + self.assertEqual(httpt.META_CHARSET_REGEXP.search(no_charset_html), None) + self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_charset_html).group(1), 'iso-8859-1') class TestDNST(unittest.TestCase): def test_represent_answer_a(self):

1 0

[ooni-probe/master] [hotfix] Ensure the _setUp() method gets called only once for every input
by art＠torproject.org 29 Apr '16

29 Apr '16

commit a0fc5e6f4ef76cd935839f08cf9f3db962d95a99 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Apr 1 19:52:25 2016 +0200 [hotfix] Ensure the _setUp() method gets called only once for every input --- ooni/nettest.py | 3 +++ ooni/tasks.py | 1 - ooni/templates/httpt.py | 4 +--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 848d958..43ae23b 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -525,6 +525,7 @@ class NetTest(object): for input in test_class.inputs: measurements = [] test_instance = test_class() + test_instance._setUp() test_instance.summary = self.summary for method in test_methods: log.debug("Running %s %s" % (test_class, method)) @@ -645,6 +646,7 @@ class NetTestCase(object): def _setUp(self): """ This is the internal setup method to be overwritten by templates. + It gets called once for every input. """ self.report = {} self.inputs = None @@ -659,6 +661,7 @@ class NetTestCase(object): def setUp(self): """ Place here your logic to be executed when the test is being setup. + It gets called once every test method + input. """ pass diff --git a/ooni/tasks.py b/ooni/tasks.py index 3f331a2..bd8cbc9 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -110,7 +110,6 @@ class Measurement(TaskWithTimeout): """ self.testInstance = test_instance self.testInstance.input = test_input - self.testInstance._setUp() if not hasattr(self.testInstance, '_start_time'): self.testInstance._start_time = time.time() diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 3702e86..2f1136c 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -141,7 +141,6 @@ class HTTPTest(NetTestCase): } if failure_string: request_response['failure'] = failure_string - self.report['requests'].append(request_response) def _processResponseBody(self, response_body, request, response, body_processor): @@ -312,8 +311,7 @@ class HTTPTest(NetTestCase): log.debug("Randomizing user agent") self.randomize_useragent(request) - if 'requests' not in self.report: - self.report['requests'] = [] + self.report['requests'] = self.report.get('requests', []) # If we have a request body payload, set the request body to such # content

1 0

[ooni-probe/master] Fix bug in converting failure to string
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 250d86d5247463447f3abf0607c1c6bf0997c9a8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 14 22:42:49 2016 +0200 Fix bug in converting failure to string --- ooni/errors.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index e523de0..f98a09f 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -78,11 +78,9 @@ def failureToString(failure): for failure_type, failure_string in known_failures: if isinstance(failure.value, failure_type): - if failure_string: - return failure_string - else: - # Failure without a corresponding failure message - return 'unknown_failure %s' % str(failure.value) + return failure_string + # Failure without a corresponding failure message + return 'unknown_failure %s' % str(failure.value) class DirectorException(Exception): pass

1 0

[ooni-probe/master] OONI Control Interface
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 754ce68db5e11e0d9b6df7fbc5e7b898489af061 Author: Poly <poly(a)darkdepths.net> Date: Tue Apr 5 15:14:25 2016 +0000 OONI Control Interface control interface initial commit small updates to spec formatting added deletion to results API transition deck start to POST added netttest support and more spelling mistakes :P added more error code specs more error codes more error codes 2 completed error codes s/deck/decks/g added link to API spec fixed link --- docs/source/architecture.rst | 128 +----------- docs/source/control_interface.rst | 429 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 435 insertions(+), 122 deletions(-) diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst index d72a75c..5e7d253 100644 --- a/docs/source/architecture.rst +++ b/docs/source/architecture.rst @@ -179,128 +179,12 @@ Here we describe how an ooniprobe run should look like: report file and/or the collector will be updated. -ooniprobe API -------------- - -Note: This is currently not under active development, but we are looking for -people interested in hacking on it! - -The goals of the ooniprobe API is that of allowing applications to interact -with an ooniprobe. - -Such API will be exposed as an HTTP service that communicates with applications -thanks to JSON formatted messages. - -When ooniprobe is launched it will start as a daemon. Through the HTTP based -API it will be possible to `start tests`_, `stop tests`_ and `monitor test -progress`_. - -By having it use HTTP it will be possible to decouple the problem of running -tests from that of controlling them. -You can think of it as the Tor Control port of ooniprobe. - -The kinds of clients that will be interested in using such API are: - - * The ooniprobe *command line interface* - - * The ooniprobe *HTML5/JS web application* - - * Any *third party* tool that is interested in running ooniprobe tests with - custom arguments. - -Towards a RPC like interface ----------------------------- - -Such API will allow ooniprobes to expose such API as a Tor Hidden -Service (if the user wishes to do so). This will allow people running ooniprobe -to give the .onion address of the probe to a censorship researcher and they -will then be able to run tests from the network vantage point of the probe. - -Draft API specification ------------------------- - -Through the ooniprobe API it will be possible to `start tests`_, `stop tests`_ and `monitor test -progress`_. - -List tests -.......... - -`GET /test` - -Shall return the list of available tests as an array. - -This is how a response looks like -:: - - [{'id': 'http_requests', - 'name': 'HTTP Requests Test', - 'description': 'This test perform a HTTP GET request for the / resource over the test network and over Tor', - 'type': [ 'blocking' ], - 'version': '0.1', - 'arguments': { - 'urllist': 'Specify the list of URLs to be used for the test' - } - }] - -*type* may be either **blocking** or **manipulation**. - -Start tests -........... - - -`POST /test/<test_id>/start` - -Is used to start a test with the specified test_id. - -Inside of the request you will specify the arguments supported by the test - -This is how a request could look like -:: - { - 'urllist': - ['http://google.com/', 'http://torproject.org/'] - } - -The server will then respond with the test object -:: - { - 'status': 'running', - 'percentage': 0, - 'current_input': 'http://google.com/', - 'urllist': - ['http://google.com/', 'http://torproject.org/'] - } - - -Stop tests -........... - -`POST /test/<test_id>/stop` - -This will terminate the execution of the test with the specified test_id. - -The request may optionally contain a reason for stopping the test such as -:: - { - 'reason': 'some reason' - } - -Monitor test progress -..................... - -`GET /test/<test_id>` - -Will return the status of a test - -Like so for example -:: - { - 'status': 'running', - 'percentage': 0, - 'current_input': 'http://google.com/', - 'urllist': - ['http://google.com/', 'http://torproject.org/'] - } +OONIprobe Control Interface +--------------------------- +.. XXX update this section once interface is implemented. +The ooniprobe client provides a rich and simple JSON-based interface for +control over HTTP. While the implementation of this interface is currently +a work in progress, the specification may be found `here <control_interface.rst>`_. Implementation status diff --git a/docs/source/control_interface.rst b/docs/source/control_interface.rst new file mode 100644 index 0000000..0104e4d --- /dev/null +++ b/docs/source/control_interface.rst @@ -0,0 +1,429 @@ +==================================== +HTTP Control Interface Specification +==================================== + +The ``ooniprobe`` client provides a HTTP-based control interface. The goal of +this interface is to allow applications to interact with ooniprobe in a +standardized manner. The control interface aims to be a RESTful, stateless +and simple protocol. + +This control interface will be exposed as an HTTP service that communicates +using JSON encoded messages. The service shall be provided via the ooniprobe +daemon, ``oonid``. + +While the HTTP Control Interface is **currently under development** with +a HTML/JS WebGUI client in mind, it will also be compatible with other +clients, such as Ledilopter (ooniprobe on the Raspberry Pi) and 3rd party +tools. + + +.. contents:: **Table of Contents** + :depth: 2 + +General Notes +............. +This document is the working specification for ooniprobe HTTP Control +Interface version 0. + +All the fields defined in this specification are mandatory, unless explicitly +marked otherwise. + +All successful client requests should return a 2xx status code, and should +return return a JSON encoded body if required in the specification below. +Failed (or deferred) requests will cause a 3xx, 4xx or 5xx status code on +response. All failure responses must return a JSON encoded body with the +following format:: + + { + 'error_code': <some_integer_error_code>, + 'error_message': <some_string_explanation> + } + +In particular, should the client make a request to a invalid URI, the service +shall respond with ``Status-code: 404`` and the reponse body:: + + { + 'error_code': 404, + 'error_message': 'Resource Not Found' + } + +If the client makes a request with invalid syntax that prevents the service +from understanding the request, the service shall return ``Status-code: 400`` +and the response body:: + + { + 'error_code': 400, + 'error_message': 'Invalid syntax' + } + +Error handling is the client's responsibility. + +All date and time records in this specification shall be encoded according to +`ISO 8601 <https://en.wikipedia.org/wiki/ISO_8601>`_. + +Decks +..... +Testing in OONI is centered around 'decks'. A deck is defined as a collection +of network tests and their associated inputs. + +List decks +^^^^^^^^^^ +To retrieve an array of all available decks: + +``GET /decks`` + +The server shall repond with ``Status-code: 200`` and a body format:: + + [{'id': 'deck-it', + 'name': 'Deck for Italy', + 'description': 'blah blah blah', + 'nettests': [ 'http_invalid_request_line', + 'http_header_field_manipulation', + 'dns_consistency'], + }] + +Where ``nettests`` must refer to valid test IDs. The service may return an +empty list if no tests are found. + +If the service is unable to retrieve the list of decks, it shall return a +reponse with ``Status-code: 500`` and a body format:: + + { + 'error_code': 500, + 'error_message': 'Internal Server Error - Could not find directory "decks"' + } + +Generate decks +^^^^^^^^^^^^^^ +Decks can either be generated for a specific country, or you can let +oonideckgen try to automatically detect the country. + +``POST /decks`` + +Request format:: + + { + 'country': 'CN' // optional, oonideckgen will autodetect otherwise + } + +On success, the service shall respond with ``status code 200`` and with the +following body format:: + + { + 'deck_id': 'deck-cn' + } + +Deck IDs are not guaranteed to be unique - if the exact same deck has been +created through an earlier request, the service shall reply with the old +deck ID. + +If the operation fails the service shall respond with the appropriate status +code and message. + +In particular, if the client makes invalid deck generation request, the +service shall reply with ``Status-Code: 400`` and body message formatted:: + + { + 'error_code': 400, + 'error_message': 'Bad Request - "xy" is not a valid ISO country code' + } + +If the server is unable to generate a deck due to an internal error, it shall +respond with ``Status-Code: 500`` and a body message formatted:: + + { + 'error_code': 500, + 'error_message': 'Internal Error - oonideckgen: couldn't fetch "http://someurl.com"' + } + +Start deck +^^^^^^^^^^ +``POST /decks/<deck_id>/start`` + +To run a deck, the above POST request is sent where ``deck_id`` must be a +valid deck ID. + +Request format:: + + { + 'collector': true, // optional, defaults to true + 'bouncer': 'http://someaddress.onion' // optional, defaults to httpo://XXX + } + +On success, the server shall respond with ``Status-Code: 200`` and with the +following body format:: + + { + 'current_nettest': 'dns_consistency', + 'time_started': '2014-03-12T13:37:27+00:00' + } + +If the service is unable to start the test, it shall respond with the +appropriate status code and message. + +In particular, if the client attempts to run multiple decks simoultensouly, +the service shall respond with ``Status-Code: 503`` and the body:: + + { + 'error_code': 503, + 'error_message': 'Unable to handle request - another deck is already running' + } + +If the service is unable to start the deck due to an interal error (for example, corrupted input files) it shall respond with ``Status-Code: 500`` and +the body:: + + { + 'error_code': 500, + 'error_message': 'Unable to handle request - oonid: unable to find input file "DNE.txt"' + } + +Stop deck +^^^^^^^^^ +``GET /decks/<deck_id>/stop`` + +To stop a deck, the above GET request is sent where ``deck_id`` must be a +valid deck ID. + +On success, the service shall respond with ``Status-Code: 204 - No Content``. + +If the requested deck is not running, the service shall repond with ``Status-Code: 400`` +and the body formatted:: + + { + 'error_code': 400, + 'error_message': 'Invalid Request - Deck is not running' + } + +Should the server be unable to stop the test, it shall repond with +``Status-Code: 500`` and display a reason in the body in the following format:: + + { + 'error_code': 500, + 'error_message': 'Unable to handle request - out of RAM' + } + +Deck progress +^^^^^^^^^^^^^ +``GET /decks/<deck_id>`` + +Returns the deck progress if the deck is running, or the deck results in +JSON format if the deck is complete. + +If the deck progress is successfully found, the service shall respond with +``Status-code: 200`` and a response body formatted as follows:: + + { + 'status': 'running', + 'percentage': 32, + 'current_nettest': 'http_headers', + 'results': null + } + +Another possible response body:: + + { + 'status': 'complete', + 'percentage': 100, + 'current_nettest': null, + 'results': '<result_id>' + } + +The ``status`` field may be one of: ``stopped``, ``running`` or ``complete``. +``results`` must be a valid result ID or ``null`` if the test is not yet +finished. + +Otherwise, if the deck exists but the progress request fails, the service +shall respond with ``Status-code: 500`` and an explanation in the following +format:: + + { + 'error_code': 500, + 'error_message': 'Internal Server Error - could not find deck result ID' + } + +Net Tests +......... +In OONI, a ``nettest`` represents an individual anomaly detection technique. +To run, nettests also require an input file which specifies on which URIs the +test is to be performed. + +Usually, several nettests are bundled along with their inputs in a deck, which +makes it easier for the end user to run. Nonetheless, the control interface +allows for nettests to be run individually. + +List tests +^^^^^^^^^^ +To retrieve an array of all available nettests: + +``GET /tests`` + +On success, the service shall repond with ``Status-code: 200`` and a body formatted:: + + [ + {'id': 'dns-consistency', + 'name': 'DNS Consistency', + 'description': 'Compares the results of two DNS lookups', + 'type': 'blocking', + 'version': '0.1', + 'arguments': { + 'urllist': 'Specify the list of URLs to be used for the test' + }, ... + }, ... + ] + +Where ``nettests`` must refer to valid test IDs. The field ``type`` may be +of value ``blocking`` or ``manipulation``. If no decks are found, the service +will still respond with ``Status-code: 200`` and an empty list. + +If the service is unable to retrieve the list of decks, it shall return a +reponse with ``Status-code: 500`` and a body format:: + + { + 'error_code': 500, + 'error_message': 'Internal Server Error - Could not find directory "decks"' + } + +Starting a Test +^^^^^^^^^^^^^^^ +To run a given test, the client must send the following request: +``POST /tests/<test_id>/start`` + +With the request body:: + + { + 'urllist': + ['http://google.com/', 'http://torproject.org'] + } + +The server shall respond with ``status code 200`` with the body:: + + { + 'time_started': '2014-03-12T13:37:27+00:00', + 'percentage': 55, + 'current_input': 'http://google.com', + 'arguments': [<list of supplied arguments>] + } + +Else, the reponse shall be a error status code and an explanation. In +particular, the service shall respond with ``Status-Code: 400`` if the user +provides an invalid argument or ``Status-Code: 500`` if the server is unable +to start the test due to an internal reason. + +Stopping a Test +^^^^^^^^^^^^^^^ +To terminate a given test, the client sends the following request: +``GET /tests/<test_id>/stop`` + +The server shall respond with status code 204 - no content - or with an error +message if it is unable to stop the test. + +If the requested nettest is not running, the service shall repond with ``Status-Code: 400`` +and the body formatted:: + + { + 'error_code': 400, + 'error_message': 'Invalid Request - nettest is not running' + } + +Should the server be unable to stop the test, it shall repond with +``Status-Code: 503`` and display a reason in the body in the following format:: + + { + 'error_code': 503, + 'error_message': 'Unable to handle request - out of RAM' + } + +Test progress +^^^^^^^^^^^^^ +``GET /tests/<test_id>`` + +The service will respond with ``Status-Code 200`` and a body formatted:: + + { + 'status': 'running', + 'time_started': '2014-03-12T13:37:27+00:00', + 'percentage': 55, + 'current_input': 'http://google.com', + 'arguments': [<list of supplied arguments>], + 'results': <result_id> + } + +The ``status`` field may be one of: ``stopped``, ``running`` or ``complete``. +``results`` must be a valid result ID or ``null`` if the test is not yet +finished. + +Should the service fail to determine the progress of a nettest, it shall return +``Status-code: 503`` along with a suitable error message. + +Results +....... +``GET /results`` + +Returns a list of all stored results of previous runs. + +The service will respond with ``Status-code: 200`` and results formatted:: + + [ + {'id': '<result_id>', + 'type': 'deck', + 'deck': '<deck_id>', + 'time_started': '2014-03-12T13:37:27+00:00', + 'time_finished': '2014-03-12T13:37:27+00:00', + 'collector': true, + 'collector-address': 'httpo://nkvphnp3p6agi5qq.onion', + 'nettests': ['http_headers', 'http_requests', ...] + }, ... + ] + +Where ``type`` shall be either ``deck`` or ``nettest``. If the result is of +type ``deck``, then the fields ``deck_id`` and ``nettests`` are mandatory. +Likewise, if the result is of type ``nettest`` the field ``test_id`` shall +be mandatory. + +To get the results of the individual nettests (that are part of a deck) +in JSON format: + +``GET /results/<result_id>/nettest/<nettest_id>`` + +Please be warned that this will return the raw output of the test, which may be +in excess of several tens of megabytes. + +The server should return ``Status-Code 500`` and an explanation should it +fail to collect the results. + +Deleting Results +^^^^^^^^^^^^^^^^ + +To delete a particular result: + +``GET /results/<result_id>/delete`` + +After deletion, the service shall reply with ``Status-code: 204`` - no content. +The server shall reply with ``Status-code: 503`` should deletion fail. + +Resources +......... +To update ooniprobe's geoIP databases or input files, the client may send +the following request: + +``POST /resources/update`` + +With the body formatted as:: + + { + 'update_geoIP': true, + 'update_inputs': false + } + +The server shall reply with the ``204`` status code or if the update fails, +``503`` status code. + +TODO +.... +Open questions include: + +* Authentication. Users probably don't want anyone who can access port 80 + on their machines to control ooniprobe. + +* Protocol signaling. Ideally we want some way to make clients aware of + different protocol versions.

1 0

[ooni-probe/master] [hotfix] fix _representBody with commit lost in rebase
by art＠torproject.org 29 Apr '16

29 Apr '16

commit d916e0144553cc2f655c358bcf6a2fb270ae7e0e Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Apr 11 17:10:13 2016 +0200 [hotfix] fix _representBody with commit lost in rebase --- ooni/templates/httpt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index b9ca2e4..6ca486a 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -139,6 +139,7 @@ class HTTPTest(NetTestCase): body = body.replace('\0', '') except UnicodeDecodeError: body = base64Dict(body) + return body log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers'])

1 0

[ooni-probe/master] [bugfix] Critical bug that sometimes lead to incorrect body content
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 0bfb58cd9d2647e6defa3bcdf0dbd7a9e2e16189 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 14 22:44:47 2016 +0200 [bugfix] Critical bug that sometimes lead to incorrect body content --- ooni/utils/trueheaders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index b7671b5..d5aa206 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -103,7 +103,7 @@ class HTTPClientParser(_newclient.HTTPClientParser): self._partialHeader = None def headerReceived(self, name, value): - if self.isConnectionControlHeader(name): + if self.isConnectionControlHeader(name.lower()): headers = self.connHeaders else: headers = self.headers

1 0

← Newer
1
2
3
4
5
6
7
8
...
165
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits April 2016