- tor-commits - lists.torproject.org

[ooni-probe/master] Add support for detecting the charset of the HTML response body via the meta tag
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 37564aa2876ba18516bb28e0e8eec7b489eea1c3 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 14 17:16:46 2016 +0200 Add support for detecting the charset of the HTML response body via the meta tag --- ooni/templates/httpt.py | 39 ++++++++++++++++++++++++++------------- ooni/tests/test_templates.py | 10 ++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 6ca486a..edab3fa 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,3 +1,4 @@ +import re import random from twisted.internet import defer @@ -16,6 +17,7 @@ from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni.utils.trueheaders import TrueHeaders from ooni.errors import handleAllFailures +META_CHARSET_REGEXP = re.compile('<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"\']*([^\s"\'/>]*)') class InvalidSocksProxyOption(Exception): pass @@ -35,6 +37,30 @@ class StreamListener(StreamListenerMixin): except: log.err("Tor Exit ip detection failed") + + +def _representBody(body): + # XXX perhaps add support for decoding gzip in the future. + body = body.replace('\0', '') + decoded = False + charsets = ['ascii', 'utf-8'] + + # If we are able to detect the charset of body from the meta tag + # try to decode using that one first + charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE) + if charset: + charsets.insert(0, charset.group(1)) + for encoding in charsets: + try: + body = unicode(body, encoding) + decoded = True + break + except UnicodeDecodeError: + pass + if not decoded: + body = base64Dict(body) + return body + class HTTPTest(NetTestCase): """ A utility class for dealing with HTTP based testing. It provides methods to @@ -128,19 +154,6 @@ class HTTPTest(NetTestCase): represented_headers[name] = value[0] return represented_headers - def _representBody(body): - # XXX perhaps add support for decoding gzip in the future. - try: - body = unicode(body, 'ascii') - body = body.replace('\0', '') - except UnicodeDecodeError: - try: - body = unicode(body, 'utf-8') - body = body.replace('\0', '') - except UnicodeDecodeError: - body = base64Dict(body) - return body - log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers']) session = { diff --git a/ooni/tests/test_templates.py b/ooni/tests/test_templates.py index bf05e56..931e052 100644 --- a/ooni/tests/test_templates.py +++ b/ooni/tests/test_templates.py @@ -44,6 +44,16 @@ class TestHTTPT(unittest.TestCase): yield self.assertFailure(http_test.doRequest('http://invaliddomain/'), DNSLookupError) assert http_test.report['requests'][0]['failure'] == 'dns_lookup_error' + def test_charset_detection(self): + no_charset_html = """ + <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html> +<head> + <title>Foo</title> +""" + with_charset_html = no_charset_html + '\n<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">' + self.assertEqual(httpt.META_CHARSET_REGEXP.search(no_charset_html), None) + self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_charset_html).group(1), 'iso-8859-1') class TestDNST(unittest.TestCase): def test_represent_answer_a(self):

1 0

[ooni-probe/master] [hotfix] Ensure the _setUp() method gets called only once for every input
by art＠torproject.org 29 Apr '16

29 Apr '16

commit a0fc5e6f4ef76cd935839f08cf9f3db962d95a99 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Apr 1 19:52:25 2016 +0200 [hotfix] Ensure the _setUp() method gets called only once for every input --- ooni/nettest.py | 3 +++ ooni/tasks.py | 1 - ooni/templates/httpt.py | 4 +--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 848d958..43ae23b 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -525,6 +525,7 @@ class NetTest(object): for input in test_class.inputs: measurements = [] test_instance = test_class() + test_instance._setUp() test_instance.summary = self.summary for method in test_methods: log.debug("Running %s %s" % (test_class, method)) @@ -645,6 +646,7 @@ class NetTestCase(object): def _setUp(self): """ This is the internal setup method to be overwritten by templates. + It gets called once for every input. """ self.report = {} self.inputs = None @@ -659,6 +661,7 @@ class NetTestCase(object): def setUp(self): """ Place here your logic to be executed when the test is being setup. + It gets called once every test method + input. """ pass diff --git a/ooni/tasks.py b/ooni/tasks.py index 3f331a2..bd8cbc9 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -110,7 +110,6 @@ class Measurement(TaskWithTimeout): """ self.testInstance = test_instance self.testInstance.input = test_input - self.testInstance._setUp() if not hasattr(self.testInstance, '_start_time'): self.testInstance._start_time = time.time() diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 3702e86..2f1136c 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -141,7 +141,6 @@ class HTTPTest(NetTestCase): } if failure_string: request_response['failure'] = failure_string - self.report['requests'].append(request_response) def _processResponseBody(self, response_body, request, response, body_processor): @@ -312,8 +311,7 @@ class HTTPTest(NetTestCase): log.debug("Randomizing user agent") self.randomize_useragent(request) - if 'requests' not in self.report: - self.report['requests'] = [] + self.report['requests'] = self.report.get('requests', []) # If we have a request body payload, set the request body to such # content

1 0

[ooni-probe/master] Fix bug in converting failure to string
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 250d86d5247463447f3abf0607c1c6bf0997c9a8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 14 22:42:49 2016 +0200 Fix bug in converting failure to string --- ooni/errors.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index e523de0..f98a09f 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -78,11 +78,9 @@ def failureToString(failure): for failure_type, failure_string in known_failures: if isinstance(failure.value, failure_type): - if failure_string: - return failure_string - else: - # Failure without a corresponding failure message - return 'unknown_failure %s' % str(failure.value) + return failure_string + # Failure without a corresponding failure message + return 'unknown_failure %s' % str(failure.value) class DirectorException(Exception): pass

1 0

[ooni-probe/master] OONI Control Interface
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 754ce68db5e11e0d9b6df7fbc5e7b898489af061 Author: Poly <poly(a)darkdepths.net> Date: Tue Apr 5 15:14:25 2016 +0000 OONI Control Interface control interface initial commit small updates to spec formatting added deletion to results API transition deck start to POST added netttest support and more spelling mistakes :P added more error code specs more error codes more error codes 2 completed error codes s/deck/decks/g added link to API spec fixed link --- docs/source/architecture.rst | 128 +----------- docs/source/control_interface.rst | 429 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 435 insertions(+), 122 deletions(-) diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst index d72a75c..5e7d253 100644 --- a/docs/source/architecture.rst +++ b/docs/source/architecture.rst @@ -179,128 +179,12 @@ Here we describe how an ooniprobe run should look like: report file and/or the collector will be updated. -ooniprobe API -------------- - -Note: This is currently not under active development, but we are looking for -people interested in hacking on it! - -The goals of the ooniprobe API is that of allowing applications to interact -with an ooniprobe. - -Such API will be exposed as an HTTP service that communicates with applications -thanks to JSON formatted messages. - -When ooniprobe is launched it will start as a daemon. Through the HTTP based -API it will be possible to `start tests`_, `stop tests`_ and `monitor test -progress`_. - -By having it use HTTP it will be possible to decouple the problem of running -tests from that of controlling them. -You can think of it as the Tor Control port of ooniprobe. - -The kinds of clients that will be interested in using such API are: - - * The ooniprobe *command line interface* - - * The ooniprobe *HTML5/JS web application* - - * Any *third party* tool that is interested in running ooniprobe tests with - custom arguments. - -Towards a RPC like interface ----------------------------- - -Such API will allow ooniprobes to expose such API as a Tor Hidden -Service (if the user wishes to do so). This will allow people running ooniprobe -to give the .onion address of the probe to a censorship researcher and they -will then be able to run tests from the network vantage point of the probe. - -Draft API specification ------------------------- - -Through the ooniprobe API it will be possible to `start tests`_, `stop tests`_ and `monitor test -progress`_. - -List tests -.......... - -`GET /test` - -Shall return the list of available tests as an array. - -This is how a response looks like -:: - - [{'id': 'http_requests', - 'name': 'HTTP Requests Test', - 'description': 'This test perform a HTTP GET request for the / resource over the test network and over Tor', - 'type': [ 'blocking' ], - 'version': '0.1', - 'arguments': { - 'urllist': 'Specify the list of URLs to be used for the test' - } - }] - -*type* may be either **blocking** or **manipulation**. - -Start tests -........... - - -`POST /test/<test_id>/start` - -Is used to start a test with the specified test_id. - -Inside of the request you will specify the arguments supported by the test - -This is how a request could look like -:: - { - 'urllist': - ['http://google.com/', 'http://torproject.org/'] - } - -The server will then respond with the test object -:: - { - 'status': 'running', - 'percentage': 0, - 'current_input': 'http://google.com/', - 'urllist': - ['http://google.com/', 'http://torproject.org/'] - } - - -Stop tests -........... - -`POST /test/<test_id>/stop` - -This will terminate the execution of the test with the specified test_id. - -The request may optionally contain a reason for stopping the test such as -:: - { - 'reason': 'some reason' - } - -Monitor test progress -..................... - -`GET /test/<test_id>` - -Will return the status of a test - -Like so for example -:: - { - 'status': 'running', - 'percentage': 0, - 'current_input': 'http://google.com/', - 'urllist': - ['http://google.com/', 'http://torproject.org/'] - } +OONIprobe Control Interface +--------------------------- +.. XXX update this section once interface is implemented. +The ooniprobe client provides a rich and simple JSON-based interface for +control over HTTP. While the implementation of this interface is currently +a work in progress, the specification may be found `here <control_interface.rst>`_. Implementation status diff --git a/docs/source/control_interface.rst b/docs/source/control_interface.rst new file mode 100644 index 0000000..0104e4d --- /dev/null +++ b/docs/source/control_interface.rst @@ -0,0 +1,429 @@ +==================================== +HTTP Control Interface Specification +==================================== + +The ``ooniprobe`` client provides a HTTP-based control interface. The goal of +this interface is to allow applications to interact with ooniprobe in a +standardized manner. The control interface aims to be a RESTful, stateless +and simple protocol. + +This control interface will be exposed as an HTTP service that communicates +using JSON encoded messages. The service shall be provided via the ooniprobe +daemon, ``oonid``. + +While the HTTP Control Interface is **currently under development** with +a HTML/JS WebGUI client in mind, it will also be compatible with other +clients, such as Ledilopter (ooniprobe on the Raspberry Pi) and 3rd party +tools. + + +.. contents:: **Table of Contents** + :depth: 2 + +General Notes +............. +This document is the working specification for ooniprobe HTTP Control +Interface version 0. + +All the fields defined in this specification are mandatory, unless explicitly +marked otherwise. + +All successful client requests should return a 2xx status code, and should +return return a JSON encoded body if required in the specification below. +Failed (or deferred) requests will cause a 3xx, 4xx or 5xx status code on +response. All failure responses must return a JSON encoded body with the +following format:: + + { + 'error_code': <some_integer_error_code>, + 'error_message': <some_string_explanation> + } + +In particular, should the client make a request to a invalid URI, the service +shall respond with ``Status-code: 404`` and the reponse body:: + + { + 'error_code': 404, + 'error_message': 'Resource Not Found' + } + +If the client makes a request with invalid syntax that prevents the service +from understanding the request, the service shall return ``Status-code: 400`` +and the response body:: + + { + 'error_code': 400, + 'error_message': 'Invalid syntax' + } + +Error handling is the client's responsibility. + +All date and time records in this specification shall be encoded according to +`ISO 8601 <https://en.wikipedia.org/wiki/ISO_8601>`_. + +Decks +..... +Testing in OONI is centered around 'decks'. A deck is defined as a collection +of network tests and their associated inputs. + +List decks +^^^^^^^^^^ +To retrieve an array of all available decks: + +``GET /decks`` + +The server shall repond with ``Status-code: 200`` and a body format:: + + [{'id': 'deck-it', + 'name': 'Deck for Italy', + 'description': 'blah blah blah', + 'nettests': [ 'http_invalid_request_line', + 'http_header_field_manipulation', + 'dns_consistency'], + }] + +Where ``nettests`` must refer to valid test IDs. The service may return an +empty list if no tests are found. + +If the service is unable to retrieve the list of decks, it shall return a +reponse with ``Status-code: 500`` and a body format:: + + { + 'error_code': 500, + 'error_message': 'Internal Server Error - Could not find directory "decks"' + } + +Generate decks +^^^^^^^^^^^^^^ +Decks can either be generated for a specific country, or you can let +oonideckgen try to automatically detect the country. + +``POST /decks`` + +Request format:: + + { + 'country': 'CN' // optional, oonideckgen will autodetect otherwise + } + +On success, the service shall respond with ``status code 200`` and with the +following body format:: + + { + 'deck_id': 'deck-cn' + } + +Deck IDs are not guaranteed to be unique - if the exact same deck has been +created through an earlier request, the service shall reply with the old +deck ID. + +If the operation fails the service shall respond with the appropriate status +code and message. + +In particular, if the client makes invalid deck generation request, the +service shall reply with ``Status-Code: 400`` and body message formatted:: + + { + 'error_code': 400, + 'error_message': 'Bad Request - "xy" is not a valid ISO country code' + } + +If the server is unable to generate a deck due to an internal error, it shall +respond with ``Status-Code: 500`` and a body message formatted:: + + { + 'error_code': 500, + 'error_message': 'Internal Error - oonideckgen: couldn't fetch "http://someurl.com"' + } + +Start deck +^^^^^^^^^^ +``POST /decks/<deck_id>/start`` + +To run a deck, the above POST request is sent where ``deck_id`` must be a +valid deck ID. + +Request format:: + + { + 'collector': true, // optional, defaults to true + 'bouncer': 'http://someaddress.onion' // optional, defaults to httpo://XXX + } + +On success, the server shall respond with ``Status-Code: 200`` and with the +following body format:: + + { + 'current_nettest': 'dns_consistency', + 'time_started': '2014-03-12T13:37:27+00:00' + } + +If the service is unable to start the test, it shall respond with the +appropriate status code and message. + +In particular, if the client attempts to run multiple decks simoultensouly, +the service shall respond with ``Status-Code: 503`` and the body:: + + { + 'error_code': 503, + 'error_message': 'Unable to handle request - another deck is already running' + } + +If the service is unable to start the deck due to an interal error (for example, corrupted input files) it shall respond with ``Status-Code: 500`` and +the body:: + + { + 'error_code': 500, + 'error_message': 'Unable to handle request - oonid: unable to find input file "DNE.txt"' + } + +Stop deck +^^^^^^^^^ +``GET /decks/<deck_id>/stop`` + +To stop a deck, the above GET request is sent where ``deck_id`` must be a +valid deck ID. + +On success, the service shall respond with ``Status-Code: 204 - No Content``. + +If the requested deck is not running, the service shall repond with ``Status-Code: 400`` +and the body formatted:: + + { + 'error_code': 400, + 'error_message': 'Invalid Request - Deck is not running' + } + +Should the server be unable to stop the test, it shall repond with +``Status-Code: 500`` and display a reason in the body in the following format:: + + { + 'error_code': 500, + 'error_message': 'Unable to handle request - out of RAM' + } + +Deck progress +^^^^^^^^^^^^^ +``GET /decks/<deck_id>`` + +Returns the deck progress if the deck is running, or the deck results in +JSON format if the deck is complete. + +If the deck progress is successfully found, the service shall respond with +``Status-code: 200`` and a response body formatted as follows:: + + { + 'status': 'running', + 'percentage': 32, + 'current_nettest': 'http_headers', + 'results': null + } + +Another possible response body:: + + { + 'status': 'complete', + 'percentage': 100, + 'current_nettest': null, + 'results': '<result_id>' + } + +The ``status`` field may be one of: ``stopped``, ``running`` or ``complete``. +``results`` must be a valid result ID or ``null`` if the test is not yet +finished. + +Otherwise, if the deck exists but the progress request fails, the service +shall respond with ``Status-code: 500`` and an explanation in the following +format:: + + { + 'error_code': 500, + 'error_message': 'Internal Server Error - could not find deck result ID' + } + +Net Tests +......... +In OONI, a ``nettest`` represents an individual anomaly detection technique. +To run, nettests also require an input file which specifies on which URIs the +test is to be performed. + +Usually, several nettests are bundled along with their inputs in a deck, which +makes it easier for the end user to run. Nonetheless, the control interface +allows for nettests to be run individually. + +List tests +^^^^^^^^^^ +To retrieve an array of all available nettests: + +``GET /tests`` + +On success, the service shall repond with ``Status-code: 200`` and a body formatted:: + + [ + {'id': 'dns-consistency', + 'name': 'DNS Consistency', + 'description': 'Compares the results of two DNS lookups', + 'type': 'blocking', + 'version': '0.1', + 'arguments': { + 'urllist': 'Specify the list of URLs to be used for the test' + }, ... + }, ... + ] + +Where ``nettests`` must refer to valid test IDs. The field ``type`` may be +of value ``blocking`` or ``manipulation``. If no decks are found, the service +will still respond with ``Status-code: 200`` and an empty list. + +If the service is unable to retrieve the list of decks, it shall return a +reponse with ``Status-code: 500`` and a body format:: + + { + 'error_code': 500, + 'error_message': 'Internal Server Error - Could not find directory "decks"' + } + +Starting a Test +^^^^^^^^^^^^^^^ +To run a given test, the client must send the following request: +``POST /tests/<test_id>/start`` + +With the request body:: + + { + 'urllist': + ['http://google.com/', 'http://torproject.org'] + } + +The server shall respond with ``status code 200`` with the body:: + + { + 'time_started': '2014-03-12T13:37:27+00:00', + 'percentage': 55, + 'current_input': 'http://google.com', + 'arguments': [<list of supplied arguments>] + } + +Else, the reponse shall be a error status code and an explanation. In +particular, the service shall respond with ``Status-Code: 400`` if the user +provides an invalid argument or ``Status-Code: 500`` if the server is unable +to start the test due to an internal reason. + +Stopping a Test +^^^^^^^^^^^^^^^ +To terminate a given test, the client sends the following request: +``GET /tests/<test_id>/stop`` + +The server shall respond with status code 204 - no content - or with an error +message if it is unable to stop the test. + +If the requested nettest is not running, the service shall repond with ``Status-Code: 400`` +and the body formatted:: + + { + 'error_code': 400, + 'error_message': 'Invalid Request - nettest is not running' + } + +Should the server be unable to stop the test, it shall repond with +``Status-Code: 503`` and display a reason in the body in the following format:: + + { + 'error_code': 503, + 'error_message': 'Unable to handle request - out of RAM' + } + +Test progress +^^^^^^^^^^^^^ +``GET /tests/<test_id>`` + +The service will respond with ``Status-Code 200`` and a body formatted:: + + { + 'status': 'running', + 'time_started': '2014-03-12T13:37:27+00:00', + 'percentage': 55, + 'current_input': 'http://google.com', + 'arguments': [<list of supplied arguments>], + 'results': <result_id> + } + +The ``status`` field may be one of: ``stopped``, ``running`` or ``complete``. +``results`` must be a valid result ID or ``null`` if the test is not yet +finished. + +Should the service fail to determine the progress of a nettest, it shall return +``Status-code: 503`` along with a suitable error message. + +Results +....... +``GET /results`` + +Returns a list of all stored results of previous runs. + +The service will respond with ``Status-code: 200`` and results formatted:: + + [ + {'id': '<result_id>', + 'type': 'deck', + 'deck': '<deck_id>', + 'time_started': '2014-03-12T13:37:27+00:00', + 'time_finished': '2014-03-12T13:37:27+00:00', + 'collector': true, + 'collector-address': 'httpo://nkvphnp3p6agi5qq.onion', + 'nettests': ['http_headers', 'http_requests', ...] + }, ... + ] + +Where ``type`` shall be either ``deck`` or ``nettest``. If the result is of +type ``deck``, then the fields ``deck_id`` and ``nettests`` are mandatory. +Likewise, if the result is of type ``nettest`` the field ``test_id`` shall +be mandatory. + +To get the results of the individual nettests (that are part of a deck) +in JSON format: + +``GET /results/<result_id>/nettest/<nettest_id>`` + +Please be warned that this will return the raw output of the test, which may be +in excess of several tens of megabytes. + +The server should return ``Status-Code 500`` and an explanation should it +fail to collect the results. + +Deleting Results +^^^^^^^^^^^^^^^^ + +To delete a particular result: + +``GET /results/<result_id>/delete`` + +After deletion, the service shall reply with ``Status-code: 204`` - no content. +The server shall reply with ``Status-code: 503`` should deletion fail. + +Resources +......... +To update ooniprobe's geoIP databases or input files, the client may send +the following request: + +``POST /resources/update`` + +With the body formatted as:: + + { + 'update_geoIP': true, + 'update_inputs': false + } + +The server shall reply with the ``204`` status code or if the update fails, +``503`` status code. + +TODO +.... +Open questions include: + +* Authentication. Users probably don't want anyone who can access port 80 + on their machines to control ooniprobe. + +* Protocol signaling. Ideally we want some way to make clients aware of + different protocol versions.

1 0

[ooni-probe/master] [hotfix] fix _representBody with commit lost in rebase
by art＠torproject.org 29 Apr '16

29 Apr '16

commit d916e0144553cc2f655c358bcf6a2fb270ae7e0e Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Apr 11 17:10:13 2016 +0200 [hotfix] fix _representBody with commit lost in rebase --- ooni/templates/httpt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index b9ca2e4..6ca486a 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -139,6 +139,7 @@ class HTTPTest(NetTestCase): body = body.replace('\0', '') except UnicodeDecodeError: body = base64Dict(body) + return body log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers'])

1 0

[ooni-probe/master] [bugfix] Critical bug that sometimes lead to incorrect body content
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 0bfb58cd9d2647e6defa3bcdf0dbd7a9e2e16189 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 14 22:44:47 2016 +0200 [bugfix] Critical bug that sometimes lead to incorrect body content --- ooni/utils/trueheaders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index b7671b5..d5aa206 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -103,7 +103,7 @@ class HTTPClientParser(_newclient.HTTPClientParser): self._partialHeader = None def headerReceived(self, name, value): - if self.isConnectionControlHeader(name): + if self.isConnectionControlHeader(name.lower()): headers = self.connHeaders else: headers = self.headers

1 0

[ooni-probe/master] Explicitly test twisted version 15.5.0
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 52bb445c881d111b19c3a8f365b435dc8bb889b8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Feb 5 20:54:34 2016 +0100 Explicitly test twisted version 15.5.0 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index acda409..e3033f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ env: - TWISTED=Twisted==15.2.1 - TWISTED=Twisted==15.3.0 - TWISTED=Twisted==15.4.0 + - TWISTED=Twisted==15.5.0 - TWISTED=Twisted before_install: - sudo apt-get update

1 0

[ooni-probe/master] Update changelog for version 1.4.0
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 2f8a8a3cfcbab9c019f613d2588b281549788f39 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Apr 22 12:52:08 2016 +0200 Update changelog for version 1.4.0 --- ChangeLog.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 79cb883..47851e1 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,22 @@ Changelog ========= +v1.4.0 (XXX, XX Apr 2016) +------------------------- + +* Support for reporting using JSON + +* Support for running ooniprobe with a message queue providing URLs to test + +* Psiphon censorship circumvention test + +* OpenVPN censorship circumvention test + +* Support for disabling reporting to disk + +* Improvements to HTTP response body decoding (includes fix that lead to empty + bodies being misrepresented) + v1.3.2 (Fri, 20 Nov 2015) -------------------------

1 0

[ooni-probe/master] Merge pull request #456 from TheTorProject/feature/json_reports
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 5252b00b29f8c9570ecb9533f4577c0a5b69f06f Merge: a0fc5e6 dd58c40 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon Apr 4 18:41:52 2016 +0200 Merge pull request #456 from TheTorProject/feature/json_reports Feature/json reports .travis.yml | 4 - ooni/deckgen/processors/citizenlab_test_lists.py | 5 + ooni/director.py | 11 +- ooni/errors.py | 3 + ooni/nettest.py | 17 +-- ooni/nettests/blocking/dns_consistency.py | 29 +++-- ooni/nettests/blocking/http_requests.py | 11 +- ooni/nettests/experimental/daphne.py | 121 ++++++++++++++++++ ooni/nettests/manipulation/captiveportal.py | 2 +- ooni/nettests/manipulation/daphne.py | 121 ------------------ ooni/nettests/manipulation/http_host.py | 4 +- .../manipulation/http_invalid_request_line.py | 7 +- ooni/oonibclient.py | 3 - ooni/oonicli.py | 50 ++++---- ooni/otime.py | 79 +----------- ooni/report/tool.py | 4 +- ooni/reporter.py | 136 ++++++++++++++------- ooni/tasks.py | 5 +- ooni/templates/dnst.py | 107 +++++++++++----- ooni/templates/httpt.py | 50 ++++++-- ooni/templates/scapyt.py | 12 +- ooni/templates/tcpt.py | 1 + ooni/tests/test_director.py | 6 +- ooni/tests/test_oonicli.py | 17 +-- ooni/tests/test_otime.py | 14 --- ooni/tests/test_reporter.py | 23 +++- ooni/tests/test_templates.py | 44 ++++++- ooni/tests/test_trueheaders.py | 12 +- ooni/tests/test_utils.py | 10 +- ooni/utils/__init__.py | 13 +- ooni/utils/log.py | 2 +- ooni/utils/trueheaders.py | 2 +- requirements.txt | 2 +- 33 files changed, 526 insertions(+), 401 deletions(-)

1 0

[ooni-probe/master] fix the dependency on pyasn1 (#470)
by art＠torproject.org 29 Apr '16

29 Apr '16

commit f0b6332693cd7ea149d2b43780e133d9489a1379 Author: Veggie Monster <vmonmoonshine(a)gmail.com> Date: Tue Apr 26 09:44:51 2016 -0400 fix the dependency on pyasn1 (#470) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0143b58..28e5f78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ scapy-real>=2.2.0-dev pypcap>=1.1 service-identity pydumbnet -pyasn1 +pyasn1>=0.1.8 pyasn1-modules characteristic zope.interface

1 0