- tor-commits - lists.torproject.org

[ooni-probe/master] Avoid installing Twisted 15.5 to avoid bugs with txtorcon
by art＠torproject.org 29 Apr '16

29 Apr '16

commit ad1cca904691cb246cf0bae079901ef965974f59 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 2 12:40:08 2015 +0100 Avoid installing Twisted 15.5 to avoid bugs with txtorcon --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8b8d5ca..b052adb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML>=3.10 -Twisted>=12.2.0 +Twisted>=12.2.0,<15.5.0 ipaddr>=2.1.10 pyOpenSSL>=0.13 geoip

1 0

[ooni-probe/master] Fix lantern test
by art＠torproject.org 29 Apr '16

29 Apr '16

commit a9e003e9cd6cc9a2767ab567f6215fb2d92cf603 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 11 11:39:59 2015 +0100 Fix lantern test --- ooni/nettests/third_party/lantern.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index c7173ed..1f1b3b0 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -4,7 +4,7 @@ from twisted.python import usage from twisted.web.client import ProxyAgent, readBody from ooni.templates.process import ProcessTest, ProcessDirector from ooni.utils import log -from ooni.errors import handleAllFailures, TaskTimedOut +from ooni.errors import handleAllFailures import os.path from os import getenv @@ -34,7 +34,7 @@ class LanternProcessDirector(ProcessDirector): def outReceived(self, data): self.stdout += data # output received, see if we have bootstrapped - if not self.bootstrapped.called and "client (http) proxy at" in self.stdout: + if not self.bootstrapped.called and "Connected to proxy on localhost" in self.stdout: log.debug("Bootstrap Detected") self.cancelTimer() self.bootstrapped.callback("bootstrapped") @@ -62,7 +62,7 @@ class LanternTest(ProcessTest): requiredOptions = ['url'] def setUp(self): - self.command = ["lantern_linux", "--headless"] + self.command = ["lantern", "--headless"] self.d = defer.Deferred() self.processDirector = LanternProcessDirector(self.d, timeout=self.timeout) self.d.addCallback(self.processEnded, self.command)

1 0

[ooni-probe/master] Add lantern test documentation
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 3d7f9448fcc4bcdfa18d56d758cf900f33e1c63a Author: anadahz <kojgelo(a)inbox.com> Date: Mon Dec 21 02:32:47 2015 +0100 Add lantern test documentation --- docs/source/tests/lantern.rst | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/source/tests/lantern.rst b/docs/source/tests/lantern.rst new file mode 100644 index 0000000..0903c3a --- /dev/null +++ b/docs/source/tests/lantern.rst @@ -0,0 +1,60 @@ +Details +======= + +*Test Name*: Lantern + +*Current version*: 0.0.1 + +*NetTest*: Name (https://github.com/TheTorProject/ooni-probe/blob/master/ooni/nettests/third…) + +*Test Helper*: None + +*Test Type*: Third Party + +*Requires Root*: No + +Description +=========== + +This test launches Lantern in --headless mode, and parses output to determine +if it has bootstrapped. After bootstrap, it fetches the URL supplied by the +--url option using Lanterns http proxy interface listening on 127.0.0.1.8787. + +The specific string used to determine bootstrap from Lantern output in version +"2.0.10" is "client (http) proxy at" from standard output. + +How to run the test +=================== + +`ooniprobe nettests/third_party/lantern.py -u http://<url>` + +Sample report +============= + +From running: +`ooniprobe nettests/third_party/lantern.py -u http://www.google.com` + +:: + +--- +input_hashes: [] +options: [-u, google.com] +probe_asn: AS1234 +probe_cc: US +probe_city: null +probe_ip: 127.0.0.1 +software_name: ooniprobe +software_version: 1.2.3-rc1 +start_time: 1428344311.0 +test_name: lantern_circumvention_tool_test +test_version: 0.0.1 +... +--- +body: "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\"\ + >\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n\ + <A HREF=\"http://www.google.com/\">here</A>.\r\n</BODY></HTML>\r\n" +bootstrapped: true +input: null +lantern --headless: {exit_reason: process_done, stderr: '', stdout: ''} +``` +

1 0

[ooni-probe/master] No need to downgrade Twisted
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 9963b3beda4ad29b050c60bbae2efdba6b3d79fc Author: anadahz <kojgelo(a)inbox.com> Date: Sun Dec 20 15:30:05 2015 +0100 No need to downgrade Twisted All Twister version are supported by ooniprobe! Fixed (https://trac.torproject.org/projects/tor/ticket/15944) --- scripts/install.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index beb9f60..01318ce 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -324,10 +324,6 @@ case "$lsb_dist" in set -x $sh_c 'apt-get install -y -q curl git-core python python-dev python-setuptools build-essential libdumbnet1 python-dumbnet python-libpcap tor tor-geoipdb libgeoip-dev libpcap0.8-dev libssl-dev libffi-dev libdumbnet-dev' $sh_c "PYTHONPATH=$PYTHONPATH pip install --install-option=\"--prefix=$PYTHON_PREFIX\" ooniprobe" - # Unistall Twisted and install a version >=12.2.0,<=14.0.0 - # https://trac.torproject.org/projects/tor/ticket/15944 - $sh_c "PYTHONPATH=$PYTHONPATH pip uninstall -y Twisted" - $sh_c "PYTHONPATH=$PYTHONPATH pip install --install-option=\"--prefix=$PYTHON_PREFIX\" 'Twisted>=12.2.0,<=14.0.0'" ) fi

1 0

[ooni-probe/master] README: proper path to example inputs
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 474d20b5e672a3310d1ff43bd8878454175cbf7b Author: ecneladis <ecneladis(a)gmail.com> Date: Sat Jan 9 17:02:37 2016 +0100 README: proper path to example inputs --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index c824c44..a1b6386 100644 --- a/README.rst +++ b/README.rst @@ -174,7 +174,7 @@ ooniprobe will be installed in ``/ooni``. 5. You can run tests with:: - ooniprobe blocking/http_requests -f /ooni/example_inputs/alexa-top-1k.txt + ooniprobe blocking/http_requests -f /ooni/var/example_inputs/alexa-top-1k.txt Using ooniprobe ===============

1 0

[ooni-probe/master] Add description of what test helpers are in introduction.
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 568b5cdd71755b3ba12861c72ec918c425b463c3 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jan 22 11:03:56 2016 +0100 Add description of what test helpers are in introduction. * Fix links --- docs/source/writing_test_helpers.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/source/writing_test_helpers.rst b/docs/source/writing_test_helpers.rst index 2206115..d391aa4 100644 --- a/docs/source/writing_test_helpers.rst +++ b/docs/source/writing_test_helpers.rst @@ -1,23 +1,23 @@ Writing Test Helpers ======== -OONI test helpers are... +OONI test helpers are used by OONI nettests to perform their meausrements. They can be used either to establish a ground truth or to exchange information with the probe to determine if some form of network manipulation is happening in the network path between the probe and the backend. Writing a Censorship Directionality Test Helper -------------------------- Our goal is to write an OONI test helper that helps an OONI-probe client determine the directionality of keyword censorship it has detected. To do this our helper will receive "encoded" data from an OONI-probe client, decode that data into a text string, and send the OONI-probe client a confirmation packet and a second packet containing the decoded string. -The OONI-backend code-base has many concise examples of test-helpers that you can build off to create your own. For this tutorial I used the `TCP echo test-helper <https://github.com/TheTorProject/ooni-backend/blob/479a1bb154037b834292ccc4…>` as my guide. +The OONI-backend code-base has many concise examples of test-helpers that you can build off to create your own. For this tutorial I used the `TCP echo test-helper <https://github.com/TheTorProject/ooni-backend/blob/479a1bb154037b834292ccc4…>`_ as my guide. -Following this tutorial requires basic knowledge of event-driven programming (specifically 'Twisted'). You will be more than ready to build and implement a test-helper after reading though one or two `tutorials online. <http://krondo.com/?page_id=1327>` +Following this tutorial requires basic knowledge of event-driven programming (specifically 'Twisted'). You will be more than ready to build and implement a test-helper after reading though one or two `tutorials online. <http://krondo.com/?page_id=1327>`_ Creating the test helper -------------- -OONI-backend keeps all the test-helpers in the `oonib/testhelpers directory <https://github.com/TheTorProject/ooni-backend/tree/master/oonib/testhelpers>` Each individual test helper is a twisted service. Most of the current test-helpers consists of a twisted Factory and a twisted Protocol defined in the test helpers directory and a `stock Twisted Server <https://twistedmatrix.com/documents/current/api/twisted.application.interne…>` that is defined in the backend code. We will follow this model in the tutorial. +OONI-backend keeps all the test-helpers in the `oonib/testhelpers directory <https://github.com/TheTorProject/ooni-backend/tree/master/oonib/testhelpers>`_ Each individual test helper is a twisted service. Most of the current test-helpers consists of a twisted Factory and a twisted Protocol defined in the test helpers directory and a `stock Twisted Server <https://twistedmatrix.com/documents/current/api/twisted.application.interne…>`_ that is defined in the backend code. We will follow this model in the tutorial. -Because of how simple this example test-helper is the job of our test-helper factory is merely to deploy a single instance of our protocol each time it's buildProtocol method is called. Because we have our factory inheret from the base `Factory object <https://twistedmatrix.com/trac/browser/tags/releases/twisted-15.5.0/twisted…>` we merely have to define its ``protocol`` property to point to our protocol.:: +Because of how simple this example test-helper is the job of our test-helper factory is merely to deploy a single instance of our protocol each time it's buildProtocol method is called. Because we have our factory inheret from the base `Factory object <https://twistedmatrix.com/trac/browser/tags/releases/twisted-15.5.0/twisted…>`_ we merely have to define its ``protocol`` property to point to our protocol.:: class TCPDirectionalityTestHelper(Factory): """ @@ -26,7 +26,7 @@ Because of how simple this example test-helper is the job of our test-helper fac protocol = TCPDirectionalityTestProtocol -The protocol for this helper needs to do two things. First, upon receiving encoded data it needs to send the OONI-probe client back confirmation that the data has been received. Second, it needs to send the decoded data back to the OONI-probe client. Because we are extending the `Protocol object <https://twistedmatrix.com/trac/browser/tags/releases/twisted-15.5.0/twisted…>` we can rewrite its ``dataReceived`` method which is called and passed data whenever it is received.:: +The protocol for this helper needs to do two things. First, upon receiving encoded data it needs to send the OONI-probe client back confirmation that the data has been received. Second, it needs to send the decoded data back to the OONI-probe client. Because we are extending the `Protocol object <https://twistedmatrix.com/trac/browser/tags/releases/twisted-15.5.0/twisted…>`_ we can rewrite its ``dataReceived`` method which is called and passed data whenever it is received.:: class TCPDirectionalityTestProtocol(Protocol): @@ -68,7 +68,7 @@ With this added we have completed the base of simple test-helper and now just ha Adding the helper to the config file ------ -OONI-backend uses a config file located at `/etc/oonibackend.conf <https://github.com/TheTorProject/ooni-backend/blob/master/oonib.conf.example>`. This file contains a `section where each test-helper can be configured. <https://github.com/TheTorProject/ooni-backend/blob/479a1bb154037b834292ccc4…>`. +OONI-backend uses a config file located at `/etc/oonibackend.conf <https://github.com/TheTorProject/ooni-backend/blob/master/oonib.conf.example>`_. This file contains a `section where each test-helper can be configured. <https://github.com/TheTorProject/ooni-backend/blob/479a1bb154037b834292ccc4…>`_. The test-helper will need to be given a unique identifyer so that it can be called from the config file. In this example we use ``tcp-directionality`` as our identifyer. @@ -81,17 +81,17 @@ For a helper to be used in the ooni-backend it needs to be given an identifyer s Adding the helper to the backend ------ -For a helper to be integrated into the ooni-backend it needs to be added to the initialization scripts contained within `oonibackend.py <https://github.com/TheTorProject/ooni-backend/blob/master/oonib/oonibackend…>`. +For a helper to be integrated into the ooni-backend it needs to be added to the initialization scripts contained within `oonibackend.py <https://github.com/TheTorProject/ooni-backend/blob/master/oonib/oonibackend…>`_. -The OONI test-helper system is a collection of `Twisted services <https://twistedmatrix.com/documents/current/core/howto/application.html>`. For our test-helper we will need to define a service that will run our test-helper factory.:: +The OONI test-helper system is a collection of `Twisted services <https://twistedmatrix.com/documents/current/core/howto/application.html>`_. For our test-helper we will need to define a service that will run our test-helper factory.:: # Create the service that will run our test-helpers factory. tcp_directionality_helper = internet.TCPServer(int(port), tcp_helpers.TCPDirectionalityTestHelper()) -**NOTE:** In this example I have placed the original service in the existing tcp_helpers file. If you created your own file for your test-helper you will have to make sure that you import that file at the top of `oonibackend.py <https://github.com/TheTorProject/ooni-backend/blob/master/oonib/oonibackend…>`. +**NOTE:** In this example I have placed the original service in the existing tcp_helpers file. If you created your own file for your test-helper you will have to make sure that you import that file at the top of `oonibackend.py <https://github.com/TheTorProject/ooni-backend/blob/master/oonib/oonibackend…>`_. -OONI uses a `Multi Service <https://twistedmatrix.com/documents/current/api/twisted.application.service…>` which allows them to combine all the OONI test-helpers and the report-collector into a singlular service for easier management. The next step for creating our test-helper is to add it to the OONI-backend `multiService <https://github.com/TheTorProject/ooni-backend/blob/479a1bb154037b834292ccc4…>`:: +OONI uses a `Multi Service <https://twistedmatrix.com/documents/current/api/twisted.application.service…>`_ which allows them to combine all the OONI test-helpers and the report-collector into a singlular service for easier management. The next step for creating our test-helper is to add it to the OONI-backend `multiService <https://github.com/TheTorProject/ooni-backend/blob/479a1bb154037b834292ccc4…>`_:: # Add the helper as a child of the backends multi-service test-helper multiService.addService(tcp_directionality_helper) @@ -103,7 +103,7 @@ Finally, we need to start our service.:: In order for our test-helper to be managed using the backend config file we will need to modify this code to check the config file for a test-helper that uses the identifyer we selected earlier. For the directionality helper we check to see if our test-helper had its port specified in the config file to determine if it should be run. I also added a default encoding in case -This snippet contains the final code that would be inserted into `oonibackend.py <https://github.com/TheTorProject/ooni-backend/blob/master/oonib/oonibackend…>`.:: +This snippet contains the final code that would be inserted into `oonibackend.py <https://github.com/TheTorProject/ooni-backend/blob/master/oonib/oonibackend…>`_.:: # Check to see if our test-helper was defined in the config if config.helpers['tcp-directionality'].port:

1 0

[ooni-probe/master] README: Vagrantfile requires ooni-backend in parent directory
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 2c18041d9b1d13a788df27292fca83c0460bac04 Author: ecneladis <ecneladis(a)gmail.com> Date: Sat Jan 9 16:54:49 2016 +0100 README: Vagrantfile requires ooni-backend in parent directory --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index bf52114..c824c44 100644 --- a/README.rst +++ b/README.rst @@ -162,6 +162,7 @@ On debian/ubuntu:: 3. Open a Terminal and run:: git clone https://git.torproject.org/ooni-probe.git + git clone https://github.com/TheTorProject/ooni-backend.git cd ooni-probe/ vagrant up

1 0

[ooni-probe/master] Fix typo in code snippet example
by art＠torproject.org 29 Apr '16

29 Apr '16

commit e0428e869f9dab9eb7dec9dab9425a0341eaedc6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jan 22 11:08:22 2016 +0100 Fix typo in code snippet example --- docs/source/writing_test_helpers.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/writing_test_helpers.rst b/docs/source/writing_test_helpers.rst index d391aa4..a8601cc 100644 --- a/docs/source/writing_test_helpers.rst +++ b/docs/source/writing_test_helpers.rst @@ -72,9 +72,9 @@ OONI-backend uses a config file located at `/etc/oonibackend.conf <https://githu The test-helper will need to be given a unique identifyer so that it can be called from the config file. In this example we use ``tcp-directionality`` as our identifyer. -For a helper to be used in the ooni-backend it needs to be given an identifyer so that it can be called from the config file.:: +For a helper to be used in the ooni-backend it needs to be given an identifyer so that it can be called from the config file:: - tcp-echo: + tcp-directionality: encoding: rot13 port: 57009

1 0

[ooni-probe/master] Add link to test_helper_tutorial in index
by art＠torproject.org 29 Apr '16

29 Apr '16

commit b97304a0cf459437ad7849ec116f29c570393315 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jan 22 11:04:19 2016 +0100 Add link to test_helper_tutorial in index --- docs/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index 10c7c8f..1eabdee 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -508,6 +508,7 @@ More developer documentation tutorial writing_tests + writing_test_helpers reports nettests/modules api/*

1 0

[ooni-probe/master] Fix to make the psiphon test run
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 5f7e947953c8af4ca56d0462e69e80ae991293b4 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 9 17:34:51 2015 +0100 Fix to make the psiphon test run * Remove unneeded imports --- ooni/nettests/third_party/psiphon.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 543e856..fb912a9 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -1,19 +1,12 @@ import tempfile -import stat import os import sys from twisted.internet import defer, reactor -from twisted.internet.endpoints import TCP4ClientEndpoint -from twisted.web.client import readBody from twisted.python import usage -from txsocksx.http import SOCKS5Agent - -from ooni.errors import handleAllFailures, TaskTimedOut from ooni.utils import log from ooni.templates import process, httpt -from ooni.templates.process import ProcessTest class UsageOptions(usage.Options): @@ -94,9 +87,9 @@ connect(False) def test_psiphon(self): log.debug('PsiphonTest.test_psiphon') + self.report['success'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') - self.report['success'] = False self.report['psiphon_installed'] = False log.debug("Adding %s to report" % self.report) # XXX: the original code written by juga0 readed @@ -116,7 +109,8 @@ connect(False) # full with some block size and therefore the test would # terminate with error finished = self.run(self.command, - env=dict(PYTHONPATH=self.psiphonpath), + env=dict(PYTHONPATH=os.path.join(self.psiphonpath, + 'pyclient')), path=self.psiphonpath, usePTY=1)

1 0