December 2015 - tor-commits - lists.torproject.org

[ooni-probe/master] Really handleAllFailures()
by art＠torproject.org 02 Dec '15

02 Dec '15

commit 2c9b6e2984df5f5bf18e78981b767e1ad5eed8f5 Author: srvetus <srvetus(a)users.noreply.github.com> Date: Fri Sep 25 00:23:53 2015 +0200 Really handleAllFailures() handleAllFailures() only handles failures which are explicitly named. Maybe it should be renamed or emit a log message when an unknown failure is reraised from `failure.trap()` The known failure list has been converted to a list of tuples as the order of the failure checking is important. Some Failures subclass other failures and so they should be listed from most specific failure to the least specific. --- ooni/errors.py | 69 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 5751baf..84c0afc 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -20,33 +20,33 @@ from txsocksx.errors import TTLExpired, CommandNotSupported from socket import gaierror -known_failures = { - ConnectionRefusedError: 'connection_refused_error', - ConnectionLost: 'connection_lost_error', - ConnectError: 'connect_error', - CancelledError: 'task_timed_out', - gaierror: 'address_family_not_supported_error', - DNSLookupError: 'dns_lookup_error', - TCPTimedOutError: 'tcp_timed_out_error', - ResponseNeverReceived: 'response_never_received', - DeferTimeoutError: 'deferred_timeout_error', - GenericTimeoutError: 'generic_timeout_error', - SOCKSError: 'socks_error', - MethodsNotAcceptedError: 'socks_methods_not_supported', - AddressNotSupported: 'socks_address_not_supported', - NetworkUnreachable: 'socks_network_unreachable', - ConnectionError: 'socks_connect_error', - ConnectionLostEarly: 'socks_connection_lost_early', - ConnectionNotAllowed: 'socks_connection_not_allowed', - NoAcceptableMethods: 'socks_no_acceptable_methods', - ServerFailure: 'socks_server_failure', - HostUnreachable: 'socks_host_unreachable', - ConnectionRefused: 'socks_connection_refused', - TTLExpired: 'socks_ttl_expired', - CommandNotSupported: 'socks_command_not_supported', - ProcessDone: 'process_done', - ConnectionDone: 'connection_done', -} +known_failures = [ + (ConnectionRefusedError, 'connection_refused_error'), + (ConnectionLost, 'connection_lost_error'), + (CancelledError, 'task_timed_out'), + (gaierror, 'address_family_not_supported_error'), + (DNSLookupError, 'dns_lookup_error'), + (TCPTimedOutError, 'tcp_timed_out_error'), + (ResponseNeverReceived, 'response_never_received'), + (DeferTimeoutError, 'deferred_timeout_error'), + (GenericTimeoutError, 'generic_timeout_error'), + (SOCKSError, 'socks_error'), + (MethodsNotAcceptedError, 'socks_methods_not_supported'), + (AddressNotSupported, 'socks_address_not_supported'), + (NetworkUnreachable, 'socks_network_unreachable'), + (ConnectionError, 'socks_connect_error'), + (ConnectionLostEarly, 'socks_connection_lost_early'), + (ConnectionNotAllowed, 'socks_connection_not_allowed'), + (NoAcceptableMethods, 'socks_no_acceptable_methods'), + (ServerFailure, 'socks_server_failure'), + (HostUnreachable, 'socks_host_unreachable'), + (ConnectionRefused, 'socks_connection_refused'), + (TTLExpired, 'socks_ttl_expired'), + (CommandNotSupported, 'socks_command_not_supported'), + (ProcessDone, 'process_done'), + (ConnectionDone, 'connection_done'), + (ConnectError, 'connect_error'), +] def handleAllFailures(failure): """ @@ -55,9 +55,7 @@ def handleAllFailures(failure): returned by failure.trap(). """ - # TODO: Should this function actually handle ALL failures and - # not just the failures configured in this list. - failure.trap(*known_failures.keys()) + failure.trap(*[failure_type for failure_type, _ in known_failures]) return failureToString(failure) @@ -75,12 +73,13 @@ def failureToString(failure): A string representing the HTTP response error message. """ - for failure_type, failure_string in known_failures.items(): + for failure_type, failure_string in known_failures: if isinstance(failure.value, failure_type): - return failure_string - - # Did not find a matching failure message - return 'unknown_failure %s' % str(failure.value) + if failure_string: + return failure_string + else: + # Failure without a corresponding failure message + return 'unknown_failure %s' % str(failure.value) class DirectorException(Exception): pass

1 0

[ooni-probe/master] Merge pull request #433 from juga0/feature/openvpn-doc
by art＠torproject.org 02 Dec '15

02 Dec '15

commit 9abf35b3e0fede04483214486bf75955aac736ee Merge: 083ccc0 5e6c34b Author: anadahz <anadahz(a)users.noreply.github.com> Date: Thu Nov 26 01:34:18 2015 +0000 Merge pull request #433 from juga0/feature/openvpn-doc add openvpn user documentation docs/source/nettests/ooni.nettests.third_party.rst | 8 ++ docs/source/tests/openvpn.rst | 148 ++++++++++++++++++++ 2 files changed, 156 insertions(+)

1 0

[ooni-probe/master] add openvpn user documentation
by art＠torproject.org 02 Dec '15

02 Dec '15

commit 5e6c34beccb798750ebf0fbc17df5ed473a2d1ae Author: juga0 <juga> Date: Thu Nov 19 19:59:58 2015 +0000 add openvpn user documentation --- docs/source/nettests/ooni.nettests.third_party.rst | 8 ++ docs/source/tests/openvpn.rst | 148 ++++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/docs/source/nettests/ooni.nettests.third_party.rst b/docs/source/nettests/ooni.nettests.third_party.rst index 54573fb..ef53443 100644 --- a/docs/source/nettests/ooni.nettests.third_party.rst +++ b/docs/source/nettests/ooni.nettests.third_party.rst @@ -8,3 +8,11 @@ ooni.nettests third_party Package :members: :undoc-members: :show-inheritance: + +:mod:`ooni.nettests.third_party.openvpn` Module +------------------------------------------------ + +.. automodule:: ooni.nettests.third_party.openvpn + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/tests/openvpn.rst b/docs/source/tests/openvpn.rst new file mode 100644 index 0000000..a95da96 --- /dev/null +++ b/docs/source/tests/openvpn.rst @@ -0,0 +1,148 @@ +Details +======= + +*Test Name*: OpenVPN + +*Current version*: 0.1 + +*NetTest*: OpenVPN (https://github.com/TheTorProject/ooni-probe/blob/master/ooni/nettests/third…) + +*Test Helper*: None + +*Test Type*: third party + +*Requires Root*: Yes + +Description +=========== + +This test first launches OpenVPN and parses output to determine if it has bootstrapped. After bootstrap, it fetches the URL specified by the --url argument using OpenVPN. + +The specific string used to determine bootstrap from OpenVPN output in version +"0.0.1" is "Initialization Sequence Completed" from standard output. + +How to run the test +=================== + +First get credentials for a OpenVPN service and create the configuration file for OpenVPN. +An example: + + client + dev tun + proto tcp + remote <vpndomain> 80 + remote <vppnip> 80 + resolv-retry infinite + tun-mtu 1500 + key-direction 1 + nobind + persist-key + persist-tun + ns-cert-type server + comp-lzo + verb 3 + auth-user-pass <vpnuserandpasswordfile> + route-method exe + route-delay 2 + + <ca> + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- + </ca> + <tls-auth> + -----BEGIN OpenVPN Static key V1----- + ... + -----END OpenVPN Static key V1----- + </tls-auth> + +To run the test: + +`ooniprobe third_party/openvpn -u http://<url>/ -c <OpenVPN configuration file>` + +How to install OpenVPN +=================== + +Run the install script: + +`scripts/openvpn_install.sh` + +To run OpenVPN manually, : + +`openvpn --config <OpenVPN configuration file>` + +Sample report +============= + +`ooniprobe blocking/http_requests -f example_inputs/url_lists_file.txt` + + :: + + ########################################### + # OONI Probe Report for openvpn_client_test (0.0.2) + # Thu Nov 19 12:16:35 2015 + ########################################### + --- + input_hashes: [] + options: [-c, openvpn-config.ovpn, -u, ''] + probe_asn: AS0 + probe_cc: ZZ + probe_city: null + probe_ip: 127.0.0.1 + report_id: oVWw3KA3UH2XrOlLa8gRaUB1ocvn6LhMeTBOtU1yFMtdyM3L6Bg9ix1ulSWZCGtK + software_name: ooniprobe + software_version: 1.3.1 + start_time: 1447928194.0 + test_helpers: {} + test_name: openvpn_client_test + test_version: 0.0.2 + ... + --- + /usr/sbin/openvpn --config openvpn-config.ovpn: { + exit_reason: process_done, stderr: '', stdout: 'Thu Nov 19 12:16:35 2015 OpenVPN + 2.3.2 x86_64-pc-linux-gnu [SSL (OpenSSL)] [LZO] [EPOLL] [PKCS11] [eurephia] [MH] + [IPv6] built on Dec 1 2014 + + Thu Nov 19 12:16:35 2015 WARNING: file ''/tmp/openvpn.txt'' is group or others + accessible + + Thu Nov 19 12:16:35 2015 Control Channel Authentication: tls-auth using INLINE + static key file + + Thu Nov 19 12:16:35 2015 Attempting to establish TCP connection with [AF_INET]10.100.0.1:993 + [nonblock] + + Thu Nov 19 12:16:37 2015 TCP connection established with [AF_INET]10.100.0.1:993 + + Thu Nov 19 12:16:37 2015 TCPv4_CLIENT link local: [undef] + + Thu Nov 19 12:16:37 2015 TCPv4_CLIENT link remote: [AF_INET]10.100.0.1:993 + + Thu Nov 19 12:16:39 2015 WARNING: this configuration may cache passwords in memory + -- use the auth-nocache option to prevent this + + Thu Nov 19 12:16:42 2015 [server] Peer Connection Initiated with [AF_INET]10.100.0.1:993 + + Thu Nov 19 12:16:44 2015 Options error: Unrecognized option or missing parameter(s) + in [PUSH-OPTIONS]:3: dhcp (2.3.2) + + Thu Nov 19 12:16:44 2015 TUN/TAP device tun0 opened + + Thu Nov 19 12:16:44 2015 do_ifconfig, tt->ipv6=0, tt->did_ifconfig_ipv6_setup=0 + + Thu Nov 19 12:16:44 2015 /sbin/ip link set dev tun0 up mtu 1500 + + Thu Nov 19 12:16:44 2015 /sbin/ip addr add dev tun0 local 10.10.0.54 peer 10.10.0.53 + + Thu Nov 19 12:16:44 2015 Initialization Sequence Completed + + '} + body: "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\"\ + >\n<TITLE>302 Moved</TITLE></HEAD><BODY>\n<H1>302 Moved</H1>\nThe document has moved\n\ + <A HREF=\"http://www.google.de/?gfe_rd=cr&ei=yq9NVsq5JYvZ8Ae58KHoDA\">here</A>.\r\ + \n</BODY></HTML>\r\n" + input: null + success: true + test_runtime: 9.549253940582275 + test_start_time: 1447928195.0 + ...

1 0

[ooni-probe/master] add script to install openvpn for the openvpn test
by art＠torproject.org 02 Dec '15

02 Dec '15

commit 39f111fa5bca3653908132bd53ee76f38d35401c Author: juga0 <juga> Date: Thu Nov 19 15:36:51 2015 +0000 add script to install openvpn for the openvpn test --- scripts/openvpn_install.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/openvpn_install.sh b/scripts/openvpn_install.sh new file mode 100755 index 0000000..b9ab549 --- /dev/null +++ b/scripts/openvpn_install.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e + +user="$(id -un 2>/dev/null || true)" + +sh_c='sh -c' + +if [ "$user" != 'root' ]; then + if command_exists sudo; then + sh_c='sudo sh -c -E' + elif command_exists su; then + sh_c='su -c --preserve-environment' + else + echo >&2 'Error: this installer needs the ability to run commands as root.' + echo >&2 'We are unable to find either "sudo" or "su" available to make this happen.' + exit 1 + fi +fi + +$sh_c "apt-get -y install openvpn"

1 0

[ooni-probe/master] Merge pull request #437 from TheTorProject/feature/fix_annotation
by art＠torproject.org 02 Dec '15

02 Dec '15

commit 6837a3ccccb01aea0381defa20847001c145f3a1 Merge: dbfaf07 5a20acd Author: Simone Basso <bassosimone(a)gmail.com> Date: Tue Dec 1 21:41:43 2015 +0100 Merge pull request #437 from TheTorProject/feature/fix_annotation Fix annotations in ooniprobe ooni/nettest.py | 4 +++- ooni/oonicli.py | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-)

1 0

[ooni-probe/master] Merge pull request #426 from TheTorProject/feature/psiphon-dev-install-script
by art＠torproject.org 02 Dec '15

02 Dec '15

commit 0c67cabc3dbf9021ce277d3005832eccfdffc73e Merge: a53ac2e a6931f0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Dec 2 12:05:05 2015 +0100 Merge pull request #426 from TheTorProject/feature/psiphon-dev-install-script Add psiphon install script scripts/psiphon_install.sh | 115 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+)

1 0

[ooni-probe/master] Fix annotations in ooniprobe
by art＠torproject.org 02 Dec '15

02 Dec '15

commit 5a20acd1784f1326f3a35225f1b1722fc1ad5de1 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 26 18:39:06 2015 +0100 Fix annotations in ooniprobe --- ooni/nettest.py | 4 +++- ooni/oonicli.py | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 9eb6f37..848d958 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -139,6 +139,7 @@ class NetTestLoader(object): "(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]{64})$") self.options = options self.testCases = [] + self.annotations = {} if test_file: self.loadNetTestFile(test_file) @@ -217,7 +218,8 @@ class NetTestLoader(object): 'options': options, 'input_hashes': input_file_hashes, 'report_id': self.reportID, - 'test_helpers': self.testHelpers + 'test_helpers': self.testHelpers, + 'annotations': self.annotations } return test_details diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 030014e..1c8d6d0 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -299,8 +299,7 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True): and (not (config.tor_state or config.tor.socks_port)): raise errors.TorNotRunning - test_details = net_test_loader.testDetails - test_details['annotations'] = global_options['annotations'] + net_test_loader.annotations = global_options['annotations'] director.startNetTest(net_test_loader, global_options['reportfile'],

1 0

[ooni-probe/master] add psiphon user documentation
by art＠torproject.org 02 Dec '15

02 Dec '15

commit 122c8b91d81b6275305456f2b373b78c72872dce Author: juga0 <juga> Date: Thu Nov 19 19:55:36 2015 +0000 add psiphon user documentation --- docs/source/nettests/ooni.nettests.third_party.rst | 8 ++ docs/source/tests/psiphon.rst | 119 ++++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/docs/source/nettests/ooni.nettests.third_party.rst b/docs/source/nettests/ooni.nettests.third_party.rst index ef53443..7c0e692 100644 --- a/docs/source/nettests/ooni.nettests.third_party.rst +++ b/docs/source/nettests/ooni.nettests.third_party.rst @@ -16,3 +16,11 @@ ooni.nettests third_party Package :members: :undoc-members: :show-inheritance: + +:mod:`ooni.nettests.third_party.psiphon` Module +------------------------------------------------ + +.. automodule:: ooni.nettests.third_party.psiphon + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/tests/psiphon.rst b/docs/source/tests/psiphon.rst new file mode 100644 index 0000000..5050b79 --- /dev/null +++ b/docs/source/tests/psiphon.rst @@ -0,0 +1,119 @@ +Details +======= + +*Test Name*: Psiphon + +*Current version*: 0.1 + +*NetTest*: Psiphon (https://github.com/TheTorProject/ooni-probe/blob/master/ooni/nettests/third…) + +*Test Helper*: None + +*Test Type*: third party + +*Requires Root*: No + +Description +=========== + +This test first check that the Psiphon path exists, then launches Psiphon and parses output to determine if it has bootstrapped. After bootstrap, it fetches google.com (or other URL specified by the --url argument) using Psiphons SOCKS proxy listening on 127.0.0.1:1080 (or otherwise specified by the --socksproxy argument). + +The specific string used to determine bootstrap from Psiphon output in version +"0.0.1" is "Press Ctrl-C to terminate." from standard output. + +How to run the test +=================== + +`ooniprobe third_party/psiphon` + +To test Psiphon with a different URL run: + +`ooniprobe third_party/psiphon -u http://<url>/` + +To test Psiphon when it is installed in a different path other than the user home: + +`ooniprobe blocking/http_requests -p <path to Psiphon repository>` + +How to install Psiphon +=================== + +Run the install script: + +`scripts/psiphon_install.sh` + +To run Psiphon manually, it must be run inside of the proper directory: + +`cd <psiphonpath>//psiphon-circumvention-system/pyclient/` +`python psi_client.py` + +Sample report +============= + +`ooniprobe blocking/http_requests -f example_inputs/url_lists_file.txt` + + :: + + ########################################### + # OONI Probe Report for psiphon_test (0.0.1) + # Mon Oct 12 21:40:52 2015 + ########################################### + --- + input_hashes: [] + options: [-u, google.com] + probe_asn: AS0 + probe_cc: ZZ + probe_city: null + probe_ip: 127.0.0.1 + report_id: 4dAHr0ceNDBmw5lUQ7pBoxqgyUSfP873Qj1zv5VyElnSSTXwcsLYeCv69DsUjb94 + software_name: ooniprobe + software_version: 1.3.1 + start_time: 1444686051.0 + test_helpers: {} + test_name: psiphon_test + test_version: 0.0.1 + ... + --- + /tmp/tmplKg8K3: {exit_reason: process_done, stderr: '', stdout: "./ssh is not a valid\ + \ executable. Using standard ssh.\r\n\r\nYour SOCKS proxy is now running at 127.0.0.1:1080\r\ + \n\r\nPress Ctrl-C to terminate.\r\nTerminating...\r\nConnection closed\r\n"} + agent: agent + input: null + psiphon_installed: true + requests: + - request: + body: null + headers: [] + method: GET + tor: {is_tor: false} + url: http://google.com + response: + body: "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\"\ + >\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has\ + \ moved\n<A HREF=\"http://www.google.com/\">here</A>.\r\n</BODY></HTML>\r\n" + code: 301 + headers: + - - Content-Length + - ['219'] + - - X-XSS-Protection + - [1; mode=block] + - - Expires + - ['Wed, 11 Nov 2015 21:40:58 GMT'] + - - Server + - [gws] + - - Connection + - [close] + - - Location + - ['http://www.google.com/'] + - - Cache-Control + - ['public, max-age=2592000'] + - - Date + - ['Mon, 12 Oct 2015 21:40:58 GMT'] + - - X-Frame-Options + - [SAMEORIGIN] + - - Content-Type + - [text/html; charset=UTF-8] + socksproxy: 127.0.0.1:1080 + test_runtime: 7.373162031173706 + test_start_time: 1444686052.0 + ... +

1 0

[ooni-probe/master] Merge pull request #432 from juga0/feature/psiphon-doc
by art＠torproject.org 02 Dec '15

02 Dec '15

commit 93d9a6b984476f2af1410343541b277981821023 Merge: 9abf35b 122c8b9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Nov 29 16:28:21 2015 +0100 Merge pull request #432 from juga0/feature/psiphon-doc add psiphon user documentation docs/source/nettests/ooni.nettests.third_party.rst | 8 ++ docs/source/tests/psiphon.rst | 119 ++++++++++++++++++++ 2 files changed, 127 insertions(+)

1 0

[ooni-probe/master] Merge pull request #413 from srvetus/refactor-error-handling
by art＠torproject.org 02 Dec '15

02 Dec '15

commit a53ac2eefc50c58124e92db78125f52ae6b16f8c Merge: 6837a3c 7284fac Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Dec 2 12:00:47 2015 +0100 Merge pull request #413 from srvetus/refactor-error-handling Refactor error handling to avoid inconsistencies ooni/errors.py | 154 +++++++++++---------------------------------- ooni/tests/test_errors.py | 20 ++++++ 2 files changed, 58 insertions(+), 116 deletions(-)

1 0