- tor-commits - lists.torproject.org

[ooni-probe/master] Fix annotations in ooniprobe
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 08d54806912c2e690f294d0b07afbe9e54f62cf8 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 26 18:39:06 2015 +0100 Fix annotations in ooniprobe --- ooni/nettest.py | 4 +++- ooni/oonicli.py | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 9eb6f37..848d958 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -139,6 +139,7 @@ class NetTestLoader(object): "(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]{64})$") self.options = options self.testCases = [] + self.annotations = {} if test_file: self.loadNetTestFile(test_file) @@ -217,7 +218,8 @@ class NetTestLoader(object): 'options': options, 'input_hashes': input_file_hashes, 'report_id': self.reportID, - 'test_helpers': self.testHelpers + 'test_helpers': self.testHelpers, + 'annotations': self.annotations } return test_details diff --git a/ooni/oonicli.py b/ooni/oonicli.py index b3e1013..1167da9 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -299,8 +299,7 @@ def runTestWithDirector(director, global_options, url=None, collector = setupCollector(global_options, net_test_loader) - test_details = net_test_loader.testDetails - test_details['annotations'] = global_options['annotations'] + net_test_loader.annotations = global_options['annotations'] director.startNetTest(net_test_loader, global_options['reportfile'],

1 0

[ooni-probe/master] add psiphon user documentation
by art＠torproject.org 29 Apr '16

29 Apr '16

commit d29782da9e3aa31feee5261a5713a04247e5e611 Author: juga0 <juga> Date: Thu Nov 19 19:55:36 2015 +0000 add psiphon user documentation --- docs/source/nettests/ooni.nettests.third_party.rst | 8 ++ docs/source/tests/psiphon.rst | 119 +++++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/docs/source/nettests/ooni.nettests.third_party.rst b/docs/source/nettests/ooni.nettests.third_party.rst index ef53443..7c0e692 100644 --- a/docs/source/nettests/ooni.nettests.third_party.rst +++ b/docs/source/nettests/ooni.nettests.third_party.rst @@ -16,3 +16,11 @@ ooni.nettests third_party Package :members: :undoc-members: :show-inheritance: + +:mod:`ooni.nettests.third_party.psiphon` Module +------------------------------------------------ + +.. automodule:: ooni.nettests.third_party.psiphon + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/tests/psiphon.rst b/docs/source/tests/psiphon.rst new file mode 100644 index 0000000..5050b79 --- /dev/null +++ b/docs/source/tests/psiphon.rst @@ -0,0 +1,119 @@ +Details +======= + +*Test Name*: Psiphon + +*Current version*: 0.1 + +*NetTest*: Psiphon (https://github.com/TheTorProject/ooni-probe/blob/master/ooni/nettests/third…) + +*Test Helper*: None + +*Test Type*: third party + +*Requires Root*: No + +Description +=========== + +This test first check that the Psiphon path exists, then launches Psiphon and parses output to determine if it has bootstrapped. After bootstrap, it fetches google.com (or other URL specified by the --url argument) using Psiphons SOCKS proxy listening on 127.0.0.1:1080 (or otherwise specified by the --socksproxy argument). + +The specific string used to determine bootstrap from Psiphon output in version +"0.0.1" is "Press Ctrl-C to terminate." from standard output. + +How to run the test +=================== + +`ooniprobe third_party/psiphon` + +To test Psiphon with a different URL run: + +`ooniprobe third_party/psiphon -u http://<url>/` + +To test Psiphon when it is installed in a different path other than the user home: + +`ooniprobe blocking/http_requests -p <path to Psiphon repository>` + +How to install Psiphon +=================== + +Run the install script: + +`scripts/psiphon_install.sh` + +To run Psiphon manually, it must be run inside of the proper directory: + +`cd <psiphonpath>//psiphon-circumvention-system/pyclient/` +`python psi_client.py` + +Sample report +============= + +`ooniprobe blocking/http_requests -f example_inputs/url_lists_file.txt` + + :: + + ########################################### + # OONI Probe Report for psiphon_test (0.0.1) + # Mon Oct 12 21:40:52 2015 + ########################################### + --- + input_hashes: [] + options: [-u, google.com] + probe_asn: AS0 + probe_cc: ZZ + probe_city: null + probe_ip: 127.0.0.1 + report_id: 4dAHr0ceNDBmw5lUQ7pBoxqgyUSfP873Qj1zv5VyElnSSTXwcsLYeCv69DsUjb94 + software_name: ooniprobe + software_version: 1.3.1 + start_time: 1444686051.0 + test_helpers: {} + test_name: psiphon_test + test_version: 0.0.1 + ... + --- + /tmp/tmplKg8K3: {exit_reason: process_done, stderr: '', stdout: "./ssh is not a valid\ + \ executable. Using standard ssh.\r\n\r\nYour SOCKS proxy is now running at 127.0.0.1:1080\r\ + \n\r\nPress Ctrl-C to terminate.\r\nTerminating...\r\nConnection closed\r\n"} + agent: agent + input: null + psiphon_installed: true + requests: + - request: + body: null + headers: [] + method: GET + tor: {is_tor: false} + url: http://google.com + response: + body: "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\"\ + >\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has\ + \ moved\n<A HREF=\"http://www.google.com/\">here</A>.\r\n</BODY></HTML>\r\n" + code: 301 + headers: + - - Content-Length + - ['219'] + - - X-XSS-Protection + - [1; mode=block] + - - Expires + - ['Wed, 11 Nov 2015 21:40:58 GMT'] + - - Server + - [gws] + - - Connection + - [close] + - - Location + - ['http://www.google.com/'] + - - Cache-Control + - ['public, max-age=2592000'] + - - Date + - ['Mon, 12 Oct 2015 21:40:58 GMT'] + - - X-Frame-Options + - [SAMEORIGIN] + - - Content-Type + - [text/html; charset=UTF-8] + socksproxy: 127.0.0.1:1080 + test_runtime: 7.373162031173706 + test_start_time: 1444686052.0 + ... +

1 0

[ooni-probe/master] Fix conditionals, move echo output
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 20fae977f9b905561435e9d64fa14eefd0edfa00 Author: anadahz <kojgelo(a)inbox.com> Date: Wed Nov 18 19:33:52 2015 +0100 Fix conditionals, move echo output --- scripts/psiphon_install.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh index e5b2a78..9a1431d 100755 --- a/scripts/psiphon_install.sh +++ b/scripts/psiphon_install.sh @@ -33,17 +33,18 @@ fi echo "installing dependencies" $sh_c "apt-get -y install zlib1g-dev libssl-dev" -if [ -z "command_exists hg" ]; then +if [ ! "command_exists hg" ]; then $sh_c "apt-get -y install mercurial" fi -echo "cloning psiphon repository" cd $PSIPHON_PATH if [ ! -d "psiphon-circumvention-system" ]; then + echo "cloning psiphon repository" hg clone $PSIPHON_REPO_URL - echo "psiphon repository cloned" fi +echo "psiphon repository cloned" + # optional, compile their ssh if [ ! -f "$PSIPHON_PYCLIENT_PATH/ssh" ]; then echo "compiling psiphon ssh" @@ -61,7 +62,7 @@ if [ `python -c 'import sys; print hasattr(sys, "real_prefix")'` = "False" ]; th # not in a virtualenv # create a virtualenv # FIXME: assuming debian version will have secure pip/virtualenv - if [ -z "command_exists virtualenv" ]; then + if [ ! "command_exists virtualenv" ]; then $sh_c "apt-get -y install python-virtualenv" fi if [ ! -f $OONI_VIRTUALENV_PATH/bin/activate ]; then

1 0

[ooni-probe/master] Really handleAllFailures()
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 864d001be170cb12fd5ad55faafe9518fbffce89 Author: srvetus <srvetus(a)users.noreply.github.com> Date: Fri Sep 25 00:23:53 2015 +0200 Really handleAllFailures() handleAllFailures() only handles failures which are explicitly named. Maybe it should be renamed or emit a log message when an unknown failure is reraised from `failure.trap()` The known failure list has been converted to a list of tuples as the order of the failure checking is important. Some Failures subclass other failures and so they should be listed from most specific failure to the least specific. --- ooni/errors.py | 69 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 5751baf..84c0afc 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -20,33 +20,33 @@ from txsocksx.errors import TTLExpired, CommandNotSupported from socket import gaierror -known_failures = { - ConnectionRefusedError: 'connection_refused_error', - ConnectionLost: 'connection_lost_error', - ConnectError: 'connect_error', - CancelledError: 'task_timed_out', - gaierror: 'address_family_not_supported_error', - DNSLookupError: 'dns_lookup_error', - TCPTimedOutError: 'tcp_timed_out_error', - ResponseNeverReceived: 'response_never_received', - DeferTimeoutError: 'deferred_timeout_error', - GenericTimeoutError: 'generic_timeout_error', - SOCKSError: 'socks_error', - MethodsNotAcceptedError: 'socks_methods_not_supported', - AddressNotSupported: 'socks_address_not_supported', - NetworkUnreachable: 'socks_network_unreachable', - ConnectionError: 'socks_connect_error', - ConnectionLostEarly: 'socks_connection_lost_early', - ConnectionNotAllowed: 'socks_connection_not_allowed', - NoAcceptableMethods: 'socks_no_acceptable_methods', - ServerFailure: 'socks_server_failure', - HostUnreachable: 'socks_host_unreachable', - ConnectionRefused: 'socks_connection_refused', - TTLExpired: 'socks_ttl_expired', - CommandNotSupported: 'socks_command_not_supported', - ProcessDone: 'process_done', - ConnectionDone: 'connection_done', -} +known_failures = [ + (ConnectionRefusedError, 'connection_refused_error'), + (ConnectionLost, 'connection_lost_error'), + (CancelledError, 'task_timed_out'), + (gaierror, 'address_family_not_supported_error'), + (DNSLookupError, 'dns_lookup_error'), + (TCPTimedOutError, 'tcp_timed_out_error'), + (ResponseNeverReceived, 'response_never_received'), + (DeferTimeoutError, 'deferred_timeout_error'), + (GenericTimeoutError, 'generic_timeout_error'), + (SOCKSError, 'socks_error'), + (MethodsNotAcceptedError, 'socks_methods_not_supported'), + (AddressNotSupported, 'socks_address_not_supported'), + (NetworkUnreachable, 'socks_network_unreachable'), + (ConnectionError, 'socks_connect_error'), + (ConnectionLostEarly, 'socks_connection_lost_early'), + (ConnectionNotAllowed, 'socks_connection_not_allowed'), + (NoAcceptableMethods, 'socks_no_acceptable_methods'), + (ServerFailure, 'socks_server_failure'), + (HostUnreachable, 'socks_host_unreachable'), + (ConnectionRefused, 'socks_connection_refused'), + (TTLExpired, 'socks_ttl_expired'), + (CommandNotSupported, 'socks_command_not_supported'), + (ProcessDone, 'process_done'), + (ConnectionDone, 'connection_done'), + (ConnectError, 'connect_error'), +] def handleAllFailures(failure): """ @@ -55,9 +55,7 @@ def handleAllFailures(failure): returned by failure.trap(). """ - # TODO: Should this function actually handle ALL failures and - # not just the failures configured in this list. - failure.trap(*known_failures.keys()) + failure.trap(*[failure_type for failure_type, _ in known_failures]) return failureToString(failure) @@ -75,12 +73,13 @@ def failureToString(failure): A string representing the HTTP response error message. """ - for failure_type, failure_string in known_failures.items(): + for failure_type, failure_string in known_failures: if isinstance(failure.value, failure_type): - return failure_string - - # Did not find a matching failure message - return 'unknown_failure %s' % str(failure.value) + if failure_string: + return failure_string + else: + # Failure without a corresponding failure message + return 'unknown_failure %s' % str(failure.value) class DirectorException(Exception): pass

1 0

[ooni-probe/master] Test that specific Failures are caught before parent Failures
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 525a8438bd601592c4f878ca5d42d3dab8943be0 Author: srvetus <srvetus(a)users.noreply.github.com> Date: Thu Nov 19 14:16:42 2015 +0100 Test that specific Failures are caught before parent Failures --- ooni/tests/test_errors.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ooni/tests/test_errors.py b/ooni/tests/test_errors.py new file mode 100644 index 0000000..7baa1ea --- /dev/null +++ b/ooni/tests/test_errors.py @@ -0,0 +1,20 @@ +from twisted.trial import unittest + +import ooni.errors + +class TestErrors(unittest.TestCase): + + def test_catch_child_failures_before_parent_failures(self): + """ + Verify that more specific Failures are caught first by + handleAllFailures() and failureToString(). + + Fails if a subclass is listed after it's parent Failure. + """ + + # Check each Failure against all subsequent failures + for index, (failure, _) in enumerate(ooni.errors.known_failures): + for sub_failure, _ in ooni.errors.known_failures[index+1:]: + + # Fail if subsequent Failure inherits from the current Failure + self.assertNotIsInstance(sub_failure(None), failure)

1 0

[ooni-probe/master] Major fixes and enhancements to psipshon install script
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 55f96585d62dcbbff41c01b9b8a97f5ee346c3d9 Author: anadahz <kojgelo(a)inbox.com> Date: Fri Nov 6 10:58:54 2015 +0100 Major fixes and enhancements to psipshon install script * Create required PSIPHON_PATH directory * Add command_exists function * Check if running as root user non sudo usage * Add required package dependencies * Fix check if hg command exits and if not install mercurial package * Check if virtualenv exists and if not install it * Fix minor format typos --- scripts/psiphon_install.sh | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh index 1b6837d..208aa83 100755 --- a/scripts/psiphon_install.sh +++ b/scripts/psiphon_install.sh @@ -6,9 +6,35 @@ set -ex PSIPHON_PATH=$HOME/test PSIPHON_PYCLIENT_PATH=$PSIPHON_PATH/psiphon-circumvention-system/pyclient +mkdir -p $PSIPHON_PATH + +command_exists() { + command -v "$@" > /dev/null 2>&1 +} + +user="$(id -un 2>/dev/null || true)" + +sh_c='sh -c' + +if [ "$user" != 'root' ]; then + if command_exists sudo; then + sh_c='sudo sh -c -E' + elif command_exists su; then + sh_c='su -c --preserve-environment' + else + echo >&2 'Error: this installer needs the ability to run commands as root.' + echo >&2 'We are unable to find either "sudo" or "su" available to make this happen.' + exit 1 + fi +fi + echo "installing dependencies" -# FIXME: check if hg command exits and if not bail out? -sudo apt-get install -y mercurial +$sh_c "apt-get -y install zlib1g-dev libssl-dev" + +if [ -z "command_exists hg" ]; then + $sh_c "apt-get -y install mercurial" +fi + echo "cloning psiphon repository" cd $PSIPHON_PATH hg clone https://bitbucket.org/psiphon/psiphon-circumvention-system @@ -29,8 +55,9 @@ if [ python -c 'import sys; print hasattr(sys, "real_prefix")' = "False"];then # we are not in a virtualenv # create a virtualenv # FIXME: assuming debian version will have secure pip/virtualenv - sudo apt-get -y install python-virtualenv - + if [ -z "command_exists virtualenv" ]; then + $sh_c "apt-get -y install python-virtualenv" + fi if [ ! -f $HOME/.virtualenvs/ooniprobe/bin/activate ]; then # Set up the virtual environment mkdir -p $HOME/.virtualenvs @@ -72,9 +99,12 @@ dat["servers"] = json.load(open('server_list'))['data'].split() json.dump(dat, open('psi_client.dat', 'w')) EOF -chmod +x psi_generate_dat.py +chmod +x psi_generate_dat.py +./psi_generate_dat.py +echo "servers data file created" +chmod +x psi_generate_dat.py ./psi_generate_dat.py -echo "serers data file created" +echo "servers data file created" mv psi_client.dat $PSIPHON_PYCLIENT_PATH

1 0

[ooni-probe/master] Move general SOCKSError after more specific failures
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 5f8bc2ca87dfcbc7ea666276c9a24b50e092c6c6 Author: srvetus <srvetus(a)users.noreply.github.com> Date: Thu Nov 19 13:52:47 2015 +0100 Move general SOCKSError after more specific failures --- ooni/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/errors.py b/ooni/errors.py index 84c0afc..55b97e1 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -30,7 +30,6 @@ known_failures = [ (ResponseNeverReceived, 'response_never_received'), (DeferTimeoutError, 'deferred_timeout_error'), (GenericTimeoutError, 'generic_timeout_error'), - (SOCKSError, 'socks_error'), (MethodsNotAcceptedError, 'socks_methods_not_supported'), (AddressNotSupported, 'socks_address_not_supported'), (NetworkUnreachable, 'socks_network_unreachable'), @@ -43,6 +42,7 @@ known_failures = [ (ConnectionRefused, 'socks_connection_refused'), (TTLExpired, 'socks_ttl_expired'), (CommandNotSupported, 'socks_command_not_supported'), + (SOCKSError, 'socks_error'), (ProcessDone, 'process_done'), (ConnectionDone, 'connection_done'), (ConnectError, 'connect_error'),

1 0

[ooni-probe/master] Add psiphon install script
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 75b88d7801fae016153e365600c3a6e649b613d0 Author: juga0 <juga> Date: Sun Sep 20 21:09:06 2015 +0000 Add psiphon install script --- scripts/psiphon_install.sh | 86 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh new file mode 100755 index 0000000..1b6837d --- /dev/null +++ b/scripts/psiphon_install.sh @@ -0,0 +1,86 @@ +#!/bin/sh + +#FIXME remove x +set -ex + +PSIPHON_PATH=$HOME/test +PSIPHON_PYCLIENT_PATH=$PSIPHON_PATH/psiphon-circumvention-system/pyclient + +echo "installing dependencies" +# FIXME: check if hg command exits and if not bail out? +sudo apt-get install -y mercurial +echo "cloning psiphon repository" +cd $PSIPHON_PATH +hg clone https://bitbucket.org/psiphon/psiphon-circumvention-system +echo "psiphon repository cloned" + +# optional, compile their ssh +echo "compiling psiphon ssh" +cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ +./configure +make +mv ssh ../../../pyclient/ +make clean +echo "psiphon ssh compiled" + +# check if we are in a virtualenv, create it otherwise +echo "checking virtualenv" +if [ python -c 'import sys; print hasattr(sys, "real_prefix")' = "False"];then + # we are not in a virtualenv + # create a virtualenv + # FIXME: assuming debian version will have secure pip/virtualenv + sudo apt-get -y install python-virtualenv + + if [ ! -f $HOME/.virtualenvs/ooniprobe/bin/activate ]; then + # Set up the virtual environment + mkdir -p $HOME/.virtualenvs + virtualenv $HOME/.virtualenvs/ooniprobe + source $HOME/.virtualenvs/ooniprobe/bin/activate + else + source $HOME/.virtualenvs/ooniprobe/bin/activate + fi +fi +echo "virtualenv activated" + +# create psi_client.dat +echo "creating servers data file" +echo "installing dependencies to create servers data file" +pip install -v --timeout 60 wget +cd /tmp +cat <<EOF > psi_generate_dat.py +#!/usr/bin/env python + +import wget +import os +import json + +# Delete 'server_list' if exists +if os.path.exists("server_list"): + # os.remove("server_list") + # os.rename("server_list", "server_list") + pass +else: + # Download 'server_list' + url ="https://psiphon3.com/server_list" + wget.download(url) + +# convert server_list to psi_client.dat +dat = {} +dat["propagation_channel_id"] = "FFFFFFFFFFFFFFFF" +dat["sponsor_id"] = "FFFFFFFFFFFFFFFF" +dat["servers"] = json.load(open('server_list'))['data'].split() +json.dump(dat, open('psi_client.dat', 'w')) +EOF + +chmod +x psi_generate_dat.py +./psi_generate_dat.py +echo "serers data file created" +mv psi_client.dat $PSIPHON_PYCLIENT_PATH + + +echo "[+] Installing all of the Python dependency requirements with pip in your virtualenv!"; +pip install -v --timeout 60 jsonpickle pexpect + +# run psiphon +# cd $PSIPHON_PYCLIENT_PATH +# python psi_client.py

1 0

[ooni-probe/master] fixes and enhancements to psipshon install script
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 955de5407b0ae3c4766ad9b48b99418ef2280a78 Author: juga0 <juga> Date: Tue Nov 10 17:35:09 2015 +0000 fixes and enhancements to psipshon install script * don't clone repository if the directory already exists * eliminate duplicated lines copying the server file * don't compile ssh client if the file already exists * fix syntax of line checking if a virtualenv is activated * fix psiphon directory path * remove x flag * remove commented lines for running psiphon * replace source command by dot * add variables for psiphon repo url and ooni virtualenv path * fix identation --- scripts/psiphon_install.sh | 89 +++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh index 208aa83..e5b2a78 100755 --- a/scripts/psiphon_install.sh +++ b/scripts/psiphon_install.sh @@ -3,13 +3,15 @@ #FIXME remove x set -ex -PSIPHON_PATH=$HOME/test +PSIPHON_PATH=$HOME PSIPHON_PYCLIENT_PATH=$PSIPHON_PATH/psiphon-circumvention-system/pyclient +PSIPHON_REPO_URL=https://bitbucket.org/psiphon/psiphon-circumvention-system +OONI_VIRTUALENV_PATH=$HOME/.virtualenvs/ooniprobe mkdir -p $PSIPHON_PATH command_exists() { - command -v "$@" > /dev/null 2>&1 + command -v "$@" > /dev/null 2>&1 } user="$(id -un 2>/dev/null || true)" @@ -17,55 +19,59 @@ user="$(id -un 2>/dev/null || true)" sh_c='sh -c' if [ "$user" != 'root' ]; then - if command_exists sudo; then - sh_c='sudo sh -c -E' - elif command_exists su; then - sh_c='su -c --preserve-environment' - else - echo >&2 'Error: this installer needs the ability to run commands as root.' - echo >&2 'We are unable to find either "sudo" or "su" available to make this happen.' - exit 1 - fi + if command_exists sudo; then + sh_c='sudo sh -c -E' + elif command_exists su; then + sh_c='su -c --preserve-environment' + else + echo >&2 'Error: this installer needs the ability to run commands as root.' + echo >&2 'We are unable to find either "sudo" or "su" available to make this happen.' + exit 1 + fi fi echo "installing dependencies" $sh_c "apt-get -y install zlib1g-dev libssl-dev" if [ -z "command_exists hg" ]; then - $sh_c "apt-get -y install mercurial" + $sh_c "apt-get -y install mercurial" fi echo "cloning psiphon repository" cd $PSIPHON_PATH -hg clone https://bitbucket.org/psiphon/psiphon-circumvention-system -echo "psiphon repository cloned" +if [ ! -d "psiphon-circumvention-system" ]; then + hg clone $PSIPHON_REPO_URL + echo "psiphon repository cloned" +fi # optional, compile their ssh -echo "compiling psiphon ssh" -cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ -./configure -make -mv ssh ../../../pyclient/ -make clean -echo "psiphon ssh compiled" +if [ ! -f "$PSIPHON_PYCLIENT_PATH/ssh" ]; then + echo "compiling psiphon ssh" + cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ + ./configure + make + mv ssh ../../../pyclient/ + make clean + echo "psiphon ssh compiled" +fi # check if we are in a virtualenv, create it otherwise echo "checking virtualenv" -if [ python -c 'import sys; print hasattr(sys, "real_prefix")' = "False"];then - # we are not in a virtualenv - # create a virtualenv - # FIXME: assuming debian version will have secure pip/virtualenv - if [ -z "command_exists virtualenv" ]; then - $sh_c "apt-get -y install python-virtualenv" - fi - if [ ! -f $HOME/.virtualenvs/ooniprobe/bin/activate ]; then - # Set up the virtual environment - mkdir -p $HOME/.virtualenvs - virtualenv $HOME/.virtualenvs/ooniprobe - source $HOME/.virtualenvs/ooniprobe/bin/activate - else - source $HOME/.virtualenvs/ooniprobe/bin/activate - fi +if [ `python -c 'import sys; print hasattr(sys, "real_prefix")'` = "False" ]; then + # not in a virtualenv + # create a virtualenv + # FIXME: assuming debian version will have secure pip/virtualenv + if [ -z "command_exists virtualenv" ]; then + $sh_c "apt-get -y install python-virtualenv" + fi + if [ ! -f $OONI_VIRTUALENV_PATH/bin/activate ]; then + # Set up the virtual environment + mkdir -p $HOME/.virtualenvs + virtualenv $OONI_VIRTUALENV_PATH + . $OONI_VIRTUALENV_PATH/bin/activate + else + . $OONI_VIRTUALENV_PATH/bin/activate + fi fi echo "virtualenv activated" @@ -83,7 +89,7 @@ import json # Delete 'server_list' if exists if os.path.exists("server_list"): - # os.remove("server_list") + # os.remove("server_list") # os.rename("server_list", "server_list") pass else: @@ -102,15 +108,8 @@ EOF chmod +x psi_generate_dat.py ./psi_generate_dat.py echo "servers data file created" -chmod +x psi_generate_dat.py -./psi_generate_dat.py -echo "servers data file created" mv psi_client.dat $PSIPHON_PYCLIENT_PATH - +rm /tmp/psi_generate_dat.py echo "[+] Installing all of the Python dependency requirements with pip in your virtualenv!"; pip install -v --timeout 60 jsonpickle pexpect - -# run psiphon -# cd $PSIPHON_PYCLIENT_PATH -# python psi_client.py

1 0

[ooni-probe/master] Refactor error handling to avoid inconsistencies
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 29119b673f2880eedd47b0531d58bb8e7425dea8 Author: srvetus <srvetus(a)users.noreply.github.com> Date: Thu Sep 24 18:04:53 2015 +0200 Refactor error handling to avoid inconsistencies The error handling code in handleAllFailures() and failureToString() is inconsistent and does not correctly trap all of the Failures for which there are error messages defined. The revised code places the Exception and error message in a single place to avoid inconsistencies arising in future. --- ooni/errors.py | 153 ++++++++++++++------------------------------------------- 1 file changed, 38 insertions(+), 115 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index cb18fe9..5751baf 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -20,39 +20,44 @@ from txsocksx.errors import TTLExpired, CommandNotSupported from socket import gaierror +known_failures = { + ConnectionRefusedError: 'connection_refused_error', + ConnectionLost: 'connection_lost_error', + ConnectError: 'connect_error', + CancelledError: 'task_timed_out', + gaierror: 'address_family_not_supported_error', + DNSLookupError: 'dns_lookup_error', + TCPTimedOutError: 'tcp_timed_out_error', + ResponseNeverReceived: 'response_never_received', + DeferTimeoutError: 'deferred_timeout_error', + GenericTimeoutError: 'generic_timeout_error', + SOCKSError: 'socks_error', + MethodsNotAcceptedError: 'socks_methods_not_supported', + AddressNotSupported: 'socks_address_not_supported', + NetworkUnreachable: 'socks_network_unreachable', + ConnectionError: 'socks_connect_error', + ConnectionLostEarly: 'socks_connection_lost_early', + ConnectionNotAllowed: 'socks_connection_not_allowed', + NoAcceptableMethods: 'socks_no_acceptable_methods', + ServerFailure: 'socks_server_failure', + HostUnreachable: 'socks_host_unreachable', + ConnectionRefused: 'socks_connection_refused', + TTLExpired: 'socks_ttl_expired', + CommandNotSupported: 'socks_command_not_supported', + ProcessDone: 'process_done', + ConnectionDone: 'connection_done', +} def handleAllFailures(failure): """ - Here we make sure to trap all the failures that are supported by the - failureToString function and we return the the string that represents the - failure. + Trap all the known Failures and we return a string that + represents the failure. Any unknown Failures will be reraised and + returned by failure.trap(). """ - failure.trap( - ConnectionRefusedError, - gaierror, - DNSLookupError, - TCPTimedOutError, - ResponseNeverReceived, - DeferTimeoutError, - GenericTimeoutError, - SOCKSError, - MethodsNotAcceptedError, - AddressNotSupported, - ConnectionError, - NetworkUnreachable, - ConnectionLostEarly, - ConnectionNotAllowed, - NoAcceptableMethods, - ServerFailure, - HostUnreachable, - ConnectionRefused, - TTLExpired, - CommandNotSupported, - ConnectError, - ConnectionLost, - CancelledError, - ConnectionDone) + # TODO: Should this function actually handle ALL failures and + # not just the failures configured in this list. + failure.trap(*known_failures.keys()) return failureToString(failure) @@ -69,95 +74,13 @@ def failureToString(failure): A string representing the HTTP response error message. """ - string = None - if isinstance(failure.value, ConnectionRefusedError): - # log.err("Connection refused.") - string = 'connection_refused_error' - elif isinstance(failure.value, ConnectionLost): - # log.err("Connection lost.") - string = 'connection_lost_error' - - elif isinstance(failure.value, ConnectError): - # log.err("Connect error.") - string = 'connect_error' - - elif isinstance(failure.value, gaierror): - # log.err("Address family for hostname not supported") - string = 'address_family_not_supported_error' - - elif isinstance(failure.value, DNSLookupError): - # log.err("DNS lookup failure") - string = 'dns_lookup_error' - - elif isinstance(failure.value, TCPTimedOutError): - # log.err("TCP Timed Out Error") - string = 'tcp_timed_out_error' - - elif isinstance(failure.value, ResponseNeverReceived): - # log.err("Response Never Received") - string = 'response_never_received' - - elif isinstance(failure.value, DeferTimeoutError): - # log.err("Deferred Timeout Error") - string = 'deferred_timeout_error' - - elif isinstance(failure.value, GenericTimeoutError): - # log.err("Time Out Error") - string = 'generic_timeout_error' - - elif isinstance(failure.value, ServerFailure): - # log.err("SOCKS error: ServerFailure") - string = 'socks_server_failure' - - elif isinstance(failure.value, ConnectionNotAllowed): - # log.err("SOCKS error: ConnectionNotAllowed") - string = 'socks_connection_not_allowed' - - elif isinstance(failure.value, NetworkUnreachable): - # log.err("SOCKS error: NetworkUnreachable") - string = 'socks_network_unreachable' - - elif isinstance(failure.value, HostUnreachable): - # log.err("SOCKS error: HostUnreachable") - string = 'socks_host_unreachable' - - elif isinstance(failure.value, ConnectionRefused): - # log.err("SOCKS error: ConnectionRefused") - string = 'socks_connection_refused' - - elif isinstance(failure.value, TTLExpired): - # log.err("SOCKS error: TTLExpired") - string = 'socks_ttl_expired' - - elif isinstance(failure.value, CommandNotSupported): - # log.err("SOCKS error: CommandNotSupported") - string = 'socks_command_not_supported' - - elif isinstance(failure.value, AddressNotSupported): - # log.err("SOCKS error: AddressNotSupported") - string = 'socks_address_not_supported' - - elif isinstance(failure.value, SOCKSError): - # log.err("Generic SOCKS error") - string = 'socks_error' - - elif isinstance(failure.value, CancelledError): - # log.err("Task timed out") - string = 'task_timed_out' - - elif isinstance(failure.value, ProcessDone): - string = 'process_done' - - elif isinstance(failure.value, ConnectionDone): - string = 'connection_done' - - else: - # log.err("Unknown failure type: %s" % type(failure.value)) - string = 'unknown_failure %s' % str(failure.value) - - return string + for failure_type, failure_string in known_failures.items(): + if isinstance(failure.value, failure_type): + return failure_string + # Did not find a matching failure message + return 'unknown_failure %s' % str(failure.value) class DirectorException(Exception): pass

1 0