April 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Fix to make the psiphon test run
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 5f7e947953c8af4ca56d0462e69e80ae991293b4 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 9 17:34:51 2015 +0100 Fix to make the psiphon test run * Remove unneeded imports --- ooni/nettests/third_party/psiphon.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 543e856..fb912a9 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -1,19 +1,12 @@ import tempfile -import stat import os import sys from twisted.internet import defer, reactor -from twisted.internet.endpoints import TCP4ClientEndpoint -from twisted.web.client import readBody from twisted.python import usage -from txsocksx.http import SOCKS5Agent - -from ooni.errors import handleAllFailures, TaskTimedOut from ooni.utils import log from ooni.templates import process, httpt -from ooni.templates.process import ProcessTest class UsageOptions(usage.Options): @@ -94,9 +87,9 @@ connect(False) def test_psiphon(self): log.debug('PsiphonTest.test_psiphon') + self.report['success'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') - self.report['success'] = False self.report['psiphon_installed'] = False log.debug("Adding %s to report" % self.report) # XXX: the original code written by juga0 readed @@ -116,7 +109,8 @@ connect(False) # full with some block size and therefore the test would # terminate with error finished = self.run(self.command, - env=dict(PYTHONPATH=self.psiphonpath), + env=dict(PYTHONPATH=os.path.join(self.psiphonpath, + 'pyclient')), path=self.psiphonpath, usePTY=1)

1 0

[ooni-probe/master] changed report structure for ProcessTest
by art＠torproject.org 29 Apr '16

29 Apr '16

commit a309440621561c47d4cb7db5b04e73b342d96307 Author: juga0 <juga> Date: Wed Dec 23 16:51:35 2015 +0000 changed report structure for ProcessTest --- ooni/templates/process.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 7ec4b2e..b7dafdd 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -103,12 +103,10 @@ class ProcessTest(NetTestCase): def processEnded(self, result, command): log.debug("Finished %s: %s" % (command, result)) - key = ' '.join(command) - self.report[key] = { - 'stdout': result['stdout'], - 'stderr': result['stderr'], - 'exit_reason': result['exit_reason'] - } + self.report['command_name'] = ' '.join(command) + self.report['command_stdout'] = result['stdout'] + self.report['command_stderr'] = result['stderr'] + self.report['command_exit_reason'] = result['exit_reason'] return result def run(self, command, finished=None, env={}, path=None, usePTY=0):

1 0

[ooni-probe/master] fix checking if commands exist, other minor changes
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 0e975d3c47a1a6d9cdbe028b2c3ce08aca6431ce Author: juga0 <juga> Date: Wed Dec 23 19:50:07 2015 +0000 fix checking if commands exist, other minor changes * fix checking if commands exist * only install virtualenv if it is needed to create a new one * add/change debug ouput --- scripts/psiphon_install.sh | 50 ++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh index 117c823..12086c8 100755 --- a/scripts/psiphon_install.sh +++ b/scripts/psiphon_install.sh @@ -20,8 +20,10 @@ sh_c='sh -c' if [ "$user" != 'root' ]; then if command_exists sudo; then sh_c='sudo sh -c -E' + echo "[D] using sudo" elif command_exists su; then sh_c='su -c --preserve-environment' + echo "[D] using su" else echo >&2 'Error: this installer needs the ability to run commands as root.' echo >&2 'We are unable to find either "sudo" or "su" available to make this happen.' @@ -29,42 +31,49 @@ if [ "$user" != 'root' ]; then fi fi -echo "installing dependencies" +echo "[D] installing dependencies" $sh_c "apt-get -y install zlib1g-dev libssl-dev" -if [ ! "command_exists hg" ]; then +if ! command_exists hg; then + echo "[D] installing mercurial" $sh_c "apt-get -y install mercurial" fi +echo "[D] mercurial installed" cd $PSIPHON_PATH if [ ! -d "psiphon-circumvention-system" ]; then - echo "cloning psiphon repository" + echo "[D] cloning psiphon repository" hg clone $PSIPHON_REPO_URL fi -echo "psiphon repository cloned" +echo "[D] psiphon repository cloned" # optional, compile their ssh if [ ! -f "$PSIPHON_PYCLIENT_PATH/ssh" ]; then - echo "compiling psiphon ssh" + echo "[D] compiling psiphon ssh" cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ ./configure make - mv ssh ../../../pyclient/ + mv ssh ../../../pyclient/pyclient/ make clean - echo "psiphon ssh compiled" + echo "[D] psiphon ssh compiled" fi # check if we are in a virtualenv, create it otherwise -echo "checking virtualenv" +echo "[D] checking virtualenv" if [ `python -c 'import sys; print hasattr(sys, "real_prefix")'` = "False" ]; then - # not in a virtualenv - # create a virtualenv - # FIXME: assuming debian version will have secure pip/virtualenv - if [ ! "command_exists virtualenv" ]; then - $sh_c "apt-get -y install python-virtualenv" - fi + echo "[D] not in a virtualenv" if [ ! -f $OONI_VIRTUALENV_PATH/bin/activate ]; then + echo "[D] virtualenv not found" + # create a virtualenv + # FIXME: assuming debian version will have secure pip/virtualenv + if ! command_exists virtualenv; then + echo "[D] installing virtualenv" + $sh_c "apt-get -y install python-virtualenv" + else + echo "[D] virtualenv command found" + fi + echo "[D] creating a virtualenv" # Set up the virtual environment mkdir -p $HOME/.virtualenvs virtualenv $OONI_VIRTUALENV_PATH @@ -72,12 +81,12 @@ if [ `python -c 'import sys; print hasattr(sys, "real_prefix")'` = "False" ]; th else . $OONI_VIRTUALENV_PATH/bin/activate fi + echo "[D] virtualenv activated" fi -echo "virtualenv activated" # create psi_client.dat -echo "creating servers data file" -echo "installing dependencies to create servers data file" +echo "[D] creating servers data file" +echo "[D] installing dependencies to create servers data file" pip install -v --timeout 60 wget cd /tmp cat <<EOF > psi_generate_dat.py @@ -107,9 +116,12 @@ EOF chmod +x psi_generate_dat.py ./psi_generate_dat.py -echo "servers data file created" +echo "[D] servers data file created" mv psi_client.dat $PSIPHON_PYCLIENT_PATH rm /tmp/psi_generate_dat.py -echo "[+] Installing all of the Python dependency requirements with pip in your virtualenv!"; +echo "[D] installing all of the Python dependency requirements with pip in the virtualenv"; pip install -v --timeout 60 jsonpickle pexpect + +echo "You can now run Psiphon: cd ~/psiphon-circumvention-system/pyclient/pyclient;python psi_client.py" +echo "NOTE that if OONI is not installed, you will not be able to run OONI Psiphon test"

1 0

[ooni-probe/master] leave success key to none in case psiphon fails
by art＠torproject.org 29 Apr '16

29 Apr '16

commit fbf9de395051b2f88d3f68a122f22ff25aa1cee0 Author: juga0 <juga> Date: Wed Dec 23 16:18:59 2015 +0000 leave success key to none in case psiphon fails --- ooni/nettests/third_party/psiphon.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 17c83c6..9314132 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -91,7 +91,6 @@ connect(False) self.report['psiphon_installed'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') - self.report['success'] = False self.report['psiphon_installed'] = False log.debug("Adding %s to report" % self.report) # XXX: the original code written by juga0 readed

1 0

[ooni-probe/master] add/change report variable names
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 1a38fe60ac1137e45b6dc1c8c038f7a2133f85d3 Author: juga0 <juga> Date: Thu Dec 24 13:42:34 2015 +0000 add/change report variable names * add bootstrapped_success report key with value true when the text 'Press Ctrl-C to terminate.' is found, false when psiphon runs but is not found and none when psiphon does not run * change success report key to request_success * keep psiphon_installed key, but change the name to psiphon_found --- ooni/nettests/third_party/psiphon.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index a25e955..bfdec70 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -82,17 +82,21 @@ connect(False) def handleRead(self, stdout, stderr): if 'Press Ctrl-C to terminate.' in self.processDirector.stdout: if not self.bootstrapped.called: + # here the text 'Press Ctrl-C to terminate.' has been found + # and it was to call doRequest + self.report['bootstrapped_success'] = True log.debug("PsiphonTest: calling bootstrapped.callback") self.bootstrapped.callback(None) def test_psiphon(self): log.debug('PsiphonTest.test_psiphon') - self.report['success'] = None - self.report['psiphon_installed'] = None + self.report['bootstrapped_success'] = None + self.report['request_success'] = None + self.report['psiphon_found'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') - self.report['psiphon_installed'] = False + self.report['psiphon_found'] = False log.debug("Adding %s to report" % self.report) # XXX: the original code written by juga0 readed # > return defer.succeed(None) @@ -103,7 +107,7 @@ connect(False) reactor.callLater(0.0, self.bootstrapped.callback, None) return self.bootstrapped - self.report['psiphon_installed'] = True + self.report['psiphon_found'] = True log.debug("Adding %s to report" % self.report) # Using pty to see output lines as soon as they get wrotten in the @@ -114,18 +118,22 @@ connect(False) env=dict(PYTHONPATH=self.psiphonpath), path=self.psiphonpath, usePTY=1) + # here psiphon command has been run, and if it finds the text + # 'Press Ctrl-C to terminate' in handleRead it will write to the + # report self.report['bootstrapped_success'] = True + self.report['bootstrapped_success'] = False def callDoRequest(_): log.debug("PsiphonTest.callDoRequest: %r" %(_,)) d = self.doRequest(self.url) def addSuccessToReport(res): log.debug("PsiphonTest.callDoRequest.addSuccessToReport") - self.report['success'] = True + self.report['request_success'] = True return res d.addCallback(addSuccessToReport) def addFailureToReport(res): log.debug("PsiphonTest.callDoRequest.addFailureToReport. res=%r" % (res,)) - self.report['success'] = False + self.report['request_success'] = False return res d.addErrback(addFailureToReport) return d

1 0

[ooni-probe/master] Log failure when OpenVPN process hangs for user input
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 1944e8e80d7605a161a3c56a8bf5ce983b1a10e7 Author: srvetus <srvetus(a)users.noreply.github.com> Date: Thu Dec 10 21:03:58 2015 +0100 Log failure when OpenVPN process hangs for user input --- ooni/nettests/third_party/openvpn.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/third_party/openvpn.py b/ooni/nettests/third_party/openvpn.py index 028bf17..bc2f0ea 100644 --- a/ooni/nettests/third_party/openvpn.py +++ b/ooni/nettests/third_party/openvpn.py @@ -57,6 +57,15 @@ class OpenVPNTest(ProcessTest): self.processDirector.transport.signalProcess('TERM') self.exited = True + def inConnectionLost(self): + """Monkeypatch inConnectionLost to log failure if the process ends + unexpectedly before OpenVPN bootstraps. + """ + log.debug("inConnectionLost") + + if not self.bootstrapped.called: + self.bootstrapped.errback(Exception("openvpn_exited_unexpectedly")) + def processExited(self, reason): """Monkeypatch processExited to log failure if the process ends unexpectedly before OpenVPN bootstraps. @@ -64,9 +73,11 @@ class OpenVPNTest(ProcessTest): log.debug("Exited %s" % handleAllFailures(reason)) # Process exited before OpenVPN bootstrapped. Add failure to report + # if it is not added already if not self.bootstrapped.called: self.bootstrapped.errback(Exception("openvpn_exited_unexpectedly")) + def handleRead(self, stdout=None, stderr=None): """handleRead is called with each chunk of data from stdout and stderr @@ -116,7 +127,8 @@ class OpenVPNTest(ProcessTest): log.debug("Spawning OpenVPN") self.d = self.run(self.command) - # Monkeypatch processExited to log when OpenVPN exits early + # Monkeypatch inConnectionLost and processExited to log when OpenVPN exits early + self.processDirector.inConnectionLost = self.inConnectionLost self.processDirector.processExited = self.processExited # Try to make a request when the OpenVPN connection successfully bootstraps

1 0

[ooni-probe/master] catch ProcessExitedAlready exception during cleanup
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 2929098b673542bb6148e61f5cf15a8eba313e00 Author: juga0 <juga> Date: Wed Dec 23 16:31:52 2015 +0000 catch ProcessExitedAlready exception during cleanup --- ooni/nettests/third_party/psiphon.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 9314132..a25e955 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -3,6 +3,7 @@ import os import sys from twisted.internet import defer, reactor +from twisted.internet.error import ProcessExitedAlready from twisted.python import usage from ooni.utils import log @@ -132,7 +133,10 @@ connect(False) def cleanup(_): log.debug('PsiphonTest:cleanup') - self.processDirector.transport.signalProcess('INT') + try: + self.processDirector.transport.signalProcess('INT') + except ProcessExitedAlready: + pass os.remove(self.command[1]) return finished

1 0

[ooni-probe/master] add success key to the report after doRequest
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 50d2f89e51885456d9d16a5df6995feb56f2370c Author: juga0 <juga> Date: Tue Dec 22 21:14:21 2015 +0000 add success key to the report after doRequest --- ooni/nettests/third_party/psiphon.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 3129d5e..94dfa51 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -116,7 +116,19 @@ connect(False) usePTY=1) def callDoRequest(_): - return self.doRequest(self.url) + log.debug("PsiphonTest.callDoRequest: %r" %(_,)) + d = self.doRequest(self.url) + def addSuccessToReport(res): + log.debug("PsiphonTest.callDoRequest.addSuccessToReport") + self.report['success'] = True + return res + d.addCallback(addSuccessToReport) + def addFailureToReport(res): + log.debug("PsiphonTest.callDoRequest.addFailureToReport. res=%r" % (res,)) + self.report['success'] = False + return res + d.addErrback(addFailureToReport) + return d self.bootstrapped.addCallback(callDoRequest) def cleanup(_):

1 0

[ooni-probe/master] change new psiphon path to the psiphonpath attribute
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 40c4b7e7d1fc392aa7657d17a76ab18894aad100 Author: juga0 <juga> Date: Tue Dec 22 21:10:13 2015 +0000 change new psiphon path to the psiphonpath attribute --- ooni/nettests/third_party/psiphon.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 1e85c78..3129d5e 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -63,7 +63,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): # are in the user's home directory. from os import path, getenv self.psiphonpath = path.join( - getenv('HOME'), 'psiphon-circumvention-system/pyclient') + getenv('HOME'), 'psiphon-circumvention-system/pyclient/pyclient') log.debug('psiphon path: %s' % self.psiphonpath) # psi_client.py can not be run directly because the paths in the @@ -91,6 +91,7 @@ connect(False) self.report['psiphon_installed'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') + self.report['success'] = False self.report['psiphon_installed'] = False log.debug("Adding %s to report" % self.report) # XXX: the original code written by juga0 readed @@ -110,8 +111,7 @@ connect(False) # full with some block size and therefore the test would # terminate with error finished = self.run(self.command, - env=dict(PYTHONPATH=os.path.join(self.psiphonpath, - 'pyclient')), + env=dict(PYTHONPATH=self.psiphonpath), path=self.psiphonpath, usePTY=1)

1 0

[ooni-probe/master] change again command report structure for ProcessTest
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 6e1cc4bb698d361154bf20bc82e44a86754b7c93 Author: juga0 <juga> Date: Thu Dec 24 13:28:38 2015 +0000 change again command report structure for ProcessTest * Change report structure as commented in https://github.com/TheTorProject/ooni-probe/pull/445#issuecomment-166981539 --- ooni/templates/process.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ooni/templates/process.py b/ooni/templates/process.py index b7dafdd..82457dc 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -103,10 +103,14 @@ class ProcessTest(NetTestCase): def processEnded(self, result, command): log.debug("Finished %s: %s" % (command, result)) - self.report['command_name'] = ' '.join(command) - self.report['command_stdout'] = result['stdout'] - self.report['command_stderr'] = result['stderr'] - self.report['command_exit_reason'] = result['exit_reason'] + if not isinstance(self.report.get('commands'), list): + self.report['commands'] = [] + self.report['commands'].append({ + 'command_name': ' '.join(command), + 'command_stdout': result['stdout'], + 'command_stderr': result['stderr'], + 'command_exit_reason': result['exit_reason'], + }) return result def run(self, command, finished=None, env={}, path=None, usePTY=0):

1 0