January 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] add success key to the report after doRequest
by art＠torproject.org 30 Jan '16

30 Jan '16

commit 36c4e194b4597323cdac78aac798e9945e922429 Author: juga0 <juga> Date: Tue Dec 22 21:14:21 2015 +0000 add success key to the report after doRequest --- ooni/nettests/third_party/psiphon.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 3129d5e..94dfa51 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -116,7 +116,19 @@ connect(False) usePTY=1) def callDoRequest(_): - return self.doRequest(self.url) + log.debug("PsiphonTest.callDoRequest: %r" %(_,)) + d = self.doRequest(self.url) + def addSuccessToReport(res): + log.debug("PsiphonTest.callDoRequest.addSuccessToReport") + self.report['success'] = True + return res + d.addCallback(addSuccessToReport) + def addFailureToReport(res): + log.debug("PsiphonTest.callDoRequest.addFailureToReport. res=%r" % (res,)) + self.report['success'] = False + return res + d.addErrback(addFailureToReport) + return d self.bootstrapped.addCallback(callDoRequest) def cleanup(_):

1 0

[ooni-probe/master] fixed identation
by art＠torproject.org 30 Jan '16

30 Jan '16

commit 2bb4bd95770905c5111179c853417d530e623bba Author: juga0 <juga> Date: Tue Dec 22 21:22:55 2015 +0000 fixed identation --- ooni/nettests/third_party/psiphon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 94dfa51..17c83c6 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -88,7 +88,7 @@ connect(False) log.debug('PsiphonTest.test_psiphon') self.report['success'] = None - self.report['psiphon_installed'] = None + self.report['psiphon_installed'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') self.report['success'] = False

1 0

[ooni-probe/master] change new psiphon path to the psiphonpath attribute
by art＠torproject.org 30 Jan '16

30 Jan '16

commit 5746afdc25515c553d100d9e77c579b8d72d7d34 Author: juga0 <juga> Date: Tue Dec 22 21:10:13 2015 +0000 change new psiphon path to the psiphonpath attribute --- ooni/nettests/third_party/psiphon.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 1e85c78..3129d5e 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -63,7 +63,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): # are in the user's home directory. from os import path, getenv self.psiphonpath = path.join( - getenv('HOME'), 'psiphon-circumvention-system/pyclient') + getenv('HOME'), 'psiphon-circumvention-system/pyclient/pyclient') log.debug('psiphon path: %s' % self.psiphonpath) # psi_client.py can not be run directly because the paths in the @@ -91,6 +91,7 @@ connect(False) self.report['psiphon_installed'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') + self.report['success'] = False self.report['psiphon_installed'] = False log.debug("Adding %s to report" % self.report) # XXX: the original code written by juga0 readed @@ -110,8 +111,7 @@ connect(False) # full with some block size and therefore the test would # terminate with error finished = self.run(self.command, - env=dict(PYTHONPATH=os.path.join(self.psiphonpath, - 'pyclient')), + env=dict(PYTHONPATH=self.psiphonpath), path=self.psiphonpath, usePTY=1)

1 0

[ooni-probe/master] leave success key to none in case psiphon fails
by art＠torproject.org 30 Jan '16

30 Jan '16

commit 694a7e451a10f89cb5b58a715ff49c2c34375b24 Author: juga0 <juga> Date: Wed Dec 23 16:18:59 2015 +0000 leave success key to none in case psiphon fails --- ooni/nettests/third_party/psiphon.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 17c83c6..9314132 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -91,7 +91,6 @@ connect(False) self.report['psiphon_installed'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') - self.report['success'] = False self.report['psiphon_installed'] = False log.debug("Adding %s to report" % self.report) # XXX: the original code written by juga0 readed

1 0

[ooni-probe/master] add variables for paths, change variable names
by art＠torproject.org 30 Jan '16

30 Jan '16

commit 16bb4478c33ee80bff9a364e950ee98818129a4d Author: juga0 <juga> Date: Fri Dec 25 19:54:31 2015 +0000 add variables for paths, change variable names --- scripts/psiphon_install.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh index 12086c8..7624c12 100755 --- a/scripts/psiphon_install.sh +++ b/scripts/psiphon_install.sh @@ -2,12 +2,14 @@ set -e -PSIPHON_PATH=$HOME -PSIPHON_PYCLIENT_PATH=$PSIPHON_PATH/psiphon-circumvention-system/pyclient -PSIPHON_REPO_URL=https://bitbucket.org/psiphon/psiphon-circumvention-system -OONI_VIRTUALENV_PATH=$HOME/.virtualenvs/ooniprobe +PSIPHON_HOME_PATH=$HOME +PSIPHON_PYCLIENT_PATH=$PSIPHON_HOME_PATH/psiphon-circumvention-system/pyclient +PSIPHON_SSH_PATH=$PSIPHON_HOME_PATH/psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1 +PSIPHON_REPO_URL=https://bitbucket.org/psiphon/psiphon-circumvention-system#af438ec2c16c +VIRTUALENVS_PATH=$HOME/.virtualenvs +OONI_VIRTUALENV_PATH=$VIRTUALENVS_PATH/ooniprobe -mkdir -p $PSIPHON_PATH +mkdir -p $PSIPHON_HOME_PATH command_exists() { command -v "$@" > /dev/null 2>&1 @@ -40,7 +42,7 @@ if ! command_exists hg; then fi echo "[D] mercurial installed" -cd $PSIPHON_PATH +cd $PSIPHON_HOME_PATH if [ ! -d "psiphon-circumvention-system" ]; then echo "[D] cloning psiphon repository" hg clone $PSIPHON_REPO_URL @@ -51,10 +53,10 @@ echo "[D] psiphon repository cloned" # optional, compile their ssh if [ ! -f "$PSIPHON_PYCLIENT_PATH/ssh" ]; then echo "[D] compiling psiphon ssh" - cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ + cd $PSIPHON_SSH_PATH ./configure make - mv ssh ../../../pyclient/pyclient/ + mv ssh $PSIPHON_PYCLIENT_PATH make clean echo "[D] psiphon ssh compiled" fi

1 0

[ooni-probe/master] add/change report variable names
by art＠torproject.org 30 Jan '16

30 Jan '16

commit c9d1d950e76ccba69a3b7373b99e26be4b679ee9 Author: juga0 <juga> Date: Thu Dec 24 13:42:34 2015 +0000 add/change report variable names * add bootstrapped_success report key with value true when the text 'Press Ctrl-C to terminate.' is found, false when psiphon runs but is not found and none when psiphon does not run * change success report key to request_success * keep psiphon_installed key, but change the name to psiphon_found --- ooni/nettests/third_party/psiphon.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index a25e955..bfdec70 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -82,17 +82,21 @@ connect(False) def handleRead(self, stdout, stderr): if 'Press Ctrl-C to terminate.' in self.processDirector.stdout: if not self.bootstrapped.called: + # here the text 'Press Ctrl-C to terminate.' has been found + # and it was to call doRequest + self.report['bootstrapped_success'] = True log.debug("PsiphonTest: calling bootstrapped.callback") self.bootstrapped.callback(None) def test_psiphon(self): log.debug('PsiphonTest.test_psiphon') - self.report['success'] = None - self.report['psiphon_installed'] = None + self.report['bootstrapped_success'] = None + self.report['request_success'] = None + self.report['psiphon_found'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') - self.report['psiphon_installed'] = False + self.report['psiphon_found'] = False log.debug("Adding %s to report" % self.report) # XXX: the original code written by juga0 readed # > return defer.succeed(None) @@ -103,7 +107,7 @@ connect(False) reactor.callLater(0.0, self.bootstrapped.callback, None) return self.bootstrapped - self.report['psiphon_installed'] = True + self.report['psiphon_found'] = True log.debug("Adding %s to report" % self.report) # Using pty to see output lines as soon as they get wrotten in the @@ -114,18 +118,22 @@ connect(False) env=dict(PYTHONPATH=self.psiphonpath), path=self.psiphonpath, usePTY=1) + # here psiphon command has been run, and if it finds the text + # 'Press Ctrl-C to terminate' in handleRead it will write to the + # report self.report['bootstrapped_success'] = True + self.report['bootstrapped_success'] = False def callDoRequest(_): log.debug("PsiphonTest.callDoRequest: %r" %(_,)) d = self.doRequest(self.url) def addSuccessToReport(res): log.debug("PsiphonTest.callDoRequest.addSuccessToReport") - self.report['success'] = True + self.report['request_success'] = True return res d.addCallback(addSuccessToReport) def addFailureToReport(res): log.debug("PsiphonTest.callDoRequest.addFailureToReport. res=%r" % (res,)) - self.report['success'] = False + self.report['request_success'] = False return res d.addErrback(addFailureToReport) return d

1 0

[ooni-probe/master] changed report structure for ProcessTest
by art＠torproject.org 30 Jan '16

30 Jan '16

commit d8b4121c2867c60af03c0129c4b14696c646ad55 Author: juga0 <juga> Date: Wed Dec 23 16:51:35 2015 +0000 changed report structure for ProcessTest --- ooni/templates/process.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 7ec4b2e..b7dafdd 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -103,12 +103,10 @@ class ProcessTest(NetTestCase): def processEnded(self, result, command): log.debug("Finished %s: %s" % (command, result)) - key = ' '.join(command) - self.report[key] = { - 'stdout': result['stdout'], - 'stderr': result['stderr'], - 'exit_reason': result['exit_reason'] - } + self.report['command_name'] = ' '.join(command) + self.report['command_stdout'] = result['stdout'] + self.report['command_stderr'] = result['stderr'] + self.report['command_exit_reason'] = result['exit_reason'] return result def run(self, command, finished=None, env={}, path=None, usePTY=0):

1 0

[ooni-probe/master] catch ProcessExitedAlready exception during cleanup
by art＠torproject.org 30 Jan '16

30 Jan '16

commit 23e22f9e9d1821ad052687d3956f4a35b591f336 Author: juga0 <juga> Date: Wed Dec 23 16:31:52 2015 +0000 catch ProcessExitedAlready exception during cleanup --- ooni/nettests/third_party/psiphon.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 9314132..a25e955 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -3,6 +3,7 @@ import os import sys from twisted.internet import defer, reactor +from twisted.internet.error import ProcessExitedAlready from twisted.python import usage from ooni.utils import log @@ -132,7 +133,10 @@ connect(False) def cleanup(_): log.debug('PsiphonTest:cleanup') - self.processDirector.transport.signalProcess('INT') + try: + self.processDirector.transport.signalProcess('INT') + except ProcessExitedAlready: + pass os.remove(self.command[1]) return finished

1 0

[ooni-probe/master] change again command report structure for ProcessTest
by art＠torproject.org 30 Jan '16

30 Jan '16

commit e5a532658904a06e9167d89a9291896cc3c1b620 Author: juga0 <juga> Date: Thu Dec 24 13:28:38 2015 +0000 change again command report structure for ProcessTest * Change report structure as commented in https://github.com/TheTorProject/ooni-probe/pull/445#issuecomment-166981539 --- ooni/templates/process.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ooni/templates/process.py b/ooni/templates/process.py index b7dafdd..82457dc 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -103,10 +103,14 @@ class ProcessTest(NetTestCase): def processEnded(self, result, command): log.debug("Finished %s: %s" % (command, result)) - self.report['command_name'] = ' '.join(command) - self.report['command_stdout'] = result['stdout'] - self.report['command_stderr'] = result['stderr'] - self.report['command_exit_reason'] = result['exit_reason'] + if not isinstance(self.report.get('commands'), list): + self.report['commands'] = [] + self.report['commands'].append({ + 'command_name': ' '.join(command), + 'command_stdout': result['stdout'], + 'command_stderr': result['stderr'], + 'command_exit_reason': result['exit_reason'], + }) return result def run(self, command, finished=None, env={}, path=None, usePTY=0):

1 0

[ooni-probe/master] fix checking if commands exist, other minor changes
by art＠torproject.org 30 Jan '16

30 Jan '16

commit b955184f3689de91b9a4b792e46ceb144e03f161 Author: juga0 <juga> Date: Wed Dec 23 19:50:07 2015 +0000 fix checking if commands exist, other minor changes * fix checking if commands exist * only install virtualenv if it is needed to create a new one * add/change debug ouput --- scripts/psiphon_install.sh | 50 ++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh index 117c823..12086c8 100755 --- a/scripts/psiphon_install.sh +++ b/scripts/psiphon_install.sh @@ -20,8 +20,10 @@ sh_c='sh -c' if [ "$user" != 'root' ]; then if command_exists sudo; then sh_c='sudo sh -c -E' + echo "[D] using sudo" elif command_exists su; then sh_c='su -c --preserve-environment' + echo "[D] using su" else echo >&2 'Error: this installer needs the ability to run commands as root.' echo >&2 'We are unable to find either "sudo" or "su" available to make this happen.' @@ -29,42 +31,49 @@ if [ "$user" != 'root' ]; then fi fi -echo "installing dependencies" +echo "[D] installing dependencies" $sh_c "apt-get -y install zlib1g-dev libssl-dev" -if [ ! "command_exists hg" ]; then +if ! command_exists hg; then + echo "[D] installing mercurial" $sh_c "apt-get -y install mercurial" fi +echo "[D] mercurial installed" cd $PSIPHON_PATH if [ ! -d "psiphon-circumvention-system" ]; then - echo "cloning psiphon repository" + echo "[D] cloning psiphon repository" hg clone $PSIPHON_REPO_URL fi -echo "psiphon repository cloned" +echo "[D] psiphon repository cloned" # optional, compile their ssh if [ ! -f "$PSIPHON_PYCLIENT_PATH/ssh" ]; then - echo "compiling psiphon ssh" + echo "[D] compiling psiphon ssh" cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ ./configure make - mv ssh ../../../pyclient/ + mv ssh ../../../pyclient/pyclient/ make clean - echo "psiphon ssh compiled" + echo "[D] psiphon ssh compiled" fi # check if we are in a virtualenv, create it otherwise -echo "checking virtualenv" +echo "[D] checking virtualenv" if [ `python -c 'import sys; print hasattr(sys, "real_prefix")'` = "False" ]; then - # not in a virtualenv - # create a virtualenv - # FIXME: assuming debian version will have secure pip/virtualenv - if [ ! "command_exists virtualenv" ]; then - $sh_c "apt-get -y install python-virtualenv" - fi + echo "[D] not in a virtualenv" if [ ! -f $OONI_VIRTUALENV_PATH/bin/activate ]; then + echo "[D] virtualenv not found" + # create a virtualenv + # FIXME: assuming debian version will have secure pip/virtualenv + if ! command_exists virtualenv; then + echo "[D] installing virtualenv" + $sh_c "apt-get -y install python-virtualenv" + else + echo "[D] virtualenv command found" + fi + echo "[D] creating a virtualenv" # Set up the virtual environment mkdir -p $HOME/.virtualenvs virtualenv $OONI_VIRTUALENV_PATH @@ -72,12 +81,12 @@ if [ `python -c 'import sys; print hasattr(sys, "real_prefix")'` = "False" ]; th else . $OONI_VIRTUALENV_PATH/bin/activate fi + echo "[D] virtualenv activated" fi -echo "virtualenv activated" # create psi_client.dat -echo "creating servers data file" -echo "installing dependencies to create servers data file" +echo "[D] creating servers data file" +echo "[D] installing dependencies to create servers data file" pip install -v --timeout 60 wget cd /tmp cat <<EOF > psi_generate_dat.py @@ -107,9 +116,12 @@ EOF chmod +x psi_generate_dat.py ./psi_generate_dat.py -echo "servers data file created" +echo "[D] servers data file created" mv psi_client.dat $PSIPHON_PYCLIENT_PATH rm /tmp/psi_generate_dat.py -echo "[+] Installing all of the Python dependency requirements with pip in your virtualenv!"; +echo "[D] installing all of the Python dependency requirements with pip in the virtualenv"; pip install -v --timeout 60 jsonpickle pexpect + +echo "You can now run Psiphon: cd ~/psiphon-circumvention-system/pyclient/pyclient;python psi_client.py" +echo "NOTE that if OONI is not installed, you will not be able to run OONI Psiphon test"

1 0