November 2015 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge pull request #416 from TheTorProject/fix/bin-ooniprobe-dev
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 8c6dad7633308a9fc6564b5087ec2a0622ea3e30 Merge: f183629 8c53b36 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Nov 8 10:18:19 2015 +0100 Merge pull request #416 from TheTorProject/fix/bin-ooniprobe-dev Improve ./bin/ooniprobe-dev bin/ooniprobe-dev | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)

1 0

[ooni-probe/master] Merge pull request #401 from TheTorProject/fix/twisted15.0
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 8449d674f69c8ca740631fab534935fc87265af2 Merge: 866ed35 b1cf6c4 Author: Simone Basso <bassosimone(a)gmail.com> Date: Fri Nov 13 08:57:08 2015 +0100 Merge pull request #401 from TheTorProject/fix/twisted15.0 Fix/twisted15.0 ooni/reporter.py | 2 +- ooni/tests/test_oonicli.py | 2 ++ ooni/utils/hacks.py | 27 --------------------------- ooni/utils/trueheaders.py | 7 ++++--- requirements.txt | 2 +- 5 files changed, 8 insertions(+), 32 deletions(-)

1 0

[ooni-probe/master] Update README.rst
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 4fdc3c376dc041836d563b0c219645fc88258698 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Nov 11 19:12:25 2015 +0100 Update README.rst --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index 5c043cd..930426c 100644 --- a/README.rst +++ b/README.rst @@ -394,3 +394,11 @@ On Debian based systems a development environment can be setup as follows: (prer ooniprobe -s # if all went well, lists available tests +Donate +------- + +Send bitcoins to: +![bitcoin address](http://i.imgur.com/CIWHb5R.png) +``` +1Ai9d4dhDBjxYVkKKf1pFXptEGfM1vxFBf +```

1 0

[ooni-probe/master] psiphon: use the python provided by sys.executable
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 4b7f72746a10c1e48df93a3c62c84fcb1d6f2bb2 Author: Simone Basso <bassosimone(a)gmail.com> Date: Sat Oct 10 11:06:11 2015 +0200 psiphon: use the python provided by sys.executable There may be more than one Python (for example on MacOS I have both the system Python and the one installed with brew). Make sure we use the same Python with which ooni-probe was started such that modules installed with pip are found. Updated to avoid setting the temporary file executable and to avoid removing my Python interpreter :). --- ooni/nettests/third_party/psiphon.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 8ac287d..6a2e08e 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -1,6 +1,7 @@ import tempfile import stat import os +import sys from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint @@ -76,15 +77,14 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): 'psiphon-circumvention-system/pyclient') log.debug('psiphon path: %s' % self.psiphonpath) - x = """#!/usr/bin/env python + x = """ from psi_client import connect connect(False) """ f = tempfile.NamedTemporaryFile(delete=False) f.write(x) f.close() - os.chmod(f.name, os.stat(f.name).st_mode | stat.S_IEXEC) - self.command = [f.name] + self.command = [sys.executable, f.name] log.debug('command: %s' % ''.join(self.command)) def handleRead(self, stdout, stderr): @@ -119,7 +119,7 @@ connect(False) def cleanup(_): log.debug('PsiphonTest:cleanup') self.processDirector.transport.signalProcess('INT') - os.remove(self.command[0]) + os.remove(self.command[1]) return finished self.bootstrapped.addBoth(cleanup)

1 0

[ooni-probe/master] Remove script to install psiphon
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 954452ba8f38569757a519bae2307001a91e7378 Author: Simone Basso <bassosimone(a)gmail.com> Date: Sun Nov 15 16:11:18 2015 +0100 Remove script to install psiphon We will write a more ad-hoc script to install psiphon as part of the lepidopter install process of OONI. This way we don't need to worry about portability to MacOS and similar issues since we know we'll be on debian. --- scripts/psiphon_install.sh | 51 -------------------------------------------- 1 file changed, 51 deletions(-) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh deleted file mode 100644 index e03e663..0000000 --- a/scripts/psiphon_install.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -PSIPHON_PATH=$HOME -PSIPHON_PYCLIENT_PATH=$PSIPHON_PATH/psiphon-circumvention-system/pyclient - -sudo apt-get install mercurial -cd $PSIPHON_PATH -hg clone https://bitbucket.org/psiphon/psiphon-circumvention-system -# optional, compile their ssh -#cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ -#./configure -#make -#cp ssh ../../../pyclient/ -#cd $PSIPHON_PATH - -# create psi_client.dat -cat <<EOF > psi_generate_dat.py -#!/usr/bin/env python - -import wget -import os -import json - -# Delete 'server_list' if exists -if os.path.exists("server_list"): - # os.remove("server_list") - # os.rename("server_list", "server_list") - pass -else: - # Download 'server_list' - url ="https://psiphon3.com/server_list' - wget.download(url) - -# convert server_list to psi_client.dat -dat = {} -dat["propagation_channel_id"] = "FFFFFFFFFFFFFFFF" -dat["sponsor_id"] = "FFFFFFFFFFFFFFFF" -dat["servers"] = json.load(open('server_list'))['data'].split() -json.dump(dat, open('psi_client.dat', 'w')) -EOF - -chmod +x psi_generate_dat.py -./psi_generate_dat.py -cp psi_client.dat $PSIPHON_PYCLIENT_PATH - -# assuming to be inside a virtualenv -pip install jsonpickle pexpect - -# run psiphon -# cd $PSIPHON_PYCLIENT_PATH -# psi_client.py

1 0

[ooni-probe/master] psiphon.py: prepare for merge with master
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 61a6b2af1f77f07232eded4d5e7971b162aef987 Author: Simone Basso <bassosimone(a)gmail.com> Date: Sun Nov 15 15:58:54 2015 +0100 psiphon.py: prepare for merge with master - for now check.torprokect.org will do - for now it's OK to keep psiphon in the home directory - couple of cosmetic changes --- ooni/nettests/third_party/psiphon.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 9fec2fd..637a505 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -59,22 +59,18 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): if self.localOptions['url']: self.url = self.localOptions['url'] else: - # FIXME: use http://google.com? - # self.url = 'https://wtfismyip.com/text' self.url = 'https://check.torproject.org' if self.localOptions['psiphonpath']: self.psiphonpath = self.localOptions['psiphonpath'] else: - # FIXME: search for pyclient path instead of assuming is in the - # home? - # psiphon is not installable and to run it manually, it has to be + # Psiphon is not installable and to run it manually, it has to be # run from the psiphon directory, so it wouldn't make sense to - # nstall it in the PATH - from os import path, getenv + # install it in the PATH. For now, we assume that Psiphon sources + # are in the user's home directory. + from os import path, getenv self.psiphonpath = path.join( - getenv('HOME'), - 'psiphon-circumvention-system/pyclient') + getenv('HOME'), 'psiphon-circumvention-system/pyclient') log.debug('psiphon path: %s' % self.psiphonpath) # psi_client.py can not be run directly because the paths in the @@ -92,7 +88,7 @@ connect(False) def handleRead(self, stdout, stderr): if 'Press Ctrl-C to terminate.' in self.processDirector.stdout: if not self.bootstrapped.called: - log.debug("PsiphonTest.test_psiphon: calling bootstrapped.callback") + log.debug("PsiphonTest: calling bootstrapped.callback") self.bootstrapped.callback(None) def test_psiphon(self): @@ -114,9 +110,9 @@ connect(False) # full with some block size and therefore the test would # terminate with error finished = self.run(self.command, - env=dict(PYTHONPATH=self.psiphonpath), - path=self.psiphonpath, - usePTY=1) + env=dict(PYTHONPATH=self.psiphonpath), + path=self.psiphonpath, + usePTY=1) def callDoRequest(_): return self.doRequest(self.url) @@ -130,6 +126,3 @@ connect(False) self.bootstrapped.addBoth(cleanup) return self.bootstrapped - - -

1 0

[ooni-probe/master] Merge pull request #419 from TheTorProject/doc/enhance-test-documentation
by art＠torproject.org 20 Nov '15

20 Nov '15

commit d14f7f668d1e95b0b44de895395e73c0ae301ad5 Merge: 7bf8c8b 054625f Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Nov 9 10:50:03 2015 +0100 Merge pull request #419 from TheTorProject/doc/enhance-test-documentation Add Bridge Reachability Test documentation docs/source/tests/bridge_reachability.rst | 129 +++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+)

1 0

[ooni-probe/master] add psiphon_install.sh, code should be improved
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 3eeaa200e29ea4dbfea36a8b68302321aefe5d4e Author: juga0 <juga> Date: Sun Sep 20 21:23:47 2015 +0000 add psiphon_install.sh, code should be improved --- scripts/psiphon_install.sh | 51 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh new file mode 100644 index 0000000..e03e663 --- /dev/null +++ b/scripts/psiphon_install.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +PSIPHON_PATH=$HOME +PSIPHON_PYCLIENT_PATH=$PSIPHON_PATH/psiphon-circumvention-system/pyclient + +sudo apt-get install mercurial +cd $PSIPHON_PATH +hg clone https://bitbucket.org/psiphon/psiphon-circumvention-system +# optional, compile their ssh +#cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ +#./configure +#make +#cp ssh ../../../pyclient/ +#cd $PSIPHON_PATH + +# create psi_client.dat +cat <<EOF > psi_generate_dat.py +#!/usr/bin/env python + +import wget +import os +import json + +# Delete 'server_list' if exists +if os.path.exists("server_list"): + # os.remove("server_list") + # os.rename("server_list", "server_list") + pass +else: + # Download 'server_list' + url ="https://psiphon3.com/server_list' + wget.download(url) + +# convert server_list to psi_client.dat +dat = {} +dat["propagation_channel_id"] = "FFFFFFFFFFFFFFFF" +dat["sponsor_id"] = "FFFFFFFFFFFFFFFF" +dat["servers"] = json.load(open('server_list'))['data'].split() +json.dump(dat, open('psi_client.dat', 'w')) +EOF + +chmod +x psi_generate_dat.py +./psi_generate_dat.py +cp psi_client.dat $PSIPHON_PYCLIENT_PATH + +# assuming to be inside a virtualenv +pip install jsonpickle pexpect + +# run psiphon +# cd $PSIPHON_PYCLIENT_PATH +# psi_client.py

1 0

[ooni-probe/master] Update README.rst
by art＠torproject.org 20 Nov '15

20 Nov '15

commit d99aa31cca332d4147c61fc6aa20791d89be97df Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Nov 11 19:13:55 2015 +0100 Update README.rst --- README.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 930426c..e4174d3 100644 --- a/README.rst +++ b/README.rst @@ -398,7 +398,8 @@ Donate ------- Send bitcoins to: -![bitcoin address](http://i.imgur.com/CIWHb5R.png) -``` -1Ai9d4dhDBjxYVkKKf1pFXptEGfM1vxFBf -``` + +.. image:: http://i.imgur.com/CIWHb5R.png + :target: http://www.coindesk.com/information/how-can-i-buy-bitcoins/ + + 1Ai9d4dhDBjxYVkKKf1pFXptEGfM1vxFBf

1 0

[ooni-probe/master] change the way HTTP request errors are handled
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 13b06e93f7f80dd6ab100bf5edb3aa2bff620628 Author: juga0 <juga> Date: Thu Sep 24 23:03:30 2015 +0000 change the way HTTP request errors are handled * eliminate addFailureToReport because the errors are handled by HTTPTest errback and addToReport * add ConnectionDone exception to handleAllErrors, so that an HTTP request to a domain that is not reachable is reported --- ooni/errors.py | 8 ++++++-- ooni/nettests/third_party/psiphon.py | 16 ++-------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 1368f45..cb18fe9 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -6,7 +6,7 @@ from twisted.web.error import Error from twisted.internet.error import ConnectionRefusedError, TCPTimedOutError from twisted.internet.error import DNSLookupError, ConnectError, ConnectionLost from twisted.internet.error import TimeoutError as GenericTimeoutError -from twisted.internet.error import ProcessDone +from twisted.internet.error import ProcessDone, ConnectionDone from twisted.python import usage @@ -50,7 +50,8 @@ def handleAllFailures(failure): CommandNotSupported, ConnectError, ConnectionLost, - CancelledError) + CancelledError, + ConnectionDone) return failureToString(failure) @@ -148,6 +149,9 @@ def failureToString(failure): elif isinstance(failure.value, ProcessDone): string = 'process_done' + elif isinstance(failure.value, ConnectionDone): + string = 'connection_done' + else: # log.err("Unknown failure type: %s" % type(failure.value)) string = 'unknown_failure %s' % str(failure.value) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index f8cb82a..8ac287d 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -60,7 +60,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): else: # FIXME: use http://google.com? # self.url = 'https://wtfismyip.com/text' - self.url = 'https://check.torproject.orggg' + self.url = 'https://check.torproject.org' if self.localOptions['psiphonpath']: self.psiphonpath = self.localOptions['psiphonpath'] @@ -112,17 +112,6 @@ connect(False) path=self.psiphonpath, usePTY=1) - - def addFailureToReport(failure): - log.debug("PsiphonTest.test_psiphon.addFailureToReport") - log.debug(repr(failure )) - self.report['failure'] = handleAllFailures(failure) - self.report['success'] = False - log.debug("Adding %s to report" % self.report) - # FIXME: these keys are not being wroten in the report - # probably because report is being defined in NetTestCase as - # a class attribute that is initialized again in NetTescase._setUp - def callDoRequest(_): return self.doRequest(self.url) self.bootstrapped.addCallback(callDoRequest) @@ -132,8 +121,7 @@ connect(False) self.processDirector.transport.signalProcess('INT') os.remove(self.command[0]) return finished - - self.bootstrapped.addErrback(addFailureToReport) + self.bootstrapped.addBoth(cleanup) return self.bootstrapped

1 0