November 2015 - tor-commits - lists.torproject.org

[ooni-probe/master] change the way HTTP request errors are handled
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 13b06e93f7f80dd6ab100bf5edb3aa2bff620628 Author: juga0 <juga> Date: Thu Sep 24 23:03:30 2015 +0000 change the way HTTP request errors are handled * eliminate addFailureToReport because the errors are handled by HTTPTest errback and addToReport * add ConnectionDone exception to handleAllErrors, so that an HTTP request to a domain that is not reachable is reported --- ooni/errors.py | 8 ++++++-- ooni/nettests/third_party/psiphon.py | 16 ++-------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 1368f45..cb18fe9 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -6,7 +6,7 @@ from twisted.web.error import Error from twisted.internet.error import ConnectionRefusedError, TCPTimedOutError from twisted.internet.error import DNSLookupError, ConnectError, ConnectionLost from twisted.internet.error import TimeoutError as GenericTimeoutError -from twisted.internet.error import ProcessDone +from twisted.internet.error import ProcessDone, ConnectionDone from twisted.python import usage @@ -50,7 +50,8 @@ def handleAllFailures(failure): CommandNotSupported, ConnectError, ConnectionLost, - CancelledError) + CancelledError, + ConnectionDone) return failureToString(failure) @@ -148,6 +149,9 @@ def failureToString(failure): elif isinstance(failure.value, ProcessDone): string = 'process_done' + elif isinstance(failure.value, ConnectionDone): + string = 'connection_done' + else: # log.err("Unknown failure type: %s" % type(failure.value)) string = 'unknown_failure %s' % str(failure.value) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index f8cb82a..8ac287d 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -60,7 +60,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): else: # FIXME: use http://google.com? # self.url = 'https://wtfismyip.com/text' - self.url = 'https://check.torproject.orggg' + self.url = 'https://check.torproject.org' if self.localOptions['psiphonpath']: self.psiphonpath = self.localOptions['psiphonpath'] @@ -112,17 +112,6 @@ connect(False) path=self.psiphonpath, usePTY=1) - - def addFailureToReport(failure): - log.debug("PsiphonTest.test_psiphon.addFailureToReport") - log.debug(repr(failure )) - self.report['failure'] = handleAllFailures(failure) - self.report['success'] = False - log.debug("Adding %s to report" % self.report) - # FIXME: these keys are not being wroten in the report - # probably because report is being defined in NetTestCase as - # a class attribute that is initialized again in NetTescase._setUp - def callDoRequest(_): return self.doRequest(self.url) self.bootstrapped.addCallback(callDoRequest) @@ -132,8 +121,7 @@ connect(False) self.processDirector.transport.signalProcess('INT') os.remove(self.command[0]) return finished - - self.bootstrapped.addErrback(addFailureToReport) + self.bootstrapped.addBoth(cleanup) return self.bootstrapped

1 0

[ooni-probe/master] Minor improvements commented on https://github.com/TheTorProject/ooni-probe/pull/410
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 53934923f0531eea4b3e4258d74044466edd66e6 Author: juga0 <juga> Date: Wed Nov 11 12:48:16 2015 +0000 Minor improvements commented on https://github.com/TheTorProject/ooni-probe/pull/410 * add comment about why using usePTY * add comment about why is needed to run psiphon from an script * fix identation --- ooni/nettests/third_party/psiphon.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 6a2e08e..9fec2fd 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -25,10 +25,10 @@ class UsageOptions(usage.Options): class PsiphonTest(httpt.HTTPTest, process.ProcessTest): - + """ This class tests Psiphon python client - + test_psiphon: Starts a Psiphon, check if it bootstraps successfully (print a line in stdout). @@ -51,7 +51,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): log.debug('PiphonTest._setUp: setting socksproxy') self.localOptions['socksproxy'] = '127.0.0.1:1080' super(PsiphonTest, self)._setUp() - + def setUp(self): log.debug('PsiphonTest.setUp') @@ -77,6 +77,8 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): 'psiphon-circumvention-system/pyclient') log.debug('psiphon path: %s' % self.psiphonpath) + # psi_client.py can not be run directly because the paths in the + # code are relative, so it'll fail to execute from this test x = """ from psi_client import connect connect(False) @@ -107,6 +109,10 @@ connect(False) self.report['psiphon_installed'] = True log.debug("Adding %s to report" % self.report) + # Using pty to see output lines as soon as they get wrotten in the + # buffer, otherwise the test might not see lines until the buffer is + # full with some block size and therefore the test would + # terminate with error finished = self.run(self.command, env=dict(PYTHONPATH=self.psiphonpath), path=self.psiphonpath, @@ -121,7 +127,7 @@ connect(False) self.processDirector.transport.signalProcess('INT') os.remove(self.command[1]) return finished - + self.bootstrapped.addBoth(cleanup) return self.bootstrapped

1 0

[ooni-probe/master] Merge pull request #405 from TheTorProject/fix/feature/install
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 2ffdb863a8800ba84a03eac04f2e5e612a35dc63 Merge: ed26fa9 0c636ba Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 31 10:59:13 2015 +0200 Merge pull request #405 from TheTorProject/fix/feature/install Add support for DNF package manager, add required build dependencies, fix indentation scripts/install.sh | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-)

1 0

[ooni-probe/master] add more spawnProcess arguments to run
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 138891558e46983a9cac4faafd7bbe533f9eb7d6 Author: juga0 <juga> Date: Sat Sep 19 19:53:45 2015 +0000 add more spawnProcess arguments to run --- ooni/templates/process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 1c5cb1d..efe53d3 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -105,9 +105,9 @@ class ProcessTest(NetTestCase): } return result - def run(self, command, finished=None): + def run(self, command, finished=None, env={}, path=None, usePTY=0): d = defer.Deferred() d.addCallback(self.processEnded, command) processDirector = ProcessDirector(d, finished, self.timeout) - reactor.spawnProcess(processDirector, command[0], command) + reactor.spawnProcess(self.processDirector, command[0], command, env=env, path=path, usePTY=usePTY) return d

1 0

[ooni-probe/master] Fix line formatting
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 7609ffc64d077e1488edaa76f81dc183ba4f9326 Author: anadahz <kojgelo(a)inbox.com> Date: Thu Sep 10 10:58:47 2015 +0200 Fix line formatting --- scripts/install.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 5deff38..beb9f60 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -327,8 +327,7 @@ case "$lsb_dist" in # Unistall Twisted and install a version >=12.2.0,<=14.0.0 # https://trac.torproject.org/projects/tor/ticket/15944 $sh_c "PYTHONPATH=$PYTHONPATH pip uninstall -y Twisted" - $sh_c "PYTHONPATH=$PYTHONPATH pip install - --install-option=\"--prefix=$PYTHON_PREFIX\" 'Twisted>=12.2.0,<=14.0.0'" + $sh_c "PYTHONPATH=$PYTHONPATH pip install --install-option=\"--prefix=$PYTHON_PREFIX\" 'Twisted>=12.2.0,<=14.0.0'" ) fi

1 0

[ooni-probe/master] check shouldClose in errReceived as outReceived does
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 94eeb20f4ce4c1ff9ef35c3a994e6f42ed670f82 Author: juga0 <juga> Date: Sat Sep 19 19:37:13 2015 +0000 check shouldClose in errReceived as outReceived does --- ooni/templates/process.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 4939441..1c5cb1d 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -63,6 +63,9 @@ class ProcessDirector(protocol.ProcessProtocol): def errReceived(self, data): log.debug("STDERR: %s" % data) self.stderr += data + if self.shouldClose(): + self.close("condition_met") + def inConnectionLost(self): log.debug("inConnectionLost")

1 0

[ooni-probe/master] Workaround: uninstall/install a Twisted version >=12.2.0, <=14.0.0
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 8d136def559fd25fdc22749474eddba169a8bef8 Author: anadahz <kojgelo(a)inbox.com> Date: Wed Sep 9 14:46:06 2015 +0200 Workaround: uninstall/install a Twisted version >=12.2.0,<=14.0.0 By default pip grabs the latest Twisted version (15.x) which is not currently supported by ooniprobe. https://trac.torproject.org/projects/tor/ticket/15944 --- scripts/install.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/install.sh b/scripts/install.sh index 01318ce..7897471 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -324,6 +324,11 @@ case "$lsb_dist" in set -x $sh_c 'apt-get install -y -q curl git-core python python-dev python-setuptools build-essential libdumbnet1 python-dumbnet python-libpcap tor tor-geoipdb libgeoip-dev libpcap0.8-dev libssl-dev libffi-dev libdumbnet-dev' $sh_c "PYTHONPATH=$PYTHONPATH pip install --install-option=\"--prefix=$PYTHON_PREFIX\" ooniprobe" + # Unistall Twisted and install a version >=12.2.0,<=14.0.0 + # https://trac.torproject.org/projects/tor/ticket/15944 + $sh_c "PYTHONPATH=$PYTHONPATH pip uninstall Twisted" + $sh_c "PYTHONPATH=$PYTHONPATH pip install + --install-option=\"--prefix=$PYTHON_PREFIX\" 'Twisted>=12.2.0,<=14.0.0'" ) fi

1 0

[ooni-probe/master] Fix typo in exporting of PYTHONPATH
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 6c29f698799a7e34ed3023f2f2854de3e2c3d461 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 6 12:26:32 2015 +0200 Fix typo in exporting of PYTHONPATH --- scripts/install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index a63c506..01318ce 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -250,7 +250,7 @@ install_pluggable_transports() { install_pluggable_transport_deps ( set -x - $sh_c "PYTHONPATH=$(PYTHONPATH) pip install --install-option=\"--prefix=$PYTHON_PREFIX\" obfsproxy fteproxy" + $sh_c "PYTHONPATH=$PYTHONPATH pip install --install-option=\"--prefix=$PYTHON_PREFIX\" obfsproxy fteproxy" ) install_obfs4proxy install_meek @@ -270,7 +270,7 @@ case "$lsb_dist" in $sh_c "${yum} -y groupinstall \"Development tools\"" $sh_c "${yum} -y install zlib-devel bzip2-devel openssl-devel sqlite-devel libpcap-devel libffi-devel libevent-devel GeoIP-devel tor python-devel libdnet-devel gcc-c++" install_pip - $sh_c "PYTHONPATH=$(PYTHONPATH) pip install --install-option=\"--prefix=$PYTHON_PREFIX\" ooniprobe" + $sh_c "PYTHONPATH=$PYTHONPATH pip install --install-option=\"--prefix=$PYTHON_PREFIX\" ooniprobe" ) install_pluggable_transports @@ -323,7 +323,7 @@ case "$lsb_dist" in ( set -x $sh_c 'apt-get install -y -q curl git-core python python-dev python-setuptools build-essential libdumbnet1 python-dumbnet python-libpcap tor tor-geoipdb libgeoip-dev libpcap0.8-dev libssl-dev libffi-dev libdumbnet-dev' - $sh_c "PYTHONPATH=$(PYTHONPATH) pip install --install-option=\"--prefix=$PYTHON_PREFIX\" ooniprobe" + $sh_c "PYTHONPATH=$PYTHONPATH pip install --install-option=\"--prefix=$PYTHON_PREFIX\" ooniprobe" ) fi

1 0

[ooni-probe/master] Bug: add notes to uninstall/downgrade Twisted via pip
by art＠torproject.org 20 Nov '15

20 Nov '15

commit c35d1f639ba987dded61073e94dfbd8cc68ef702 Author: anadahz <kojgelo(a)inbox.com> Date: Fri Sep 11 10:15:28 2015 +0200 Bug: add notes to uninstall/downgrade Twisted via pip pypi ooniprobe requirements installs the latest version of Twisted Currently due to a bug in Twisted ooniprobe requires with Twisted versions >=12.2.0 and <=14.0.0 --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 2934c20..66b712c 100644 --- a/README.rst +++ b/README.rst @@ -60,6 +60,12 @@ On unix systems:: sudo pip install ooniprobe +**BUG** Note: Twisted version needs to be >=12.2.0 and <=14.0.0 +unistall/downgrage Twisted:: + + sudo pip uninstall Twisted + sudo pip install 'Twisted>=12.2.0,<=14.0.0' + To install it from the current master run:: sudo pip install https://github.com/TheTorProject/ooni-probe/archive/master.zip

1 0

[ooni-probe/master] Fix developer script
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 52b02e0e863dec2969e0858780bbd3f5fc0f20e6 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Sep 24 15:51:36 2015 +0000 Fix developer script --- bin/ooniprobe-dev | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ooniprobe-dev b/bin/ooniprobe-dev index e0d5abb..b85d3a3 100755 --- a/bin/ooniprobe-dev +++ b/bin/ooniprobe-dev @@ -2,5 +2,5 @@ # Developer script for running ooniprobe directly from the repository. # We don't automatically add "$PWD" to PYTHONPATH as that is a security risk # when run as /usr/bin/ooniprobe on an end user's computer. -cd "$(realpath "$(dirname "$0")")" -PYTHONPATH="$PWD" exec ./bin/ooniprobe +cd "$(realpath "$(dirname "$0")")/.." +PYTHONPATH="$PWD" exec ./bin/ooniprobe $@

1 0